diff options
115 files changed, 7379 insertions, 1377 deletions
diff --git a/.gitmodules b/.gitmodules index 91bddb0a3..6e2339fb9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,3 +9,6 @@ [submodule "external/rapidjson"] path = external/rapidjson url = https://github.com/Tencent/rapidjson +[submodule "external/trezor-common"] + path = external/trezor-common + url = https://github.com/trezor/trezor-common.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a685ee8e..8fd5543f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,8 +176,8 @@ if(NOT MANUAL_SUBMODULES) find_package(Git) if(GIT_FOUND) function (check_submodule relative_path) - execute_process(COMMAND git -C ${relative_path} rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE localHead) - execute_process(COMMAND git rev-parse "HEAD:${relative_path}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE checkedHead) + execute_process(COMMAND git rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path} OUTPUT_VARIABLE localHead) + execute_process(COMMAND git rev-parse "HEAD:${relative_path}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE checkedHead) string(COMPARE EQUAL "${localHead}" "${checkedHead}" upToDate) if (upToDate) message(STATUS "Submodule '${relative_path}' is up-to-date") @@ -190,6 +190,7 @@ if(NOT MANUAL_SUBMODULES) check_submodule(external/miniupnp) check_submodule(external/unbound) check_submodule(external/rapidjson) + check_submodule(external/trezor-common) endif() endif() @@ -512,6 +513,16 @@ else (HIDAPI_FOUND) message(STATUS "Could not find HIDAPI") endif() +# Protobuf, optional. Required for TREZOR. +include(FindProtobuf) +find_package(Protobuf) +if(Protobuf_FOUND) + set(HAVE_PROTOBUF 1) + add_definitions(-DHAVE_PROTOBUF=1) +else(Protobuf_FOUND) + message(STATUS "Could not find Protobuf") +endif() + if(MSVC) add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__") # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline") @@ -805,6 +816,9 @@ else() set(DEBUG_FLAGS "${DEBUG_FLAGS} -O0 ") endif() + # At least some CLANGs default to not enough for monero + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftemplate-depth=900") + if(NOT DEFINED USE_LTO_DEFAULT) set(USE_LTO_DEFAULT false) endif() @@ -887,9 +901,9 @@ if(MINGW) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wa,-mbig-obj") set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi;crypt32;bcrypt) if(DEPENDS) - set(ICU_LIBRARIES ${Boost_LOCALE_LIBRARY} sicuio sicuin sicuuc sicudt sicutu iconv) + set(ICU_LIBRARIES sicuio sicuin sicuuc sicudt sicutu iconv) else() - set(ICU_LIBRARIES ${Boost_LOCALE_LIBRARY} icuio icuin icuuc icudt icutu iconv) + set(ICU_LIBRARIES icuio icuin icuuc icudt icutu iconv) endif() elseif(APPLE OR OPENBSD OR ANDROID) set(EXTRA_LIBRARIES "") diff --git a/Dockerfile b/Dockerfile index cd3e7df70..86d833a98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,9 @@ RUN set -ex && \ libtool-bin \ autoconf \ automake \ - bzip2 + bzip2 \ + xsltproc \ + gperf WORKDIR /usr/local @@ -109,6 +111,42 @@ RUN set -ex \ && make check \ && make install +# Udev +ARG UDEV_VERSION=v3.2.6 +ARG UDEV_HASH=0c35b136c08d64064efa55087c54364608e65ed6 +RUN set -ex \ + && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \ + && cd eudev \ + && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \ + && ./autogen.sh \ + && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ + && make \ + && make install + +# Libusb +ARG USB_VERSION=v1.0.22 +ARG USB_HASH=0034b2afdcdb1614e78edaa2a9e22d5936aeae5d +RUN set -ex \ + && git clone https://github.com/libusb/libusb.git -b ${USB_VERSION} \ + && cd libusb \ + && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \ + && ./autogen.sh \ + && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-shared \ + && make \ + && make install + +# Hidapi +ARG HIDAPI_VERSION=hidapi-0.8.0-rc1 +ARG HIDAPI_HASH=40cf516139b5b61e30d9403a48db23d8f915f52c +RUN set -ex \ + && git clone https://github.com/signal11/hidapi -b ${HIDAPI_VERSION} \ + && cd hidapi \ + && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \ + && ./bootstrap \ + && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && make \ + && make install + WORKDIR /src COPY . . @@ -16,6 +16,15 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - Our [Vulnerability Response Process](https://github.com/monero-project/meta/blob/master/VULNERABILITY_RESPONSE_PROCESS.md) encourages responsible disclosure - We are also available via [HackerOne](https://hackerone.com/monero) +## Research + +The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with our researchers if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. + +Our researchers are available on IRC in [#monero-research-lab on Freenode](https://webchat.freenode.net/?randomnick=1&channels=%23monero-research-lab&prompt=1&uio=d4) or by email: + +- Sarang Noether, Ph.D.: [sarang@getmonero.org](mailto:sarang@getmonero.org) or [sarang.noether@protonmail.com](mailto:sarang.noether@protonmail.com); [research repository](https://github.com/SarangNoether/research-lab) +- Surae Noether (Brandon Goodell), Ph.D.: [surae@getmonero.org](mailto:surae@getmonero.org) or [surae.noether@protonmail.com](mailto:surae.noether@protonmail.com); [research repository](https://github.com/b-g-goodell/research-lab) + ## Announcements - You can subscribe to an [announcement listserv](https://lists.getmonero.org) to get critical announcements from the Monero core team. The announcement list can be very helpful for knowing when software updates are needed. @@ -236,7 +245,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch ( ``` sudo /etc/init.d/dphys-swapfile stop sudo nano /etc/dphys-swapfile - CONF_SWAPSIZE=1024 + CONF_SWAPSIZE=2048 sudo /etc/init.d/dphys-swapfile start ``` * If using an external hard disk without an external power supply, ensure it gets enough power to avoid hardware issues when syncing, by adding the line "max_usb_current=1" to /boot/config.txt @@ -270,7 +279,7 @@ If you are using the older Raspbian Jessie image, compiling Monero is a bit more ``` sudo /etc/init.d/dphys-swapfile stop sudo nano /etc/dphys-swapfile - CONF_SWAPSIZE=1024 + CONF_SWAPSIZE=2048 sudo /etc/init.d/dphys-swapfile start ``` diff --git a/cmake/FindReadline.cmake b/cmake/FindReadline.cmake index de9ddc46d..f26911b26 100644 --- a/cmake/FindReadline.cmake +++ b/cmake/FindReadline.cmake @@ -66,7 +66,9 @@ check_function_exists(rl_copy_text HAVE_COPY_TEXT) check_function_exists(rl_filename_completion_function HAVE_COMPLETION_FUNCTION) if(NOT HAVE_COMPLETION_FUNCTION) - set(CMAKE_REQUIRED_LIBRARIES ${Readline_LIBRARY} ${Termcap_LIBRARY}) + if (Readline_LIBRARY) + set(CMAKE_REQUIRED_LIBRARIES ${Readline_LIBRARY} ${Termcap_LIBRARY}) + endif(Readline_LIBRARY) check_function_exists(rl_copy_text HAVE_COPY_TEXT_TC) check_function_exists(rl_filename_completion_function HAVE_COMPLETION_FUNCTION_TC) set(HAVE_COMPLETION_FUNCTION ${HAVE_COMPLETION_FUNCTION_TC}) diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 530f8e636..9100a8db3 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -32,7 +32,9 @@ #include "easylogging++.h" +#undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "default" + #define MAX_LOG_FILE_SIZE 104850000 // 100 MB - 7600 bytes #define MAX_LOG_FILES 50 diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 5dbb7a478..9b03941ee 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -393,7 +393,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) //ask it inside(!) critical region if we still able to go in event wait... size_t cnt = socket_.get_io_service().poll_one(); if(!cnt) - misc_utils::sleep_no_w(0); + misc_utils::sleep_no_w(1); } return true; @@ -889,7 +889,9 @@ POP_WARNINGS { try { - io_service_.run(); + size_t cnt = io_service_.run(); + if (cnt == 0) + misc_utils::sleep_no_w(1); } catch(const std::exception& ex) { diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index 8c6c1a64d..31495ae13 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -39,12 +39,14 @@ namespace misc_utils inline std::string transform_to_escape_sequence(const std::string& src) { static const char escaped[] = "\b\f\n\r\t\v\"\\/"; - if (std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)) == src.end()) + std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)); + if (it == src.end()) return src; std::string res; res.reserve(2 * src.size()); - for(std::string::const_iterator it = src.begin(); it!=src.end(); ++it) + res.assign(src.begin(), it); + for(; it!=src.end(); ++it) { switch(*it) { diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 5b2eafa9a..0307b732c 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -125,16 +125,22 @@ namespace epee { if(is_signed) { - int64_t nval = boost::lexical_cast<int64_t>(val); + errno = 0; + int64_t nval = strtoll(val.c_str(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + val); stg.set_value(name, nval, current_section); }else { - uint64_t nval = boost::lexical_cast<uint64_t >(val); + errno = 0; + uint64_t nval = strtoull(val.c_str(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + val); stg.set_value(name, nval, current_section); } }else { - double nval = boost::lexical_cast<double>(val); + errno = 0; + double nval = strtod(val.c_str(), NULL); + if (errno) throw std::runtime_error("Invalid number: " + val); stg.set_value(name, nval, current_section); } state = match_state_wonder_after_value; @@ -208,12 +214,25 @@ namespace epee match_number2(it, buf_end, val, is_v_float, is_signed_val); if(!is_v_float) { - int64_t nval = boost::lexical_cast<int64_t>(val);//bool res = string_tools::string_to_num_fast(val, nval); - h_array = stg.insert_first_value(name, nval, current_section); + if (is_signed_val) + { + errno = 0; + int64_t nval = strtoll(val.c_str(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + val); + h_array = stg.insert_first_value(name, nval, current_section); + }else + { + errno = 0; + uint64_t nval = strtoull(val.c_str(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + val); + h_array = stg.insert_first_value(name, nval, current_section); + } CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry"); }else { - double nval = boost::lexical_cast<double>(val);//bool res = string_tools::string_to_num_fast(val, nval); + errno = 0; + double nval = strtod(val.c_str(), NULL); + if (errno) throw std::runtime_error("Invalid number: " + val); h_array = stg.insert_first_value(name, nval, current_section); CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry"); } @@ -286,13 +305,24 @@ namespace epee bool insert_res = false; if(!is_v_float) { - int64_t nval = boost::lexical_cast<int64_t>(val); //bool res = string_tools::string_to_num_fast(val, nval); - insert_res = stg.insert_next_value(h_array, nval); - + if (is_signed_val) + { + errno = 0; + int64_t nval = strtoll(val.c_str(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + val); + insert_res = stg.insert_next_value(h_array, nval); + }else + { + errno = 0; + uint64_t nval = strtoull(val.c_str(), NULL, 10); + if (errno) throw std::runtime_error("Invalid number: " + val); + insert_res = stg.insert_next_value(h_array, nval); + } }else { - //TODO: optimize here if need - double nval = boost::lexical_cast<double>(val); //string_tools::string_to_num_fast(val, nval); + errno = 0; + double nval = strtod(val.c_str(), NULL); + if (errno) throw std::runtime_error("Invalid number: " + val); insert_res = stg.insert_next_value(h_array, nval); } CHECK_AND_ASSERT_THROW_MES(insert_res, "Failed to insert next value"); diff --git a/contrib/epee/src/mlocker.cpp b/contrib/epee/src/mlocker.cpp index 5573d591a..c3262e8f4 100644 --- a/contrib/epee/src/mlocker.cpp +++ b/contrib/epee/src/mlocker.cpp @@ -84,8 +84,8 @@ namespace epee boost::mutex &mlocker::mutex() { - static boost::mutex vmutex; - return vmutex; + static boost::mutex *vmutex = new boost::mutex(); + return *vmutex; } std::map<size_t, unsigned int> &mlocker::map() { diff --git a/contrib/epee/src/network_throttle-detail.cpp b/contrib/epee/src/network_throttle-detail.cpp index 28c85bb78..6f727a1cf 100644 --- a/contrib/epee/src/network_throttle-detail.cpp +++ b/contrib/epee/src/network_throttle-detail.cpp @@ -150,6 +150,7 @@ network_throttle::network_throttle(const std::string &nameshort, const std::stri m_any_packet_yet = false; m_slot_size = 1.0; // hard coded in few places m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle + m_last_sample_time = 0; } void network_throttle::set_name(const std::string &name) diff --git a/external/easylogging++/ea_config.h b/external/easylogging++/ea_config.h index 4c74925d3..4fb48ce3e 100644 --- a/external/easylogging++/ea_config.h +++ b/external/easylogging++/ea_config.h @@ -6,6 +6,7 @@ #define ELPP_NO_CHECK_MACROS #define ELPP_WINSOCK2 #define ELPP_NO_DEBUG_MACROS +#define ELPP_UTC_DATETIME #ifdef EASYLOGGING_CC #if !(!defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__) diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index a4bdad4cf..d57f3f3a0 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -1,13 +1,14 @@ // // Bismillah ar-Rahmaan ar-Raheem // -// Easylogging++ v9.94.1 +// Easylogging++ v9.96.5 // Cross-platform logging library for C++ applications // -// Copyright (c) 2017 muflihun.com +// Copyright (c) 2012-2018 Muflihun Labs +// Copyright (c) 2012-2018 @abumusamq // // This library is released under the MIT Licence. -// http://labs.muflihun.com/easyloggingpp/licence.php +// https://github.com/muflihun/easyloggingpp/blob/master/LICENSE // // https://github.com/muflihun/easyloggingpp // https://muflihun.github.io/easyloggingpp @@ -25,8 +26,95 @@ INITIALIZE_EASYLOGGINGPP namespace el { -// el::base::utils +// el::base namespace base { +// el::base::consts +namespace consts { + +// Level log values - These are values that are replaced in place of %level format specifier +// Extra spaces after format specifiers are only for readability purposes in log files +static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO"); +static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); +static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARNING"); +static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); +static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); +static const base::type::char_t* kVerboseLevelLogValue = + ELPP_LITERAL("VERBOSE"); // will become VERBOSE-x where x = verbose level +static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); +static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); +static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); +static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); +static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); +static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); +static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); +static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); +// Format specifiers - These are used to define log format +static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); +static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); +static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); +static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); +static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); +static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); +static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); +static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); +static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); +static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); +static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); +static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); +static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); +static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); +static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); +static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; +// Date/time +static const char* kDays[7] = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; +static const char* kDaysAbbrev[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; +static const char* kMonths[12] = { "January", "February", "March", "Apri", "May", "June", "July", "August", + "September", "October", "November", "December" + }; +static const char* kMonthsAbbrev[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; +static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; +static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; +static const int kYearBase = 1900; +static const char* kAm = "AM"; +static const char* kPm = "PM"; +// Miscellaneous constants + +static const char* kNullPointer = "nullptr"; +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED +static const base::type::VerboseLevel kMaxVerboseLevel = 9; +static const char* kUnknownUser = "user"; +static const char* kUnknownHost = "unknown-host"; + + +//---------------- DEFAULT LOG FILE ----------------------- + +#if defined(ELPP_NO_DEFAULT_LOG_FILE) +# if ELPP_OS_UNIX +static const char* kDefaultLogFile = "/dev/null"; +# elif ELPP_OS_WINDOWS +static const char* kDefaultLogFile = "nul"; +# endif // ELPP_OS_UNIX +#elif defined(ELPP_DEFAULT_LOG_FILE) +static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; +#else +static const char* kDefaultLogFile = "myeasylog.log"; +#endif // defined(ELPP_NO_DEFAULT_LOG_FILE) + + +#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +static const char* kDefaultLogFileParam = "--default-log-file"; +#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kLoggingFlagsParam = "--logging-flags"; +#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kValidLoggerIdSymbols = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; +static const char* kConfigurationComment = "##"; +static const char* kConfigurationLevel = "*"; +static const char* kConfigurationLoggerId = "--"; +} +// el::base::utils namespace utils { /// @brief Aborts application due with user-defined status @@ -303,11 +391,7 @@ void Configurations::set(Configuration* conf) { void Configurations::setToDefault(void) { setGlobally(ConfigurationType::Enabled, std::string("true"), true); -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) setGlobally(ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile), true); -#else - ELPP_UNUSED(base::consts::kDefaultLogFile); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) #if defined(ELPP_NO_LOG_TO_FILE) setGlobally(ConfigurationType::ToFile, std::string("false"), true); #else @@ -336,9 +420,7 @@ void Configurations::setRemainingToDefault(void) { #else unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("true")); #endif // defined(ELPP_NO_LOG_TO_FILE) -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) unsafeSetIfNotExist(Level::Global, ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile)); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) unsafeSetIfNotExist(Level::Global, ConfigurationType::ToStandardOutput, std::string("true")); unsafeSetIfNotExist(Level::Global, ConfigurationType::SubsecondPrecision, std::string("3")); unsafeSetIfNotExist(Level::Global, ConfigurationType::PerformanceTracking, std::string("true")); @@ -596,7 +678,6 @@ void Logger::configure(const Configurations& configurations) { if (m_typedConfigurations != nullptr) { Configurations* c = const_cast<Configurations*>(m_typedConfigurations->configurations()); if (c->hasConfiguration(Level::Global, ConfigurationType::Filename)) { - // This check is definitely needed for cases like ELPP_NO_DEFAULT_LOG_FILE flush(); } } @@ -640,7 +721,11 @@ void Logger::flush(Level level, base::type::fstream_t* fs) { } if (fs != nullptr) { fs->flush(); - m_unflushedCount.find(level)->second = 0; + std::unordered_map<Level, unsigned int>::iterator iter = m_unflushedCount.find(level); + if (iter != m_unflushedCount.end()) { + iter->second = 0; + } + Helpers::validateFileRolling(this, level); } } @@ -699,10 +784,9 @@ std::size_t File::getSizeOfFile(base::type::fstream_t* fs) { if (fs == nullptr) { return 0; } - std::streampos currPos = fs->tellg(); - fs->seekg(0, fs->end); + // Since the file stream is appended to or truncated, the current + // offset is the file size. std::size_t size = static_cast<std::size_t>(fs->tellg()); - fs->seekg(currPos); return size; } @@ -894,7 +978,10 @@ void Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::st #endif // defined(ELPP_UNICODE) std::string& Str::toUpper(std::string& str) { - std::transform(str.begin(), str.end(), str.begin(), ::toupper); + std::transform(str.begin(), str.end(), str.begin(), + [](char c) { + return static_cast<char>(::toupper(c)); + }); return str; } @@ -1022,11 +1109,13 @@ const std::string OS::getBashOutput(const char* command) { char hBuff[4096]; if (fgets(hBuff, sizeof(hBuff), proc) != nullptr) { pclose(proc); - const size_t len = strlen(hBuff); - if (len > 0 && hBuff[len - 1] == '\n') { - hBuff[len- 1] = '\0'; + const std::size_t buffLen = strlen(hBuff); + if (buffLen > 0 && hBuff[buffLen - 1] == '\n') { + hBuff[buffLen- 1] = '\0'; } return std::string(hBuff); + } else { + pclose(proc); } return std::string(); #else @@ -1172,19 +1261,23 @@ unsigned long long DateTime::getTimeDifference(const struct timeval& endTime, co struct ::tm* DateTime::buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo) { #if ELPP_OS_UNIX time_t rawTime = currTime->tv_sec; - ::gmtime_r(&rawTime, timeInfo); + ::elpptime_r(&rawTime, timeInfo); return timeInfo; #else # if ELPP_COMPILER_MSVC ELPP_UNUSED(currTime); time_t t; +# if defined(_USE_32BIT_TIME_T) + _time32(&t); +# else _time64(&t); - gmtime_s(timeInfo, &t); +# endif + elpptime_s(timeInfo, &t); return timeInfo; # else // For any other compilers that don't have CRT warnings issue e.g, MinGW or TDM GCC- we use different method time_t rawTime = currTime->tv_sec; - struct tm* tmInf = gmtime(&rawTime); + struct tm* tmInf = elpptime(&rawTime); *timeInfo = *tmInf; return timeInfo; # endif // ELPP_COMPILER_MSVC @@ -1292,7 +1385,8 @@ bool CommandLineArgs::hasParamWithValue(const char* paramKey) const { } const char* CommandLineArgs::getParamValue(const char* paramKey) const { - return m_paramsWithValue.find(std::string(paramKey))->second.c_str(); + std::unordered_map<std::string, std::string>::const_iterator iter = m_paramsWithValue.find(std::string(paramKey)); + return iter != m_paramsWithValue.end() ? iter->second.c_str() : ""; } bool CommandLineArgs::hasParam(const char* paramKey) const { @@ -1641,10 +1735,11 @@ void TypedConfigurations::build(Configurations* configurations) { } else if (conf->configurationType() == ConfigurationType::PerformanceTracking) { setValue(Level::Global, getBool(conf->value()), &m_performanceTrackingMap); } else if (conf->configurationType() == ConfigurationType::MaxLogFileSize) { - setValue(conf->level(), static_cast<std::size_t>(getULong(conf->value())), &m_maxLogFileSizeMap); -#if !defined(ELPP_NO_DEFAULT_LOG_FILE) - withFileSizeLimit.push_back(conf); -#endif // !defined(ELPP_NO_DEFAULT_LOG_FILE) + auto v = getULong(conf->value()); + setValue(conf->level(), static_cast<std::size_t>(v), &m_maxLogFileSizeMap); + if (v != 0) { + withFileSizeLimit.push_back(conf); + } } else if (conf->configurationType() == ConfigurationType::LogFlushThreshold) { setValue(conf->level(), static_cast<std::size_t>(getULong(conf->value())), &m_logFlushThresholdMap); } @@ -1718,12 +1813,6 @@ std::string TypedConfigurations::resolveFilename(const std::string& filename) { } void TypedConfigurations::insertFile(Level level, const std::string& fullFilename) { -#if defined(ELPP_NO_LOG_TO_FILE) - setValue(level, false, &m_toFileMap); - ELPP_UNUSED(fullFilename); - m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(nullptr))); - return; -#endif if (fullFilename.empty()) return; std::string resolvedFilename = resolveFilename(fullFilename); @@ -1861,8 +1950,10 @@ bool RegisteredLoggers::remove(const std::string& id) { if (id == base::consts::kDefaultLoggerId) { return false; } + // get has internal lock Logger* logger = base::utils::Registry<Logger, std::string>::get(id); if (logger != nullptr) { + // unregister has internal lock unregister(logger); } return true; @@ -1981,6 +2072,12 @@ void VRegistry::setCategories(const char* categories, bool clear) { m_categoriesString += ","; m_categoriesString += categories; + size_t n_fields = m_categories.size() + 1; + for (const char *ptr = categories; *ptr; ++ptr) + if (*ptr == ',') + ++n_fields; + m_categories.reserve(n_fields); + bool isCat = true; bool isLevel = false; std::stringstream ss; @@ -2042,7 +2139,7 @@ bool VRegistry::allowed(Level level, const char* category) { if (m_categories.empty() || category == nullptr) { return false; } else { - std::deque<std::pair<std::string, Level>>::const_reverse_iterator it = m_categories.rbegin(); + std::vector<std::pair<std::string, Level>>::const_reverse_iterator it = m_categories.rbegin(); for (; it != m_categories.rend(); ++it) { if (base::utils::Str::wildCardMatch(category, it->first.c_str())) { const int p = priority(it->second); @@ -2060,9 +2157,11 @@ bool VRegistry::allowed(base::type::VerboseLevel vlevel, const char* file) { if (m_modules.empty() || file == nullptr) { return vlevel <= m_level; } else { - std::map<std::string, base::type::VerboseLevel>::iterator it = m_modules.begin(); + char baseFilename[base::consts::kSourceFilenameMaxLength] = ""; + base::utils::File::buildBaseFilename(file, baseFilename); + std::unordered_map<std::string, base::type::VerboseLevel>::iterator it = m_modules.begin(); for (; it != m_modules.end(); ++it) { - if (base::utils::Str::wildCardMatch(file, it->first.c_str())) { + if (base::utils::Str::wildCardMatch(baseFilename, it->first.c_str())) { return vlevel <= it->second; } } @@ -2170,7 +2269,7 @@ Storage::~Storage(void) { } bool Storage::hasCustomFormatSpecifier(const char* formatSpecifier) { - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); return std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), formatSpecifier) != m_customFormatSpecifiers.end(); } @@ -2179,12 +2278,12 @@ void Storage::installCustomFormatSpecifier(const CustomFormatSpecifier& customFo if (hasCustomFormatSpecifier(customFormatSpecifier.formatSpecifier())) { return; } - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); m_customFormatSpecifiers.push_back(customFormatSpecifier); } bool Storage::uninstallCustomFormatSpecifier(const char* formatSpecifier) { - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); std::vector<CustomFormatSpecifier>::iterator it = std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), formatSpecifier); if (it != m_customFormatSpecifiers.end() && strcmp(formatSpecifier, it->formatSpecifier()) == 0) { @@ -2222,9 +2321,35 @@ void Storage::setApplicationArguments(int argc, char** argv) { #endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) } +} // namespace base + +// LogDispatchCallback +void LogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) + base::threading::ScopedLock scopedLock(m_fileLocksMapLock); + std::string filename = data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level()); + auto lock = m_fileLocks.find(filename); + if (lock == m_fileLocks.end()) { + m_fileLocks.emplace(std::make_pair(filename, std::unique_ptr<base::threading::Mutex>(new base::threading::Mutex))); + } +#endif +} + +base::threading::Mutex& LogDispatchCallback::fileHandle(const LogDispatchData* data) { + auto it = m_fileLocks.find(data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level())); + return *(it->second.get()); +} + +namespace base { // DefaultLogDispatchCallback void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) +#if 0 + LogDispatchCallback::handle(data); + base::threading::ScopedLock scopedLock(fileHandle(data)); +#endif +#endif m_data = data; dispatch(m_data->logMessage()->logger()->logBuilder()->build(m_data->logMessage(), m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog)); @@ -2475,6 +2600,8 @@ base::type::string_t DefaultLogBuilder::build(const LogMessage* logMessage, bool base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kMessageFormatSpecifier, logMessage->message()); } #if !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) + el::base::threading::ScopedLock lock_(ELPP->customFormatSpecifiersLock()); + ELPP_UNUSED(lock_); for (std::vector<CustomFormatSpecifier>::const_iterator it = ELPP->customFormatSpecifiers()->begin(); it != ELPP->customFormatSpecifiers()->end(); ++it) { std::string fs(it->formatSpecifier()); @@ -2495,10 +2622,15 @@ void LogDispatcher::dispatch(void) { if (!m_proceed) { return; } +#ifndef ELPP_NO_GLOBAL_LOCK + // see https://github.com/muflihun/easyloggingpp/issues/580 + // global lock is turned off by default unless + // ELPP_NO_GLOBAL_LOCK is defined base::threading::ScopedLock scopedLock(ELPP->lock()); - base::TypedConfigurations* tc = m_logMessage.logger()->m_typedConfigurations; +#endif + base::TypedConfigurations* tc = m_logMessage->logger()->m_typedConfigurations; if (ELPP->hasFlag(LoggingFlag::StrictLogFileSizeCheck)) { - tc->validateFileRolling(m_logMessage.level(), ELPP->preRollOutCallback()); + tc->validateFileRolling(m_logMessage->level(), ELPP->preRollOutCallback()); } LogDispatchCallback* callback = nullptr; LogDispatchData data; @@ -2506,7 +2638,7 @@ void LogDispatcher::dispatch(void) { : ELPP->m_logDispatchCallbacks) { callback = h.second.get(); if (callback != nullptr && callback->enabled()) { - data.setLogMessage(&m_logMessage); + data.setLogMessage(m_logMessage); data.setDispatchAction(m_dispatchAction); callback->handle(&data); } @@ -2553,6 +2685,7 @@ Writer& Writer::construct(int count, const char* loggerIds, ...) { va_list loggersList; va_start(loggersList, loggerIds); const char* id = loggerIds; + m_loggerIds.reserve(count); for (int i = 0; i < count; ++i) { m_loggerIds.push_back(std::string(id)); id = va_arg(loggersList, const char*); @@ -2571,12 +2704,12 @@ void Writer::initializeLogger(const std::string& loggerId, bool lookup, bool nee m_logger = ELPP->registeredLoggers()->get(loggerId, ELPP->hasFlag(LoggingFlag::CreateLoggerAutomatically)); } if (m_logger == nullptr) { - ELPP->acquireLock(); - if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { - // Somehow default logger has been unregistered. Not good! Register again - ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); + { + if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { + // Somehow default logger has been unregistered. Not good! Register again + ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); + } } - ELPP->releaseLock(); // Need to unlock it for next writer Writer(Level::Debug, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) << "Logger [" << loggerId << "] is not registered yet!"; m_proceed = false; @@ -2637,8 +2770,13 @@ void Writer::processDispatch() { void Writer::triggerDispatch(void) { if (m_proceed) { - base::LogDispatcher(m_proceed, LogMessage(m_level, m_file, m_line, m_func, m_verboseLevel, - m_logger), m_dispatchAction).dispatch(); + if (m_msg == nullptr) { + LogMessage msg(m_level, m_file, m_line, m_func, m_verboseLevel, + m_logger); + base::LogDispatcher(m_proceed, &msg, m_dispatchAction).dispatch(); + } else { + base::LogDispatcher(m_proceed, m_msg, m_dispatchAction).dispatch(); + } } if (m_logger != nullptr) { m_logger->stream().str(ELPP_LITERAL("")); @@ -2651,7 +2789,7 @@ void Writer::triggerDispatch(void) { std::stringstream reasonStream; reasonStream << "Fatal log at [" << m_file << ":" << m_line << "]" << " If you wish to disable 'abort on fatal log' please use " - << "el::Helpers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; + << "el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; base::utils::abort(1, reasonStream.str()); } m_proceed = false; @@ -2767,18 +2905,19 @@ namespace debug { // StackTrace -StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const char* loc, const char* demang, const char* hex, - const char* addr) { - m_index = index; - m_location = std::string(loc); - m_demangled = std::string(demang); - m_hex = std::string(hex); - m_addr = std::string(addr); +StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, + const std::string& hex, + const std::string& addr) : + m_index(index), + m_location(loc), + m_demangled(demang), + m_hex(hex), + m_addr(addr) { } std::ostream& operator<<(std::ostream& ss, const StackTrace::StackTraceEntry& si) { - ss << "[" << si.m_index << "] " << si.m_location << (si.m_demangled.empty() ? "" : ":") << si.m_demangled - << (si.m_hex.empty() ? "" : "+") << si.m_hex << si.m_addr; + ss << "[" << si.m_index << "] " << si.m_location << (si.m_hex.empty() ? "" : "+") << si.m_hex << " " << si.m_addr << + (si.m_demangled.empty() ? "" : ":") << si.m_demangled; return ss; } @@ -2798,44 +2937,40 @@ void StackTrace::generateNew(void) { char** strings = backtrace_symbols(stack, size); if (size > kStackStart) { // Skip StackTrace c'tor and generateNew for (std::size_t i = kStackStart; i < size; ++i) { - char* mangName = nullptr; - char* hex = nullptr; - char* addr = nullptr; - for (char* c = strings[i]; *c; ++c) { - switch (*c) { - case '(': - mangName = c; - break; - case '+': - hex = c; - break; - case ')': - addr = c; - break; - default: - break; - } + std::string mangName; + std::string location; + std::string hex; + std::string addr; + + // entry: 2 crash.cpp.bin 0x0000000101552be5 _ZN2el4base5debug10StackTraceC1Ev + 21 + const std::string line(strings[i]); + auto p = line.find("_"); + if (p != std::string::npos) { + mangName = line.substr(p); + mangName = mangName.substr(0, mangName.find(" +")); + } + p = line.find("0x"); + if (p != std::string::npos) { + addr = line.substr(p); + addr = addr.substr(0, addr.find("_")); } // Perform demangling if parsed properly - if (mangName != nullptr && hex != nullptr && addr != nullptr && mangName < hex) { - *mangName++ = '\0'; - *hex++ = '\0'; - *addr++ = '\0'; + if (!mangName.empty()) { int status = 0; - char* demangName = abi::__cxa_demangle(mangName, 0, 0, &status); + char* demangName = abi::__cxa_demangle(mangName.data(), 0, 0, &status); // if demangling is successful, output the demangled function name if (status == 0) { // Success (see http://gcc.gnu.org/onlinedocs/libstdc++/libstdc++-html-USERS-4.3/a01696.html) - StackTraceEntry entry(i - 1, strings[i], demangName, hex, addr); + StackTraceEntry entry(i - 1, location, demangName, hex, addr); m_stack.push_back(entry); } else { // Not successful - we will use mangled name - StackTraceEntry entry(i - 1, strings[i], mangName, hex, addr); + StackTraceEntry entry(i - 1, location, mangName, hex, addr); m_stack.push_back(entry); } free(demangName); } else { - StackTraceEntry entry(i - 1, strings[i]); + StackTraceEntry entry(i - 1, line); m_stack.push_back(entry); } } @@ -2869,6 +3004,9 @@ static std::string crashReason(int sig) { } /// @brief Logs reason of crash from sig static void logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { + if (sig == SIGINT && ELPP->hasFlag(el::LoggingFlag::IgnoreSigInt)) { + return; + } std::stringstream ss; ss << "CRASH HANDLED; "; ss << crashReason(sig); @@ -2947,7 +3085,6 @@ void Helpers::logCrashReason(int sig, bool stackTraceIfAvailable, Level level, c // Loggers Logger* Loggers::getLogger(const std::string& identity, bool registerIfNotAvailable) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->get(identity, registerIfNotAvailable); } @@ -2956,12 +3093,10 @@ void Loggers::setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr) { } bool Loggers::unregisterLogger(const std::string& identity) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->remove(identity); } bool Loggers::hasLogger(const std::string& identity) { - base::threading::ScopedLock scopedLock(ELPP->lock()); return ELPP->registeredLoggers()->has(identity); } @@ -3131,11 +3266,11 @@ const std::string &Loggers::getFilenameCommonPrefix() { // VersionInfo const std::string VersionInfo::version(void) { - return std::string("9.94.1"); + return std::string("9.96.5"); } /// @brief Release date of current version const std::string VersionInfo::releaseDate(void) { - return std::string("25-02-2017 0813hrs"); + return std::string("07-09-2018 0950hrs"); } } // namespace el diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index 8e8b7094b..046252a5b 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -1,18 +1,20 @@ // // Bismillah ar-Rahmaan ar-Raheem // -// Easylogging++ v9.94.1 +// Easylogging++ v9.96.5 // Single-header only, cross-platform logging library for C++ applications // -// Copyright (c) 2017 muflihun.com +// Copyright (c) 2012-2018 Muflihun Labs +// Copyright (c) 2012-2018 @abumusamq // // This library is released under the MIT Licence. -// http://labs.muflihun.com/easyloggingpp/licence.php +// https://github.com/muflihun/easyloggingpp/blob/master/LICENSE // // https://github.com/muflihun/easyloggingpp // https://muflihun.github.io/easyloggingpp // http://muflihun.com // + #ifndef EASYLOGGINGPP_H #define EASYLOGGINGPP_H #include "ea_config.h" @@ -94,7 +96,7 @@ #else # define ELPP_OS_MAC 0 #endif -#if (defined(__FreeBSD__)) +#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) # define ELPP_OS_FREEBSD 1 #else # define ELPP_OS_FREEBSD 0 @@ -114,13 +116,23 @@ #else # define ELPP_OS_SOLARIS 0 #endif +#if (defined(_AIX)) +# define ELPP_OS_AIX 1 +#else +# define ELPP_OS_AIX 0 +#endif +#if (defined(__NetBSD__)) +# define ELPP_OS_NETBSD 1 +#else +# define ELPP_OS_NETBSD 0 +#endif #if (defined(__DragonFly__)) # define ELPP_OS_DRAGONFLY 1 #else # define ELPP_OS_DRAGONFLY 0 #endif // Unix -#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_SOLARIS || ELPP_OS_DRAGONFLY || ELPP_OS_OPENBSD || ELPP_OS_NETBSD ) && (!ELPP_OS_WINDOWS)) +#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_NETBSD || ELPP_OS_SOLARIS || ELPP_OS_AIX || ELPP_OS_DRAGONFLY || ELPP_OS_OPENBSD) && (!ELPP_OS_WINDOWS)) # define ELPP_OS_UNIX 1 #else # define ELPP_OS_UNIX 0 @@ -205,7 +217,7 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre # define ELPP_INTERNAL_INFO(lvl, msg) #endif // (defined(ELPP_DEBUG_INFO)) #if (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) -# if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_OPENBSD && !ELPP_OS_NETBSD) +# if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_OPENBSD && !ELPP_OS_NETBSD && !ELPP_OS_ANDROID) # define ELPP_STACKTRACE 1 # else # define ELPP_STACKTRACE 0 @@ -386,7 +398,7 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre #include <string> #include <vector> #include <map> -#include <deque> +#include <unordered_map> #include <utility> #include <functional> #include <algorithm> @@ -424,9 +436,6 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre # if defined(ELPP_LOG_STD_ARRAY) # include <array> # endif // defined(ELPP_LOG_STD_ARRAY) -# if defined(ELPP_LOG_UNORDERED_MAP) -# include <unordered_map> -# endif // defined(ELPP_LOG_UNORDERED_MAP) # if defined(ELPP_LOG_UNORDERED_SET) # include <unordered_set> # endif // defined(ELPP_UNORDERED_SET) @@ -461,6 +470,15 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre // For logging wxWidgets based classes & templates # include <wx/vector.h> #endif // defined(ELPP_WXWIDGETS_LOGGING) +#if defined(ELPP_UTC_DATETIME) +# define elpptime_r gmtime_r +# define elpptime_s gmtime_s +# define elpptime gmtime +#else +# define elpptime_r localtime_r +# define elpptime_s localtime_s +# define elpptime localtime +#endif // defined(ELPP_UTC_DATETIME) // Forward declarations namespace el { class Logger; @@ -585,6 +603,16 @@ enum class Level : base::type::EnumType { /// @brief Represents unknown level Unknown = 1010 }; +} // namespace el +namespace std { +template<> struct hash<el::Level> { + public: + std::size_t operator()(const el::Level& l) const { + return hash<el::base::type::EnumType> {}(static_cast<el::base::type::EnumType>(l)); + } +}; +} +namespace el { /// @brief Static class that contains helper functions for el::Level class LevelHelper : base::StaticClass { public: @@ -711,113 +739,41 @@ enum class LoggingFlag : base::type::EnumType { /// @brief Adds spaces b/w logs that separated by left-shift operator AutoSpacing = 8192, /// @brief Preserves time format and does not convert it to sec, hour etc (performance tracking only) - FixedTimeFormat = 16384 + FixedTimeFormat = 16384, + // @brief Ignore SIGINT or crash + IgnoreSigInt = 32768, }; namespace base { /// @brief Namespace containing constants used internally. namespace consts { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" -#endif -// Level log values - These are values that are replaced in place of %level format specifier -static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO "); -static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); -static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARN "); -static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); -static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); -static const base::type::char_t* kVerboseLevelLogValue = ELPP_LITERAL("VER"); -static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); -static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); -static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); -static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); -static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); -static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); -static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); -static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); -// Format specifiers - These are used to define log format -static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); -static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); -static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); -static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); -static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); -static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); -static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); -static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); -static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); -static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); -static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); -static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); -static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); -static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); -static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); -static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; -// Date/time -static const char* kDays[7] = { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; -static const char* kDaysAbbrev[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; -static const char* kMonths[12] = { "January", "February", "March", "Apri", "May", "June", "July", "August", - "September", "October", "November", "December" - }; -static const char* kMonthsAbbrev[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; -static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; -static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; -static const int kYearBase = 1900; -static const char* kAm = "AM"; -static const char* kPm = "PM"; -// Miscellaneous constants +static const char kFormatSpecifierCharValue = 'v'; +static const char kFormatSpecifierChar = '%'; +static const unsigned int kMaxLogPerCounter = 100000; +static const unsigned int kMaxLogPerContainer = 100; +static const unsigned int kDefaultSubsecondPrecision = 3; + #ifdef ELPP_DEFAULT_LOGGER static const char* kDefaultLoggerId = ELPP_DEFAULT_LOGGER; #else static const char* kDefaultLoggerId = "default"; #endif + #ifdef ELPP_DEFAULT_PERFORMANCE_LOGGER static const char* kPerformanceLoggerId = ELPP_DEFAULT_PERFORMANCE_LOGGER; #else static const char* kPerformanceLoggerId = "performance"; #endif + #if defined(ELPP_SYSLOG) static const char* kSysLogLoggerId = "syslog"; #endif // defined(ELPP_SYSLOG) -static const char* kNullPointer = "nullptr"; -static const char kFormatSpecifierChar = '%'; -#if ELPP_VARIADIC_TEMPLATES_SUPPORTED -static const char kFormatSpecifierCharValue = 'v'; -#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED -static const unsigned int kMaxLogPerContainer = 100; -static const unsigned int kMaxLogPerCounter = 100000; -static const unsigned int kDefaultSubsecondPrecision = 3; -static const base::type::VerboseLevel kMaxVerboseLevel = 9; -static const char* kUnknownUser = "user"; -static const char* kUnknownHost = "unknown-host"; -#if defined(ELPP_DEFAULT_LOG_FILE) -static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; -#else -# if ELPP_OS_UNIX -# if ELPP_OS_ANDROID -static const char* kDefaultLogFile = "logs/myeasylog.log"; -# else -static const char* kDefaultLogFile = "logs/myeasylog.log"; -# endif // ELPP_OS_ANDROID -# elif ELPP_OS_WINDOWS -static const char* kDefaultLogFile = "logs\\myeasylog.log"; -# endif // ELPP_OS_UNIX -#endif // defined(ELPP_DEFAULT_LOG_FILE) -#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -static const char* kDefaultLogFileParam = "--default-log-file"; -#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) -static const char* kLoggingFlagsParam = "--logging-flags"; -#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) + #if ELPP_OS_WINDOWS static const char* kFilePathSeperator = "\\"; #else static const char* kFilePathSeperator = "/"; #endif // ELPP_OS_WINDOWS -static const char* kValidLoggerIdSymbols = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; -static const char* kConfigurationComment = "##"; -static const char* kConfigurationLevel = "*"; -static const char* kConfigurationLoggerId = "--"; + static const std::size_t kSourceFilenameMaxLength = 100; static const std::size_t kSourceLineMaxLength = 10; static const Level kPerformanceTrackerDefaultLevel = Level::Info; @@ -862,9 +818,6 @@ const struct { }, }; static const int kCrashSignalsCount = sizeof(kCrashSignals) / sizeof(kCrashSignals[0]); -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif } // namespace consts } // namespace base typedef std::function<void(const char*, std::size_t)> PreRollOutCallback; @@ -1271,8 +1224,8 @@ class DateTime : base::StaticClass { base::TimestampUnit timestampUnit); - private: static struct ::tm* buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo); + private: static char* parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, std::size_t msec, const base::SubsecondPrecision* ssPrec); }; @@ -1311,7 +1264,7 @@ class CommandLineArgs { private: int m_argc; char** m_argv; - std::map<std::string, std::string> m_paramsWithValue; + std::unordered_map<std::string, std::string> m_paramsWithValue; std::vector<std::string> m_params; }; /// @brief Abstract registry (aka repository) that provides basic interface for pointer repository specified by T_Ptr type. @@ -1436,7 +1389,7 @@ class AbstractRegistry : public base::threading::ThreadSafe { /// of AbstractRegistry<T_Ptr, Container>. Any implementation of this class should be /// explicitly (by using lock functions) template <typename T_Ptr, typename T_Key = const char*> -class Registry : public AbstractRegistry<T_Ptr, std::map<T_Key, T_Ptr*>> { +class Registry : public AbstractRegistry<T_Ptr, std::unordered_map<T_Key, T_Ptr*>> { public: typedef typename Registry<T_Ptr, T_Key>::iterator iterator; typedef typename Registry<T_Ptr, T_Key>::const_iterator const_iterator; @@ -1486,8 +1439,8 @@ class Registry : public AbstractRegistry<T_Ptr, std::map<T_Key, T_Ptr*>> { void unregister(const T_Key& uniqKey) { T_Ptr* existing = get(uniqKey); if (existing != nullptr) { - base::utils::safeDelete(existing); this->list().erase(uniqKey); + base::utils::safeDelete(existing); } } @@ -1500,7 +1453,7 @@ class Registry : public AbstractRegistry<T_Ptr, std::map<T_Key, T_Ptr*>> { } private: - virtual void deepCopy(const AbstractRegistry<T_Ptr, std::map<T_Key, T_Ptr*>>& sr) ELPP_FINAL { + virtual void deepCopy(const AbstractRegistry<T_Ptr, std::unordered_map<T_Key, T_Ptr*>>& sr) ELPP_FINAL { for (const_iterator it = sr.cbegin(); it != sr.cend(); ++it) { registerNew(it->first, new T_Ptr(*it->second)); } @@ -1600,7 +1553,7 @@ class RegistryWithPred : public AbstractRegistry<T_Ptr, std::vector<T_Ptr*>> { class Utils { public: template <typename T, typename TPtr> - static bool installCallback(const std::string& id, std::map<std::string, TPtr>* mapT) { + static bool installCallback(const std::string& id, std::unordered_map<std::string, TPtr>* mapT) { if (mapT->find(id) == mapT->end()) { mapT->insert(std::make_pair(id, TPtr(new T()))); return true; @@ -1609,15 +1562,15 @@ class Utils { } template <typename T, typename TPtr> - static void uninstallCallback(const std::string& id, std::map<std::string, TPtr>* mapT) { + static void uninstallCallback(const std::string& id, std::unordered_map<std::string, TPtr>* mapT) { if (mapT->find(id) != mapT->end()) { mapT->erase(id); } } template <typename T, typename TPtr> - static T* callback(const std::string& id, std::map<std::string, TPtr>* mapT) { - typename std::map<std::string, TPtr>::iterator iter = mapT->find(id); + static T* callback(const std::string& id, std::unordered_map<std::string, TPtr>* mapT) { + typename std::unordered_map<std::string, TPtr>::iterator iter = mapT->find(id); if (iter != mapT->end()) { return static_cast<T*>(iter->second.get()); } @@ -1962,7 +1915,7 @@ class Configurations : public base::utils::RegistryWithPred<Configuration, Confi namespace base { typedef std::shared_ptr<base::type::fstream_t> FileStreamPtr; -typedef std::map<std::string, FileStreamPtr> LogStreamsReferenceMap; +typedef std::unordered_map<std::string, FileStreamPtr> LogStreamsReferenceMap; /// @brief Configurations with data types. /// /// @detail el::Configurations have string based values. This is whats used internally in order to read correct configurations. @@ -1999,16 +1952,16 @@ class TypedConfigurations : public base::threading::ThreadSafe { private: Configurations* m_configurations; - std::map<Level, bool> m_enabledMap; - std::map<Level, bool> m_toFileMap; - std::map<Level, std::string> m_filenameMap; - std::map<Level, bool> m_toStandardOutputMap; - std::map<Level, base::LogFormat> m_logFormatMap; - std::map<Level, base::SubsecondPrecision> m_subsecondPrecisionMap; - std::map<Level, bool> m_performanceTrackingMap; - std::map<Level, base::FileStreamPtr> m_fileStreamMap; - std::map<Level, std::size_t> m_maxLogFileSizeMap; - std::map<Level, std::size_t> m_logFlushThresholdMap; + std::unordered_map<Level, bool> m_enabledMap; + std::unordered_map<Level, bool> m_toFileMap; + std::unordered_map<Level, std::string> m_filenameMap; + std::unordered_map<Level, bool> m_toStandardOutputMap; + std::unordered_map<Level, base::LogFormat> m_logFormatMap; + std::unordered_map<Level, base::SubsecondPrecision> m_subsecondPrecisionMap; + std::unordered_map<Level, bool> m_performanceTrackingMap; + std::unordered_map<Level, base::FileStreamPtr> m_fileStreamMap; + std::unordered_map<Level, std::size_t> m_maxLogFileSizeMap; + std::unordered_map<Level, std::size_t> m_logFlushThresholdMap; base::LogStreamsReferenceMap* m_logStreamsReference; friend class el::Helpers; @@ -2018,21 +1971,21 @@ class TypedConfigurations : public base::threading::ThreadSafe { friend class el::base::LogDispatcher; template <typename Conf_T> - inline Conf_T getConfigByVal(Level level, const std::map<Level, Conf_T>* confMap, const char* confName) { + inline Conf_T getConfigByVal(Level level, const std::unordered_map<Level, Conf_T>* confMap, const char* confName) { base::threading::ScopedLock scopedLock(lock()); return unsafeGetConfigByVal(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope } template <typename Conf_T> - inline Conf_T& getConfigByRef(Level level, std::map<Level, Conf_T>* confMap, const char* confName) { + inline Conf_T& getConfigByRef(Level level, std::unordered_map<Level, Conf_T>* confMap, const char* confName) { base::threading::ScopedLock scopedLock(lock()); return unsafeGetConfigByRef(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope } template <typename Conf_T> - Conf_T unsafeGetConfigByVal(Level level, const std::map<Level, Conf_T>* confMap, const char* confName) { + Conf_T unsafeGetConfigByVal(Level level, const std::unordered_map<Level, Conf_T>* confMap, const char* confName) { ELPP_UNUSED(confName); - typename std::map<Level, Conf_T>::const_iterator it = confMap->find(level); + typename std::unordered_map<Level, Conf_T>::const_iterator it = confMap->find(level); if (it == confMap->end()) { try { return confMap->at(Level::Global); @@ -2047,9 +2000,9 @@ class TypedConfigurations : public base::threading::ThreadSafe { } template <typename Conf_T> - Conf_T& unsafeGetConfigByRef(Level level, std::map<Level, Conf_T>* confMap, const char* confName) { + Conf_T& unsafeGetConfigByRef(Level level, std::unordered_map<Level, Conf_T>* confMap, const char* confName) { ELPP_UNUSED(confName); - typename std::map<Level, Conf_T>::iterator it = confMap->find(level); + typename std::unordered_map<Level, Conf_T>::iterator it = confMap->find(level); if (it == confMap->end()) { try { return confMap->at(Level::Global); @@ -2063,14 +2016,15 @@ class TypedConfigurations : public base::threading::ThreadSafe { } template <typename Conf_T> - void setValue(Level level, const Conf_T& value, std::map<Level, Conf_T>* confMap, bool includeGlobalLevel = true) { + void setValue(Level level, const Conf_T& value, std::unordered_map<Level, Conf_T>* confMap, + bool includeGlobalLevel = true) { // If map is empty and we are allowed to add into generic level (Level::Global), do it! if (confMap->empty() && includeGlobalLevel) { confMap->insert(std::make_pair(Level::Global, value)); return; } // If same value exist in generic level already, dont add it to explicit level - typename std::map<Level, Conf_T>::iterator it = confMap->find(Level::Global); + typename std::unordered_map<Level, Conf_T>::iterator it = confMap->find(Level::Global); if (it != confMap->end() && it->second == value) { return; } @@ -2232,21 +2186,26 @@ class LogDispatchData { inline base::DispatchAction dispatchAction(void) const { return m_dispatchAction; } - private: - LogMessage* m_logMessage; - base::DispatchAction m_dispatchAction; - friend class base::LogDispatcher; - inline void setLogMessage(LogMessage* logMessage) { m_logMessage = logMessage; } inline void setDispatchAction(base::DispatchAction dispatchAction) { m_dispatchAction = dispatchAction; } + private: + LogMessage* m_logMessage; + base::DispatchAction m_dispatchAction; + friend class base::LogDispatcher; + }; class LogDispatchCallback : public Callback<LogDispatchData> { + protected: + virtual void handle(const LogDispatchData* data); + base::threading::Mutex& fileHandle(const LogDispatchData* data); private: friend class base::LogDispatcher; + std::unordered_map<std::string, std::unique_ptr<base::threading::Mutex>> m_fileLocks; + base::threading::Mutex m_fileLocksMapLock; }; class PerformanceTrackingCallback : public Callback<PerformanceTrackingData> { private: @@ -2364,7 +2323,7 @@ inline void FUNCTION_NAME(const T&); std::string m_parentApplicationName; bool m_isConfigured; Configurations m_configurations; - std::map<Level, unsigned int> m_unflushedCount; + std::unordered_map<Level, unsigned int> m_unflushedCount; base::LogStreamsReferenceMap* m_logStreamsReference; LogBuilderPtr m_logBuilder; @@ -2470,7 +2429,7 @@ class RegisteredLoggers : public base::utils::Registry<Logger, std::string> { LogBuilderPtr m_defaultLogBuilder; Configurations m_defaultConfigurations; base::LogStreamsReferenceMap m_logStreamsReference; - std::map<std::string, base::type::LoggerRegistrationCallbackPtr> m_loggerRegistrationCallbacks; + std::unordered_map<std::string, base::type::LoggerRegistrationCallbackPtr> m_loggerRegistrationCallbacks; friend class el::base::Storage; void unsafeFlushAll(void); @@ -2508,7 +2467,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { bool allowed(base::type::VerboseLevel vlevel, const char* file); - inline const std::map<std::string, base::type::VerboseLevel>& modules(void) const { + inline const std::unordered_map<std::string, base::type::VerboseLevel>& modules(void) const { return m_modules; } @@ -2530,8 +2489,8 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { private: base::type::VerboseLevel m_level; base::type::EnumType* m_pFlags; - std::map<std::string, base::type::VerboseLevel> m_modules; - std::deque<std::pair<std::string, Level>> m_categories; + std::unordered_map<std::string, base::type::VerboseLevel> m_modules; + std::vector<std::pair<std::string, Level>> m_categories; std::map<std::string, int> m_cached_allowed_categories; std::string m_categoriesString; std::string m_filenameCommonPrefix; @@ -2718,6 +2677,10 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { return &m_customFormatSpecifiers; } + base::threading::Mutex& customFormatSpecifiersLock() { + return m_customFormatSpecifiersLock; + } + inline void setLoggingLevel(Level level) { m_loggingLevel = level; } @@ -2758,13 +2721,13 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { /// @brief Sets thread name for current thread. Requires std::thread inline void setThreadName(const std::string& name) { if (name.empty()) return; - base::threading::ScopedLock scopedLock(lock()); + base::threading::ScopedLock scopedLock(m_threadNamesLock); m_threadNames[base::threading::getCurrentThreadId()] = name; } inline std::string getThreadName(const std::string& threadId) { - base::threading::ScopedLock scopedLock(lock()); - std::map<std::string, std::string>::const_iterator it = m_threadNames.find(threadId); + base::threading::ScopedLock scopedLock(m_threadNamesLock); + std::unordered_map<std::string, std::string>::const_iterator it = m_threadNames.find(threadId); if (it == m_threadNames.end()) { return threadId; } @@ -2784,10 +2747,12 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { #endif // ELPP_ASYNC_LOGGING base::utils::CommandLineArgs m_commandLineArgs; PreRollOutCallback m_preRollOutCallback; - std::map<std::string, base::type::LogDispatchCallbackPtr> m_logDispatchCallbacks; - std::map<std::string, base::type::PerformanceTrackingCallbackPtr> m_performanceTrackingCallbacks; - std::map<std::string, std::string> m_threadNames; + std::unordered_map<std::string, base::type::LogDispatchCallbackPtr> m_logDispatchCallbacks; + std::unordered_map<std::string, base::type::PerformanceTrackingCallbackPtr> m_performanceTrackingCallbacks; + std::unordered_map<std::string, std::string> m_threadNames; std::vector<CustomFormatSpecifier> m_customFormatSpecifiers; + base::threading::Mutex m_customFormatSpecifiersLock; + base::threading::Mutex m_threadNamesLock; Level m_loggingLevel; friend class el::Helpers; @@ -2830,7 +2795,7 @@ class AsyncDispatchWorker : public base::IWorker, public base::threading::Thread void run(void); void setContinueRunning(bool value) { - base::threading::ScopedLock scopedLock(m_continueRunningMutex); + base::threading::ScopedLock scopedLock(m_continueRunningLock); m_continueRunning = value; } @@ -2840,7 +2805,7 @@ class AsyncDispatchWorker : public base::IWorker, public base::threading::Thread private: std::condition_variable cv; bool m_continueRunning; - base::threading::Mutex m_continueRunningMutex; + base::threading::Mutex m_continueRunningLock; }; #endif // ELPP_ASYNC_LOGGING } // namespace base @@ -2852,9 +2817,9 @@ class DefaultLogBuilder : public LogBuilder { /// @brief Dispatches log messages class LogDispatcher : base::NoCopy { public: - LogDispatcher(bool proceed, LogMessage&& logMessage, base::DispatchAction dispatchAction) : + LogDispatcher(bool proceed, LogMessage* logMessage, base::DispatchAction dispatchAction) : m_proceed(proceed), - m_logMessage(std::move(logMessage)), + m_logMessage(logMessage), m_dispatchAction(std::move(dispatchAction)) { } @@ -2862,7 +2827,7 @@ class LogDispatcher : base::NoCopy { private: bool m_proceed; - LogMessage m_logMessage; + LogMessage* m_logMessage; base::DispatchAction m_dispatchAction; }; #if defined(ELPP_STL_LOGGING) @@ -3275,10 +3240,15 @@ class Writer : base::NoCopy { Writer(Level level, const char* file, base::type::LineNumber line, const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, base::type::VerboseLevel verboseLevel = 0) : - m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), + m_msg(nullptr), m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { } + Writer(LogMessage* msg, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog) : + m_msg(msg), m_level(msg != nullptr ? msg->level() : Level::Unknown), + m_line(0), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { + } + virtual ~Writer(void) { processDispatch(); } @@ -3321,6 +3291,7 @@ class Writer : base::NoCopy { Writer& construct(Logger* logger, bool needLock = true); Writer& construct(int count, const char* loggerIds, ...); protected: + LogMessage* m_msg; Level m_level; const char* m_file; const base::type::LineNumber m_line; @@ -3379,6 +3350,7 @@ void Logger::log_(Level level, int vlevel, const T& log) { base::DispatchAction::NormalLog, vlevel).construct(this, false) << log; } else { stream().str(ELPP_LITERAL("")); + releaseLock(); } } else { base::Writer(level, "FILE", 0, "FUNCTION").construct(this, false) << log; @@ -3386,23 +3358,23 @@ void Logger::log_(Level level, int vlevel, const T& log) { } template <typename T, typename... Args> inline void Logger::log(Level level, const char* s, const T& value, const Args&... args) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(level, 0, s, value, args...); } template <typename T> inline void Logger::log(Level level, const T& log) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(level, 0, log); } # if ELPP_VERBOSE_LOG template <typename T, typename... Args> inline void Logger::verbose(int vlevel, const char* s, const T& value, const Args&... args) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(el::Level::Verbose, vlevel, s, value, args...); } template <typename T> inline void Logger::verbose(int vlevel, const T& log) { - base::threading::ScopedLock scopedLock(lock()); + acquireLock(); // released in Writer! log_(el::Level::Verbose, vlevel, log); } # else @@ -3657,8 +3629,9 @@ class StackTrace : base::NoCopy { static const unsigned int kStackStart = 2; // We want to skip c'tor and StackTrace::generateNew() class StackTraceEntry { public: - StackTraceEntry(std::size_t index, const char* loc, const char* demang, const char* hex, const char* addr); - StackTraceEntry(std::size_t index, char* loc) : + StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, const std::string& hex, + const std::string& addr); + StackTraceEntry(std::size_t index, const std::string& loc) : m_index(index), m_location(loc) { } @@ -3843,6 +3816,11 @@ class Helpers : base::StaticClass { static inline const el::base::utils::CommandLineArgs* commandLineArgs(void) { return ELPP->commandLineArgs(); } + /// @brief Reserve space for custom format specifiers for performance + /// @see std::vector::reserve + static inline void reserveCustomFormatSpecifiers(std::size_t size) { + ELPP->m_customFormatSpecifiers.reserve(size); + } /// @brief Installs user defined format specifier and handler static inline void installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { ELPP->installCustomFormatSpecifier(customFormatSpecifier); diff --git a/external/trezor-common b/external/trezor-common new file mode 160000 +Subproject 588f8e03f5ac111adf719f0a437de67481a26ae diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3b71c38cd..6ee7effdd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -145,3 +145,4 @@ if(PER_BLOCK_CHECKPOINT) endif() add_subdirectory(device) +add_subdirectory(device_trezor) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 71c46d76b..7118b0881 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1349,7 +1349,7 @@ public: * * @param details the details of the transaction to add */ - virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) = 0; + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) = 0; /** * @brief update a txpool transaction's metadata diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 84a083c26..ea3638a85 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -85,6 +85,10 @@ inline void throw1(const T &e) #define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} +#define MDB_val_sized(var, val) MDB_val var = {val.size(), (void *)val.data()} + +#define MDB_val_str(var, val) MDB_val var = {strlen(val) + 1, (void *)val} + template<typename T> struct MDB_val_copy: public MDB_val { @@ -714,7 +718,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const diff CURSOR(block_info) // this call to mdb_cursor_put will change height() - MDB_val_copy<blobdata> blob(block_to_blob(blk)); + cryptonote::blobdata block_blob(block_to_blob(blk)); + MDB_val_sized(blob, block_blob); result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str())); @@ -828,7 +833,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); cryptonote::blobdata blob = tx_to_blob(tx); - MDB_val_copy<blobdata> blobval(blob); + MDB_val_sized(blobval, blob); std::stringstream ss; binary_archive<true> ba(ss); @@ -836,7 +841,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (!r) throw0(DB_ERROR("Failed to serialize pruned tx")); std::string pruned = ss.str(); - MDB_val_copy<blobdata> pruned_blob(pruned); + MDB_val_sized(pruned_blob, pruned); result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str())); @@ -844,7 +849,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (pruned.size() > blob.size()) throw0(DB_ERROR("pruned tx size is larger than tx size")); cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size()); - MDB_val_copy<blobdata> prunable_blob(prunable); + MDB_val_sized(prunable_blob, prunable); result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); @@ -1331,7 +1336,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) bool compatible = true; - MDB_val_copy<const char*> k("version"); + MDB_val_str(k, "version"); MDB_val v; auto get_result = mdb_get(txn, m_properties, &k, &v); if(get_result == MDB_SUCCESS) @@ -1379,7 +1384,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) // only write version on an empty DB if (m_height == 0) { - MDB_val_copy<const char*> k("version"); + MDB_val_str(k, "version"); MDB_val_copy<uint32_t> v(VERSION); auto put_result = mdb_put(txn, m_properties, &k, &v, 0); if (put_result != MDB_SUCCESS) @@ -1476,7 +1481,7 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_properties: ", result).c_str())); // init with current version - MDB_val_copy<const char*> k("version"); + MDB_val_str(k, "version"); MDB_val_copy<uint32_t> v(VERSION); if (auto result = mdb_put(txn, m_properties, &k, &v, 0)) throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str())); @@ -1591,7 +1596,7 @@ void BlockchainLMDB::unlock() auto_txn.commit(); \ } while(0) -void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t &meta) +void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1600,8 +1605,6 @@ void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t CURSOR(txpool_meta) CURSOR(txpool_blob) - const crypto::hash txid = get_transaction_hash(tx); - MDB_val k = {sizeof(txid), (void *)&txid}; MDB_val v = {sizeof(meta), (void *)&meta}; if (auto result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) { @@ -1610,7 +1613,7 @@ void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t else throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str())); } - MDB_val_copy<cryptonote::blobdata> blob_val(tx_to_blob(tx)); + MDB_val_sized(blob_val, blob); if (auto result = mdb_cursor_put(m_cur_txpool_blob, &k, &blob_val, MDB_NODUPDATA)) { if (result == MDB_KEYEXIST) throw1(DB_ERROR("Attempting to add txpool tx blob that's already in the db")); @@ -2499,7 +2502,8 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 MDB_val_set(v, index); auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist")); + throw1(OUTPUT_DNE(std::string("Attempting to get output pubkey by index, but key does not exist: amount " + + std::to_string(amount) + ", index " + std::to_string(index)).c_str())); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); @@ -3171,6 +3175,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); tx_out_indices.clear(); + tx_out_indices.reserve(global_indices.size()); TXN_PREFIX_RDONLY(); RCURSOR(output_txs); @@ -3185,9 +3190,8 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - outtx *ot = (outtx *)v.mv_data; - auto result = tx_out_index(ot->tx_hash, ot->local_index); - tx_out_indices.push_back(result); + const outtx *ot = (const outtx *)v.mv_data; + tx_out_indices.push_back(tx_out_index(ot->tx_hash, ot->local_index)); } TXN_POSTFIX_RDONLY(); @@ -3199,6 +3203,7 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui TIME_MEASURE_START(db3); check_open(); outputs.clear(); + outputs.reserve(offsets.size()); TXN_PREFIX_RDONLY(); @@ -3222,19 +3227,19 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui else if (get_result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); - output_data_t data; if (amount == 0) { const outkey *okp = (const outkey *)v.mv_data; - data = okp->data; + outputs.push_back(okp->data); } else { const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; + outputs.resize(outputs.size() + 1); + output_data_t &data = outputs.back(); memcpy(&data, &okp->data, sizeof(pre_rct_output_data_t)); data.commitment = rct::zeroCommit(amount); } - outputs.push_back(data); } TXN_POSTFIX_RDONLY(); @@ -3250,6 +3255,7 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std:: indices.clear(); std::vector <uint64_t> tx_indices; + tx_indices.reserve(offsets.size()); TXN_PREFIX_RDONLY(); RCURSOR(output_amounts); @@ -4063,7 +4069,7 @@ void BlockchainLMDB::migrate_0_1() uint32_t version = 1; v.mv_data = (void *)&version; v.mv_size = sizeof(version); - MDB_val_copy<const char *> vk("version"); + MDB_val_str(vk, "version"); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); @@ -4205,7 +4211,7 @@ void BlockchainLMDB::migrate_1_2() uint32_t version = 2; v.mv_data = (void *)&version; v.mv_size = sizeof(version); - MDB_val_copy<const char *> vk("version"); + MDB_val_str(vk, "version"); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); @@ -4340,7 +4346,7 @@ void BlockchainLMDB::migrate_2_3() uint32_t version = 3; v.mv_data = (void *)&version; v.mv_size = sizeof(version); - MDB_val_copy<const char *> vk("version"); + MDB_val_str(vk, "version"); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index e1f748ed8..7e76236a5 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -256,7 +256,7 @@ public: virtual bool has_key_image(const crypto::key_image& img) const; - virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& meta); + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta); virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta); virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const; virtual bool txpool_has_tx(const crypto::hash &txid) const; diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 6e6e4c6f1..c9ad1cebe 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -101,6 +101,14 @@ set(blockchain_depth_private_headers) monero_private_headers(blockchain_depth ${blockchain_depth_private_headers}) +set(blockchain_stats_sources + blockchain_stats.cpp + ) + +set(blockchain_stats_private_headers) + +monero_private_headers(blockchain_stats + ${blockchain_stats_private_headers}) monero_add_executable(blockchain_import @@ -237,3 +245,23 @@ set_property(TARGET blockchain_depth OUTPUT_NAME "monero-blockchain-depth") install(TARGETS blockchain_depth DESTINATION bin) +monero_add_executable(blockchain_stats + ${blockchain_stats_sources} + ${blockchain_stats_private_headers}) + +target_link_libraries(blockchain_stats + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_stats + PROPERTY + OUTPUT_NAME "monero-blockchain-stats") +install(TARGETS blockchain_stats DESTINATION bin) diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index 2f0bbffd6..e01a8892c 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -407,8 +407,7 @@ int main(int argc, char* argv[]) for (uint64_t h = state.height; h < db_height; ++h) { size_t block_ancestry_size = 0; - const crypto::hash block_hash = db->get_block_hash_from_height(h); - const cryptonote::blobdata bd = db->get_block_blob(block_hash); + const cryptonote::blobdata bd = db->get_block_blob_from_height(h); ++total_blocks; cryptonote::block b; if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) @@ -482,8 +481,7 @@ int main(int argc, char* argv[]) } else { - const crypto::hash block_hash = db->get_block_hash_from_height(od.height); - cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::blobdata bd = db->get_block_blob_from_height(od.height); if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) { LOG_PRINT_L0("Bad block from db"); @@ -620,8 +618,7 @@ int main(int argc, char* argv[]) } else { - const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); - const cryptonote::blobdata bd = db->get_block_blob(block_hash); + const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); cryptonote::block b; if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) { @@ -678,8 +675,7 @@ int main(int argc, char* argv[]) { add_ancestor(ancestry, amount, offset); const output_data_t od = db->get_output_key(amount, offset); - const crypto::hash block_hash = db->get_block_hash_from_height(od.height); - bd = db->get_block_blob(block_hash); + bd = db->get_block_blob_from_height(od.height); cryptonote::block b; if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) { diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index dd2387e5b..8060b0de4 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -187,8 +187,7 @@ int main(int argc, char* argv[]) } else { - const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); - const cryptonote::blobdata bd = db->get_block_blob(block_hash); + const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); cryptonote::block b; if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) { diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 7f92ecd87..eae078ea2 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -396,7 +396,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { std::cout << refresh_string << "block " << h-1 << " / " << block_stop - << std::flush; + << "\r" << std::flush; std::cout << ENDL << ENDL; MINFO("Specified block number reached - stopping. block: " << h-1 << " total blocks: " << h); quit = 1; @@ -432,7 +432,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { std::cout << refresh_string << "block " << h-1 << " / " << block_stop - << std::flush; + << "\r" << std::flush; } if (opt_verify) diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp new file mode 100644 index 000000000..716b33cae --- /dev/null +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -0,0 +1,337 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + uint64_t block_start = 0; + uint64_t block_stop = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<uint64_t> arg_block_start = {"block-start", "start at block number", block_start}; + const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop}; + const command_line::arg_descriptor<bool> arg_inputs = {"with-inputs", "with input stats", false}; + const command_line::arg_descriptor<bool> arg_outputs = {"with-outputs", "with output stats", false}; + const command_line::arg_descriptor<bool> arg_ringsize = {"with-ringsize", "with ringsize stats", false}; + const command_line::arg_descriptor<bool> arg_hours = {"with-hours", "with txns per hour", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_block_start); + command_line::add_arg(desc_cmd_sett, arg_block_stop); + command_line::add_arg(desc_cmd_sett, arg_inputs); + command_line::add_arg(desc_cmd_sett, arg_outputs); + command_line::add_arg(desc_cmd_sett, arg_ringsize); + command_line::add_arg(desc_cmd_sett, arg_hours); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-stats.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + block_start = command_line::get_arg(vm, arg_block_start); + block_stop = command_line::get_arg(vm, arg_block_stop); + bool do_inputs = command_line::get_arg(vm, arg_inputs); + bool do_outputs = command_line::get_arg(vm, arg_outputs); + bool do_ringsize = command_line::get_arg(vm, arg_ringsize); + bool do_hours = command_line::get_arg(vm, arg_hours); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + const uint64_t db_height = db->height(); + if (!block_stop) + block_stop = db_height; + MINFO("Starting from height " << block_start << ", stopping at height " << block_stop); + +/* + * The default output can be plotted with GnuPlot using these commands: +set key autotitle columnhead +set title "Monero Blockchain Growth" +set timefmt "%Y-%m-%d" +set xdata time +set xrange ["2014-04-17":*] +set format x "%Y-%m-%d" +set yrange [0:*] +set y2range [0:*] +set ylabel "Txs/Day" +set y2label "Bytes" +set y2tics nomirror +plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' using (timecolumn(1,"%Y-%m-%d")):7 axes x1y2 with lines + */ + + // spit out a comment that GnuPlot can use as an index + std::cout << ENDL << "# DATA" << ENDL; + std::cout << "Date\tBlocks/day\tBlocks\tTxs/Day\tTxs\tBytes/Day\tBytes"; + if (do_inputs) + std::cout << "\tInMin\tInMax\tInAvg"; + if (do_outputs) + std::cout << "\tOutMin\tOutMax\tOutAvg"; + if (do_ringsize) + std::cout << "\tRingMin\tRingMax\tRingAvg"; + if (do_hours) { + char buf[8]; + unsigned int i; + for (i=0; i<24; i++) { + sprintf(buf, "\t%02d:00", i); + std::cout << buf; + } + } + std::cout << ENDL; + + struct tm prevtm = {0}, currtm; + uint64_t prevsz = 0, currsz = 0; + uint64_t prevtxs = 0, currtxs = 0; + uint64_t currblks = 0; + uint64_t totins = 0, totouts = 0, totrings = 0; + uint32_t minins = 10, maxins = 0; + uint32_t minouts = 10, maxouts = 0; + uint32_t minrings = 50, maxrings = 0; + uint32_t io, tottxs = 0; + uint32_t txhr[24] = {0}; + unsigned int i; + + for (uint64_t h = block_start; h < block_stop; ++h) + { + cryptonote::blobdata bd = db->get_block_blob_from_height(h); + cryptonote::block blk; + if (!cryptonote::parse_and_validate_block_from_blob(bd, blk)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + time_t tt = blk.timestamp; + char timebuf[64]; + gmtime_r(&tt, &currtm); + if (!prevtm.tm_year) + prevtm = currtm; + // catch change of day + if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27)) + { + // check for timestamp fudging around month ends + if (prevtm.tm_mday == 1 && currtm.tm_mday > 27) + goto skip; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm); + prevtm = currtm; + std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; + prevsz += currsz; + currsz = 0; + currblks = 0; + prevtxs += currtxs; + currtxs = 0; + if (!tottxs) + tottxs = 1; + if (do_inputs) { + std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs; + minins = 10; maxins = 0; totins = 0; + } + if (do_outputs) { + std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs; + minouts = 10; maxouts = 0; totouts = 0; + } + if (do_ringsize) { + std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs; + minrings = 50; maxrings = 0; totrings = 0; + } + tottxs = 0; + if (do_hours) { + for (i=0; i<24; i++) { + std::cout << "\t" << txhr[i]; + txhr[i] = 0; + } + } + std::cout << ENDL; + } +skip: + currsz += bd.size(); + for (const auto& tx_id : blk.tx_hashes) + { + if (tx_id == crypto::null_hash) + { + throw std::runtime_error("Aborting: tx == null_hash"); + } + if (!db->get_tx_blob(tx_id, bd)) + { + throw std::runtime_error("Aborting: tx not found"); + } + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad txn from db"); + return 1; + } + currsz += bd.size(); + currtxs++; + if (do_hours) + txhr[currtm.tm_hour]++; + if (do_inputs) { + io = tx.vin.size(); + if (io < minins) + minins = io; + else if (io > maxins) + maxins = io; + totins += io; + } + if (do_ringsize) { + const cryptonote::txin_to_key& tx_in_to_key + = boost::get<cryptonote::txin_to_key>(tx.vin[0]); + io = tx_in_to_key.key_offsets.size(); + if (io < minrings) + minrings = io; + else if (io > maxrings) + maxrings = io; + totrings += io; + } + if (do_outputs) { + io = tx.vout.size(); + if (io < minouts) + minouts = io; + else if (io > maxouts) + maxouts = io; + totouts += io; + } + tottxs++; + } + currblks++; + + if (stop_requested) + break; + } + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Stats reporting error", 1); +} diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index beaad2abc..a8c46d661 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -304,7 +304,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem } if (m_cur_height % progress_interval == 0) { std::cout << refresh_string; - std::cout << "block " << m_cur_height << "/" << block_stop << std::flush; + std::cout << "block " << m_cur_height << "/" << block_stop << "\r" << std::flush; } } // NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported. @@ -479,7 +479,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s bytes_read += count_bytes(import_file, progress_interval, blocks, quit); h += blocks; std::cout << "\r" << "block height: " << h-1 << - " " << + " \r" << std::flush; // std::cout << refresh_string; diff --git a/src/common/base58.cpp b/src/common/base58.cpp index 75556cad9..b28a04f20 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -109,20 +109,8 @@ namespace tools assert(1 <= size && size <= sizeof(uint64_t)); uint64_t res = 0; - switch (9 - size) - { - case 1: res |= *data++; /* FALLTHRU */ - case 2: res <<= 8; res |= *data++; /* FALLTHRU */ - case 3: res <<= 8; res |= *data++; /* FALLTHRU */ - case 4: res <<= 8; res |= *data++; /* FALLTHRU */ - case 5: res <<= 8; res |= *data++; /* FALLTHRU */ - case 6: res <<= 8; res |= *data++; /* FALLTHRU */ - case 7: res <<= 8; res |= *data++; /* FALLTHRU */ - case 8: res <<= 8; res |= *data; break; - default: assert(false); - } - - return res; + memcpy(reinterpret_cast<uint8_t*>(&res) + sizeof(uint64_t) - size, data, size); + return SWAP64BE(res); } void uint_64_to_8be(uint64_t num, size_t size, uint8_t* data) diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index f2b270981..606a2c7b7 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -37,6 +37,7 @@ #include <boost/thread/mutex.hpp> #include <boost/thread/thread.hpp> #include <boost/algorithm/string/join.hpp> +#include <boost/optional.hpp> using namespace epee; namespace bf = boost::filesystem; @@ -119,10 +120,25 @@ get_builtin_ds(void) namespace tools { +static const char *get_record_name(int record_type) +{ + switch (record_type) + { + case DNS_TYPE_A: return "A"; + case DNS_TYPE_TXT: return "TXT"; + case DNS_TYPE_AAAA: return "AAAA"; + default: return "unknown"; + } +} + // fuck it, I'm tired of dealing with getnameinfo()/inet_ntop/etc -std::string ipv4_to_string(const char* src, size_t len) +boost::optional<std::string> ipv4_to_string(const char* src, size_t len) { - assert(len >= 4); + if (len < 4) + { + MERROR("Invalid IPv4 address: " << std::string(src, len)); + return boost::none; + } std::stringstream ss; unsigned int bytes[4]; @@ -140,9 +156,13 @@ std::string ipv4_to_string(const char* src, size_t len) // this obviously will need to change, but is here to reflect the above // stop-gap measure and to make the tests pass at least... -std::string ipv6_to_string(const char* src, size_t len) +boost::optional<std::string> ipv6_to_string(const char* src, size_t len) { - assert(len >= 8); + if (len < 8) + { + MERROR("Invalid IPv4 address: " << std::string(src, len)); + return boost::none; + } std::stringstream ss; unsigned int bytes[8]; @@ -162,8 +182,10 @@ std::string ipv6_to_string(const char* src, size_t len) return ss.str(); } -std::string txt_to_string(const char* src, size_t len) +boost::optional<std::string> txt_to_string(const char* src, size_t len) { + if (len == 0) + return boost::none; return std::string(src+1, len-1); } @@ -266,7 +288,7 @@ DNSResolver::~DNSResolver() } } -std::vector<std::string> DNSResolver::get_record(const std::string& url, int record_type, std::string (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid) +std::vector<std::string> DNSResolver::get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid) { std::vector<std::string> addresses; dnssec_available = false; @@ -289,7 +311,12 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec { for (size_t i=0; result->data[i] != NULL; i++) { - addresses.push_back((*reader)(result->data[i], result->len[i])); + boost::optional<std::string> res = (*reader)(result->data[i], result->len[i]); + if (res) + { + MINFO("Found \"" << *res << "\" in " << get_record_name(record_type) << " record for " << url); + addresses.push_back(*res); + } } } } diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index f46bca3dd..3a6ef68a1 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -30,6 +30,7 @@ #include <vector> #include <string> #include <functional> +#include <boost/optional/optional_fwd.hpp> namespace tools { @@ -143,7 +144,7 @@ private: * @return A vector of strings containing the requested record; or an empty vector */ // TODO: modify this to accommodate DNSSEC - std::vector<std::string> get_record(const std::string& url, int record_type, std::string (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); + std::vector<std::string> get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); /** * @brief Checks a string to see if it looks like a URL diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 47f01de65..d9f1f65c1 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -110,6 +110,7 @@ LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, const std { MCLOG(level, cat.c_str(), "PERF ----------"); performance_timers = new std::vector<LoggingPerformanceTimer*>(); + performance_timers->reserve(16); // how deep before realloc } else { diff --git a/src/crypto/aesb.c b/src/crypto/aesb.c index 5d57b8af4..8a22a4b93 100644 --- a/src/crypto/aesb.c +++ b/src/crypto/aesb.c @@ -19,6 +19,7 @@ Issue Date: 20/12/2007 */ #include <stdint.h> +#include "common/int-util.h" #if defined(__cplusplus) extern "C" @@ -50,7 +51,7 @@ extern "C" #define state_out(y,x) so(y,x,0); so(y,x,1); so(y,x,2); so(y,x,3) #define round(rm,y,x,k) rm(y,x,k,0); rm(y,x,k,1); rm(y,x,k,2); rm(y,x,k,3) #define to_byte(x) ((x) & 0xff) -#define bval(x,n) to_byte((x) >> (8 * (n))) +#define bval(x,n) to_byte(SWAP32LE(x) >> (8 * (n))) #define fwd_var(x,r,c)\ ( r == 0 ? ( c == 0 ? s(x,0) : c == 1 ? s(x,1) : c == 2 ? s(x,2) : s(x,3))\ @@ -58,7 +59,7 @@ extern "C" : r == 2 ? ( c == 0 ? s(x,2) : c == 1 ? s(x,3) : c == 2 ? s(x,0) : s(x,1))\ : ( c == 0 ? s(x,3) : c == 1 ? s(x,0) : c == 2 ? s(x,1) : s(x,2))) -#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ four_tables(x,t_use(f,n),fwd_var,rf1,c)) +#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ SWAP32LE(four_tables(x,t_use(f,n),fwd_var,rf1,c))) #define sb_data(w) {\ w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), w(0xc5),\ diff --git a/src/crypto/groestl.c b/src/crypto/groestl.c index c8258add3..d5e2989a8 100644 --- a/src/crypto/groestl.c +++ b/src/crypto/groestl.c @@ -20,9 +20,15 @@ const uint8_t shift_Values[2][8] = {{0,1,2,3,4,5,6,7},{1,3,5,7,0,2,4,6}}; const uint8_t indices_cyclic[15] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6}; +#if BYTE_ORDER == LITTLE_ENDIAN #define ROTATE_COLUMN_DOWN(v1, v2, amount_bytes, temp_var) {temp_var = (v1<<(8*amount_bytes))|(v2>>(8*(4-amount_bytes))); \ v2 = (v2<<(8*amount_bytes))|(v1>>(8*(4-amount_bytes))); \ v1 = temp_var;} +#else +#define ROTATE_COLUMN_DOWN(v1, v2, amount_bytes, temp_var) {temp_var = (v1>>(8*amount_bytes))|(v2<<(8*(4-amount_bytes))); \ + v2 = (v2>>(8*amount_bytes))|(v1<<(8*(4-amount_bytes))); \ + v1 = temp_var;} +#endif #define COLUMN(x,y,i,c0,c1,c2,c3,c4,c5,c6,c7,tv1,tv2,tu,tl,t) \ @@ -68,14 +74,14 @@ const uint8_t indices_cyclic[15] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6}; static void RND512P(uint8_t *x, uint32_t *y, uint32_t r) { uint32_t temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp; uint32_t* x32 = (uint32_t*)x; - x32[ 0] ^= 0x00000000^r; - x32[ 2] ^= 0x00000010^r; - x32[ 4] ^= 0x00000020^r; - x32[ 6] ^= 0x00000030^r; - x32[ 8] ^= 0x00000040^r; - x32[10] ^= 0x00000050^r; - x32[12] ^= 0x00000060^r; - x32[14] ^= 0x00000070^r; + x32[ 0] ^= SWAP32LE(0x00000000)^r; + x32[ 2] ^= SWAP32LE(0x00000010)^r; + x32[ 4] ^= SWAP32LE(0x00000020)^r; + x32[ 6] ^= SWAP32LE(0x00000030)^r; + x32[ 8] ^= SWAP32LE(0x00000040)^r; + x32[10] ^= SWAP32LE(0x00000050)^r; + x32[12] ^= SWAP32LE(0x00000060)^r; + x32[14] ^= SWAP32LE(0x00000070)^r; COLUMN(x,y, 0, 0, 2, 4, 6, 9, 11, 13, 15, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 2, 2, 4, 6, 8, 11, 13, 15, 1, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 4, 4, 6, 8, 10, 13, 15, 1, 3, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); @@ -91,21 +97,22 @@ static void RND512Q(uint8_t *x, uint32_t *y, uint32_t r) { uint32_t temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp; uint32_t* x32 = (uint32_t*)x; x32[ 0] = ~x32[ 0]; - x32[ 1] ^= 0xffffffff^r; + x32[ 1] ^= SWAP32LE(0xffffffff)^r; x32[ 2] = ~x32[ 2]; - x32[ 3] ^= 0xefffffff^r; + x32[ 3] ^= SWAP32LE(0xefffffff)^r; x32[ 4] = ~x32[ 4]; - x32[ 5] ^= 0xdfffffff^r; + x32[ 5] ^= SWAP32LE(0xdfffffff)^r; x32[ 6] = ~x32[ 6]; - x32[ 7] ^= 0xcfffffff^r; + x32[ 7] ^= SWAP32LE(0xcfffffff)^r; x32[ 8] = ~x32[ 8]; - x32[ 9] ^= 0xbfffffff^r; + x32[ 9] ^= SWAP32LE(0xbfffffff)^r; x32[10] = ~x32[10]; - x32[11] ^= 0xafffffff^r; + x32[11] ^= SWAP32LE(0xafffffff)^r; x32[12] = ~x32[12]; - x32[13] ^= 0x9fffffff^r; + x32[13] ^= SWAP32LE(0x9fffffff)^r; x32[14] = ~x32[14]; - x32[15] ^= 0x8fffffff^r; + x32[15] ^= SWAP32LE(0x8fffffff)^r; + COLUMN(x,y, 0, 2, 6, 10, 14, 1, 5, 9, 13, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 2, 4, 8, 12, 0, 3, 7, 11, 15, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 4, 6, 10, 14, 2, 5, 9, 13, 1, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); @@ -130,28 +137,28 @@ static void F512(uint32_t *h, const uint32_t *m) { } /* compute Q(m) */ - RND512Q((uint8_t*)z, y, 0x00000000); - RND512Q((uint8_t*)y, z, 0x01000000); - RND512Q((uint8_t*)z, y, 0x02000000); - RND512Q((uint8_t*)y, z, 0x03000000); - RND512Q((uint8_t*)z, y, 0x04000000); - RND512Q((uint8_t*)y, z, 0x05000000); - RND512Q((uint8_t*)z, y, 0x06000000); - RND512Q((uint8_t*)y, z, 0x07000000); - RND512Q((uint8_t*)z, y, 0x08000000); - RND512Q((uint8_t*)y, Qtmp, 0x09000000); + RND512Q((uint8_t*)z, y, SWAP32LE(0x00000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x01000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x02000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x03000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x04000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x05000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x06000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x07000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x08000000)); + RND512Q((uint8_t*)y, Qtmp, SWAP32LE(0x09000000)); /* compute P(h+m) */ - RND512P((uint8_t*)Ptmp, y, 0x00000000); - RND512P((uint8_t*)y, z, 0x00000001); - RND512P((uint8_t*)z, y, 0x00000002); - RND512P((uint8_t*)y, z, 0x00000003); - RND512P((uint8_t*)z, y, 0x00000004); - RND512P((uint8_t*)y, z, 0x00000005); - RND512P((uint8_t*)z, y, 0x00000006); - RND512P((uint8_t*)y, z, 0x00000007); - RND512P((uint8_t*)z, y, 0x00000008); - RND512P((uint8_t*)y, Ptmp, 0x00000009); + RND512P((uint8_t*)Ptmp, y, SWAP32LE(0x00000000)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000001)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000002)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000003)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000004)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000005)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000006)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000007)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000008)); + RND512P((uint8_t*)y, Ptmp, SWAP32LE(0x00000009)); /* compute P(h+m) + Q(m) + h */ for (i = 0; i < 2*COLS512; i++) { @@ -188,16 +195,16 @@ static void OutputTransformation(hashState *ctx) { for (j = 0; j < 2*COLS512; j++) { temp[j] = ctx->chaining[j]; } - RND512P((uint8_t*)temp, y, 0x00000000); - RND512P((uint8_t*)y, z, 0x00000001); - RND512P((uint8_t*)z, y, 0x00000002); - RND512P((uint8_t*)y, z, 0x00000003); - RND512P((uint8_t*)z, y, 0x00000004); - RND512P((uint8_t*)y, z, 0x00000005); - RND512P((uint8_t*)z, y, 0x00000006); - RND512P((uint8_t*)y, z, 0x00000007); - RND512P((uint8_t*)z, y, 0x00000008); - RND512P((uint8_t*)y, temp, 0x00000009); + RND512P((uint8_t*)temp, y, SWAP32LE(0x00000000)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000001)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000002)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000003)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000004)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000005)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000006)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000007)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000008)); + RND512P((uint8_t*)y, temp, SWAP32LE(0x00000009)); for (j = 0; j < 2*COLS512; j++) { ctx->chaining[j] ^= temp[j]; } @@ -213,7 +220,7 @@ static void Init(hashState* ctx) { } /* set initial value */ - ctx->chaining[2*COLS512-1] = u32BIG((uint32_t)HASH_BIT_LEN); + ctx->chaining[2*COLS512-1] = SWAP32LE(u32BIG((uint32_t)HASH_BIT_LEN)); /* set other variables */ ctx->buf_ptr = 0; diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index c4b368584..53594c569 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -29,7 +29,10 @@ #ifndef __tables_h #define __tables_h +#include "common/int-util.h" + +#if BYTE_ORDER == LITTLE_ENDIAN const uint32_t T[512] = {0xa5f432c6, 0xc6a597f4, 0x84976ff8, 0xf884eb97, 0x99b05eee, 0xee99c7b0, 0x8d8c7af6, 0xf68df78c, 0xd17e8ff, 0xff0de517, 0xbddc0ad6, 0xd6bdb7dc, 0xb1c816de, 0xdeb1a7c8, 0x54fc6d91, 0x915439fc , 0x50f09060, 0x6050c0f0, 0x3050702, 0x2030405, 0xa9e02ece, 0xcea987e0, 0x7d87d156, 0x567dac87, 0x192bcce7, 0xe719d52b, 0x62a613b5, 0xb56271a6, 0xe6317c4d, 0x4de69a31, 0x9ab559ec, 0xec9ac3b5 , 0x45cf408f, 0x8f4505cf, 0x9dbca31f, 0x1f9d3ebc, 0x40c04989, 0x894009c0, 0x879268fa, 0xfa87ef92, 0x153fd0ef, 0xef15c53f, 0xeb2694b2, 0xb2eb7f26, 0xc940ce8e, 0x8ec90740, 0xb1de6fb, 0xfb0bed1d @@ -62,5 +65,39 @@ const uint32_t T[512] = {0xa5f432c6, 0xc6a597f4, 0x84976ff8, 0xf884eb97, 0x99b05 , 0xb6c1ec2d, 0x2db65ac1, 0x22665a3c, 0x3c227866, 0x92adb815, 0x15922aad, 0x2060a9c9, 0xc9208960, 0x49db5c87, 0x874915db, 0xff1ab0aa, 0xaaff4f1a, 0x7888d850, 0x5078a088, 0x7a8e2ba5, 0xa57a518e , 0x8f8a8903, 0x38f068a, 0xf8134a59, 0x59f8b213, 0x809b9209, 0x980129b, 0x1739231a, 0x1a173439, 0xda751065, 0x65daca75, 0x315384d7, 0xd731b553, 0xc651d584, 0x84c61351, 0xb8d303d0, 0xd0b8bbd3 , 0xc35edc82, 0x82c31f5e, 0xb0cbe229, 0x29b052cb, 0x7799c35a, 0x5a77b499, 0x11332d1e, 0x1e113c33, 0xcb463d7b, 0x7bcbf646, 0xfc1fb7a8, 0xa8fc4b1f, 0xd6610c6d, 0x6dd6da61, 0x3a4e622c, 0x2c3a584e}; +#else +const uint32_t T[512] = {0xc632f4a5, 0xf497a5c6, 0xf86f9784, 0x97eb84f8, 0xee5eb099, 0xb0c799ee, 0xf67a8c8d, 0x8cf78df6, 0xffe8170d, 0x17e50dff, 0xd60adcbd, 0xdcb7bdd6, 0xde16c8b1, 0xc8a7b1de, 0x916dfc54, 0xfc395491 +, 0x6090f050, 0xf0c05060, 0x02070503, 0x05040302, 0xce2ee0a9, 0xe087a9ce, 0x56d1877d, 0x87ac7d56, 0xe7cc2b19, 0x2bd519e7, 0xb513a662, 0xa67162b5, 0x4d7c31e6, 0x319ae64d, 0xec59b59a, 0xb5c39aec +, 0x8f40cf45, 0xcf05458f, 0x1fa3bc9d, 0xbc3e9d1f, 0x8949c040, 0xc0094089, 0xfa689287, 0x92ef87fa, 0xefd03f15, 0x3fc515ef, 0xb29426eb, 0x267febb2, 0x8ece40c9, 0x4007c98e, 0xfbe61d0b, 0x1ded0bfb +, 0x416e2fec, 0x2f82ec41, 0xb31aa967, 0xa97d67b3, 0x5f431cfd, 0x1cbefd5f, 0x456025ea, 0x258aea45, 0x23f9dabf, 0xda46bf23, 0x535102f7, 0x02a6f753, 0xe445a196, 0xa1d396e4, 0x9b76ed5b, 0xed2d5b9b +, 0x75285dc2, 0x5deac275, 0xe1c5241c, 0x24d91ce1, 0x3dd4e9ae, 0xe97aae3d, 0x4cf2be6a, 0xbe986a4c, 0x6c82ee5a, 0xeed85a6c, 0x7ebdc341, 0xc3fc417e, 0xf5f30602, 0x06f102f5, 0x8352d14f, 0xd11d4f83 +, 0x688ce45c, 0xe4d05c68, 0x515607f4, 0x07a2f451, 0xd18d5c34, 0x5cb934d1, 0xf9e11808, 0x18e908f9, 0xe24cae93, 0xaedf93e2, 0xab3e9573, 0x954d73ab, 0x6297f553, 0xf5c45362, 0x2a6b413f, 0x41543f2a +, 0x081c140c, 0x14100c08, 0x9563f652, 0xf6315295, 0x46e9af65, 0xaf8c6546, 0x9d7fe25e, 0xe2215e9d, 0x30487828, 0x78602830, 0x37cff8a1, 0xf86ea137, 0x0a1b110f, 0x11140f0a, 0x2febc4b5, 0xc45eb52f +, 0x0e151b09, 0x1b1c090e, 0x247e5a36, 0x5a483624, 0x1badb69b, 0xb6369b1b, 0xdf98473d, 0x47a53ddf, 0xcda76a26, 0x6a8126cd, 0x4ef5bb69, 0xbb9c694e, 0x7f334ccd, 0x4cfecd7f, 0xea50ba9f, 0xbacf9fea +, 0x123f2d1b, 0x2d241b12, 0x1da4b99e, 0xb93a9e1d, 0x58c49c74, 0x9cb07458, 0x3446722e, 0x72682e34, 0x3641772d, 0x776c2d36, 0xdc11cdb2, 0xcda3b2dc, 0xb49d29ee, 0x2973eeb4, 0x5b4d16fb, 0x16b6fb5b +, 0xa4a501f6, 0x0153f6a4, 0x76a1d74d, 0xd7ec4d76, 0xb714a361, 0xa37561b7, 0x7d3449ce, 0x49face7d, 0x52df8d7b, 0x8da47b52, 0xdd9f423e, 0x42a13edd, 0x5ecd9371, 0x93bc715e, 0x13b1a297, 0xa2269713 +, 0xa6a204f5, 0x0457f5a6, 0xb901b868, 0xb86968b9, 0x00000000, 0x00000000, 0xc1b5742c, 0x74992cc1, 0x40e0a060, 0xa0806040, 0xe3c2211f, 0x21dd1fe3, 0x793a43c8, 0x43f2c879, 0xb69a2ced, 0x2c77edb6 +, 0xd40dd9be, 0xd9b3bed4, 0x8d47ca46, 0xca01468d, 0x671770d9, 0x70ced967, 0x72afdd4b, 0xdde44b72, 0x94ed79de, 0x7933de94, 0x98ff67d4, 0x672bd498, 0xb09323e8, 0x237be8b0, 0x855bde4a, 0xde114a85 +, 0xbb06bd6b, 0xbd6d6bbb, 0xc5bb7e2a, 0x7e912ac5, 0x4f7b34e5, 0x349ee54f, 0xedd73a16, 0x3ac116ed, 0x86d254c5, 0x5417c586, 0x9af862d7, 0x622fd79a, 0x6699ff55, 0xffcc5566, 0x11b6a794, 0xa7229411 +, 0x8ac04acf, 0x4a0fcf8a, 0xe9d93010, 0x30c910e9, 0x040e0a06, 0x0a080604, 0xfe669881, 0x98e781fe, 0xa0ab0bf0, 0x0b5bf0a0, 0x78b4cc44, 0xccf04478, 0x25f0d5ba, 0xd54aba25, 0x4b753ee3, 0x3e96e34b +, 0xa2ac0ef3, 0x0e5ff3a2, 0x5d4419fe, 0x19bafe5d, 0x80db5bc0, 0x5b1bc080, 0x0580858a, 0x850a8a05, 0x3fd3ecad, 0xec7ead3f, 0x21fedfbc, 0xdf42bc21, 0x70a8d848, 0xd8e04870, 0xf1fd0c04, 0x0cf904f1 +, 0x63197adf, 0x7ac6df63, 0x772f58c1, 0x58eec177, 0xaf309f75, 0x9f4575af, 0x42e7a563, 0xa5846342, 0x20705030, 0x50403020, 0xe5cb2e1a, 0x2ed11ae5, 0xfdef120e, 0x12e10efd, 0xbf08b76d, 0xb7656dbf +, 0x8155d44c, 0xd4194c81, 0x18243c14, 0x3c301418, 0x26795f35, 0x5f4c3526, 0xc3b2712f, 0x719d2fc3, 0xbe8638e1, 0x3867e1be, 0x35c8fda2, 0xfd6aa235, 0x88c74fcc, 0x4f0bcc88, 0x2e654b39, 0x4b5c392e +, 0x936af957, 0xf93d5793, 0x55580df2, 0x0daaf255, 0xfc619d82, 0x9de382fc, 0x7ab3c947, 0xc9f4477a, 0xc827efac, 0xef8bacc8, 0xba8832e7, 0x326fe7ba, 0x324f7d2b, 0x7d642b32, 0xe642a495, 0xa4d795e6 +, 0xc03bfba0, 0xfb9ba0c0, 0x19aab398, 0xb3329819, 0x9ef668d1, 0x6827d19e, 0xa322817f, 0x815d7fa3, 0x44eeaa66, 0xaa886644, 0x54d6827e, 0x82a87e54, 0x3bdde6ab, 0xe676ab3b, 0x0b959e83, 0x9e16830b +, 0x8cc945ca, 0x4503ca8c, 0xc7bc7b29, 0x7b9529c7, 0x6b056ed3, 0x6ed6d36b, 0x286c443c, 0x44503c28, 0xa72c8b79, 0x8b5579a7, 0xbc813de2, 0x3d63e2bc, 0x1631271d, 0x272c1d16, 0xad379a76, 0x9a4176ad +, 0xdb964d3b, 0x4dad3bdb, 0x649efa56, 0xfac85664, 0x74a6d24e, 0xd2e84e74, 0x1436221e, 0x22281e14, 0x92e476db, 0x763fdb92, 0x0c121e0a, 0x1e180a0c, 0x48fcb46c, 0xb4906c48, 0xb88f37e4, 0x376be4b8 +, 0x9f78e75d, 0xe7255d9f, 0xbd0fb26e, 0xb2616ebd, 0x43692aef, 0x2a86ef43, 0xc435f1a6, 0xf193a6c4, 0x39dae3a8, 0xe372a839, 0x31c6f7a4, 0xf762a431, 0xd38a5937, 0x59bd37d3, 0xf274868b, 0x86ff8bf2 +, 0xd5835632, 0x56b132d5, 0x8b4ec543, 0xc50d438b, 0x6e85eb59, 0xebdc596e, 0xda18c2b7, 0xc2afb7da, 0x018e8f8c, 0x8f028c01, 0xb11dac64, 0xac7964b1, 0x9cf16dd2, 0x6d23d29c, 0x49723be0, 0x3b92e049 +, 0xd81fc7b4, 0xc7abb4d8, 0xacb915fa, 0x1543faac, 0xf3fa0907, 0x09fd07f3, 0xcfa06f25, 0x6f8525cf, 0xca20eaaf, 0xea8fafca, 0xf47d898e, 0x89f38ef4, 0x476720e9, 0x208ee947, 0x10382818, 0x28201810 +, 0x6f0b64d5, 0x64ded56f, 0xf0738388, 0x83fb88f0, 0x4afbb16f, 0xb1946f4a, 0x5cca9672, 0x96b8725c, 0x38546c24, 0x6c702438, 0x575f08f1, 0x08aef157, 0x732152c7, 0x52e6c773, 0x9764f351, 0xf3355197 +, 0xcbae6523, 0x658d23cb, 0xa125847c, 0x84597ca1, 0xe857bf9c, 0xbfcb9ce8, 0x3e5d6321, 0x637c213e, 0x96ea7cdd, 0x7c37dd96, 0x611e7fdc, 0x7fc2dc61, 0x0d9c9186, 0x911a860d, 0x0f9b9485, 0x941e850f +, 0xe04bab90, 0xabdb90e0, 0x7cbac642, 0xc6f8427c, 0x712657c4, 0x57e2c471, 0xcc29e5aa, 0xe583aacc, 0x90e373d8, 0x733bd890, 0x06090f05, 0x0f0c0506, 0xf7f40301, 0x03f501f7, 0x1c2a3612, 0x3638121c +, 0xc23cfea3, 0xfe9fa3c2, 0x6a8be15f, 0xe1d45f6a, 0xaebe10f9, 0x1047f9ae, 0x69026bd0, 0x6bd2d069, 0x17bfa891, 0xa82e9117, 0x9971e858, 0xe8295899, 0x3a536927, 0x6974273a, 0x27f7d0b9, 0xd04eb927 +, 0xd9914838, 0x48a938d9, 0xebde3513, 0x35cd13eb, 0x2be5ceb3, 0xce56b32b, 0x22775533, 0x55443322, 0xd204d6bb, 0xd6bfbbd2, 0xa9399070, 0x904970a9, 0x07878089, 0x800e8907, 0x33c1f2a7, 0xf266a733 +, 0x2decc1b6, 0xc15ab62d, 0x3c5a6622, 0x6678223c, 0x15b8ad92, 0xad2a9215, 0xc9a96020, 0x608920c9, 0x875cdb49, 0xdb154987, 0xaab01aff, 0x1a4fffaa, 0x50d88878, 0x88a07850, 0xa52b8e7a, 0x8e517aa5 +, 0x03898a8f, 0x8a068f03, 0x594a13f8, 0x13b2f859, 0x09929b80, 0x9b128009, 0x1a233917, 0x3934171a, 0x651075da, 0x75cada65, 0xd7845331, 0x53b531d7, 0x84d551c6, 0x5113c684, 0xd003d3b8, 0xd3bbb8d0 +, 0x82dc5ec3, 0x5e1fc382, 0x29e2cbb0, 0xcb52b029, 0x5ac39977, 0x99b4775a, 0x1e2d3311, 0x333c111e, 0x7b3d46cb, 0x46f6cb7b, 0xa8b71ffc, 0x1f4bfca8, 0x6d0c61d6, 0x61dad66d, 0x2c624e3a, 0x4e583a2c}; +#endif #endif /* __tables_h */ diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 42f272e34..43ce32957 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -36,7 +36,14 @@ #include "keccak.h" void hash_permutation(union hash_state *state) { +#if BYTE_ORDER == LITTLE_ENDIAN keccakf((uint64_t*)state, 24); +#else + uint64_t le_state[25]; + memcpy_swap64le(le_state, state, 25); + keccakf(le_state, 24); + memcpy_swap64le(state, le_state, 25); +#endif } void hash_process(union hash_state *state, const uint8_t *buf, size_t count) { diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index b5946036e..b095b5ce2 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -145,7 +145,7 @@ void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md) #define IS_ALIGNED_64(p) (0 == (7 & ((const char*)(p) - (const char*)0))) #define KECCAK_PROCESS_BLOCK(st, block) { \ for (int i_ = 0; i_ < KECCAK_WORDS; i_++){ \ - ((st))[i_] ^= ((block))[i_]; \ + ((st))[i_] ^= swap64le(((block))[i_]); \ }; \ keccakf(st, KECCAK_ROUNDS); } @@ -207,7 +207,8 @@ void keccak_finish(KECCAK_CTX * ctx, uint8_t *md){ } static_assert(KECCAK_BLOCKLEN > KECCAK_DIGESTSIZE, ""); + static_assert(KECCAK_DIGESTSIZE % sizeof(uint64_t) == 0, ""); if (md) { - memcpy(md, ctx->hash, KECCAK_DIGESTSIZE); + memcpy_swap64le(md, ctx->hash, KECCAK_DIGESTSIZE / sizeof(uint64_t)); } } diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 40cfb0461..ed61e1017 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -47,8 +47,8 @@ #define INIT_SIZE_BLK 8 #define INIT_SIZE_BYTE (INIT_SIZE_BLK * AES_BLOCK_SIZE) -extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); -extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +extern void aesb_single_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); #define VARIANT1_1(p) \ do if (variant == 1) \ @@ -1408,9 +1408,6 @@ static void (*const extra_hashes[4])(const void *, size_t, char *) = { hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein }; -extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); -extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); - static size_t e2i(const uint8_t* a, size_t count) { return (*((uint64_t*)a) / AES_BLOCK_SIZE) & (count - 1); } static void mul(const uint8_t* a, const uint8_t* b, uint8_t* res) { diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index 1dc1ad71d..edbc2c561 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -139,6 +139,15 @@ DISABLE_VS_WARNINGS(4244 4345) m_creation_timestamp = 0; } //----------------------------------------------------------------- + void account_base::deinit() + { + try{ + m_keys.get_device().disconnect(); + } catch (const std::exception &e){ + MERROR("Device disconnect exception: " << e.what()); + } + } + //----------------------------------------------------------------- void account_base::forget_spend_key() { m_keys.m_spend_secret_key = crypto::secret_key(); @@ -206,11 +215,16 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::create_from_device(hw::device &hwdev) { m_keys.set_device(hwdev); - MCDEBUG("ledger", "device type: "<<typeid(hwdev).name()); - hwdev.init(); - hwdev.connect(); - hwdev.get_public_address(m_keys.m_account_address); - hwdev.get_secret_keys(m_keys.m_view_secret_key, m_keys.m_spend_secret_key); + MCDEBUG("device", "device type: "<<typeid(hwdev).name()); + CHECK_AND_ASSERT_THROW_MES(hwdev.init(), "Device init failed"); + CHECK_AND_ASSERT_THROW_MES(hwdev.connect(), "Device connect failed"); + try { + CHECK_AND_ASSERT_THROW_MES(hwdev.get_public_address(m_keys.m_account_address), "Cannot get a device address"); + CHECK_AND_ASSERT_THROW_MES(hwdev.get_secret_keys(m_keys.m_view_secret_key, m_keys.m_spend_secret_key), "Cannot get device secret"); + } catch (const std::exception &e){ + hwdev.disconnect(); + throw; + } struct tm timestamp = {0}; timestamp.tm_year = 2014 - 1900; // year 2014 timestamp.tm_mon = 4 - 1; // month april diff --git a/src/cryptonote_basic/account.h b/src/cryptonote_basic/account.h index 98bba55b1..021f84029 100644 --- a/src/cryptonote_basic/account.h +++ b/src/cryptonote_basic/account.h @@ -89,6 +89,7 @@ namespace cryptonote hw::device& get_device() const {return m_keys.get_device();} void set_device( hw::device &hwdev) {m_keys.set_device(hwdev);} + void deinit(); uint64_t get_createtime() const { return m_creation_timestamp; } void set_createtime(uint64_t val) { m_creation_timestamp = val; } diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index b18ef1c5c..c4e10851e 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -328,7 +328,7 @@ bool parse_hash256(const std::string str_hash, crypto::hash& hash) bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf); if (!res || buf.size() != sizeof(crypto::hash)) { - std::cout << "invalid hash format: <" << str_hash << '>' << std::endl; + MERROR("invalid hash format: " << str_hash); return false; } else diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index f05b25901..87a394918 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -56,12 +56,13 @@ static uint8_t get_block_version(const cryptonote::block &b) HardFork::HardFork(cryptonote::BlockchainDB &db, uint8_t original_version, uint64_t original_version_till_height, time_t forked_time, time_t update_time, uint64_t window_size, uint8_t default_threshold_percent): db(db), - original_version(original_version), - original_version_till_height(original_version_till_height), forked_time(forked_time), update_time(update_time), window_size(window_size), - default_threshold_percent(default_threshold_percent) + default_threshold_percent(default_threshold_percent), + original_version(original_version), + original_version_till_height(original_version_till_height), + current_fork_index(0) { if (window_size == 0) throw "window_size needs to be strictly positive"; diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index c62eeb738..496678b5e 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -113,6 +113,8 @@ #define P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT 5000 //5 seconds #define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT 70 #define P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT 2 +#define P2P_DEFAULT_LIMIT_RATE_UP 2048 // kB/s +#define P2P_DEFAULT_LIMIT_RATE_DOWN 8192 // kB/s #define P2P_FAILED_ADDR_FORGET_SECONDS (60*60) //1 hour #define P2P_IP_BLOCKTIME (60*60*24) //24 hour diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 77b6d0b69..19cc90b61 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2530,7 +2530,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } - if (hf_version >= HF_VERSION_MIN_MIXIN_10 && mixin != 10) + if (((hf_version == HF_VERSION_MIN_MIXIN_10 || hf_version == HF_VERSION_MIN_MIXIN_10+1) && mixin != 10) || (hf_version >= HF_VERSION_MIN_MIXIN_10+2 && mixin > 10)) { MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); tvc.m_low_mixin = true; @@ -3786,8 +3786,7 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) } //------------------------------------------------------------------ -//FIXME: unused parameter txs -void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, std::unordered_map<crypto::hash, cryptonote::transaction> &txs) const +void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) const { try { @@ -4164,21 +4163,19 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete offsets.second.erase(last, offsets.second.end()); } - // [output] stores all transactions for each tx_out_index::hash found - std::vector<std::unordered_map<crypto::hash, cryptonote::transaction>> transactions(amounts.size()); - + // gather all the output keys threads = tpool.get_max_concurrency(); if (!m_db->can_thread_bulk_indices()) threads = 1; - if (threads > 1) + if (threads > 1 && amounts.size() > 1) { tools::threadpool::waiter waiter; for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i])), true); + tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount])), true); } waiter.wait(&tpool); } @@ -4187,7 +4184,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - output_scan_worker(amount, offset_map[amount], tx_map[amount], transactions[i]); + output_scan_worker(amount, offset_map[amount], tx_map[amount]); } } @@ -4254,9 +4251,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete return true; } -void Blockchain::add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta) +void Blockchain::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta) { - m_db->add_txpool_tx(tx, meta); + m_db->add_txpool_tx(txid, blob, meta); } void Blockchain::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta) diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index f140d7719..dfe833fb4 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -916,11 +916,9 @@ namespace cryptonote * @param amount the amount * @param offsets the indices (indexed to the amount) of the outputs * @param outputs return-by-reference the outputs collected - * @param txs unused, candidate for removal */ void output_scan_worker(const uint64_t amount,const std::vector<uint64_t> &offsets, - std::vector<output_data_t> &outputs, std::unordered_map<crypto::hash, - cryptonote::transaction> &txs) const; + std::vector<output_data_t> &outputs) const; /** * @brief computes the "short" and "long" hashes for a set of blocks @@ -939,7 +937,7 @@ namespace cryptonote */ std::list<std::pair<block_extended_info,std::vector<crypto::hash>>> get_alternative_chains() const; - void add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta); + void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta); void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta); void remove_txpool_tx(const crypto::hash &txid); uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index d8c38bf9e..10ab3fe65 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -909,7 +909,7 @@ namespace cryptonote } const size_t weight = get_transaction_weight(results[i].tx, it->size()); - ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -1127,7 +1127,7 @@ namespace cryptonote blobdata bl; t_serializable_object_to_blob(tx, bl); size_t tx_weight = get_transaction_weight(tx, bl.size()); - return add_new_tx(tx, tx_hash, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); + return add_new_tx(tx, tx_hash, bl, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -1135,7 +1135,7 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { if (keeped_by_block) get_blockchain_storage().on_new_tx_from_block(tx); @@ -1153,7 +1153,7 @@ namespace cryptonote } uint8_t version = m_blockchain_storage.get_current_hard_fork_version(); - return m_mempool.add_tx(tx, tx_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); + return m_mempool.add_tx(tx, tx_hash, blob, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); } //----------------------------------------------------------------------------------------------- bool core::relay_txpool_transactions() @@ -1716,7 +1716,8 @@ namespace cryptonote for (size_t n = 0; n < sizeof(seconds)/sizeof(seconds[0]); ++n) { unsigned int b = 0; - for (time_t ts: timestamps) b += ts >= now - seconds[n]; + const time_t time_boundary = now - static_cast<time_t>(seconds[n]); + for (time_t ts: timestamps) b += ts >= time_boundary; const double p = probability(b, seconds[n] / DIFFICULTY_TARGET_V2); MDEBUG("blocks in the last " << seconds[n] / 60 << " minutes: " << b << " (probability " << p << ")"); if (p < threshold) diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 80c452f53..2eb6c842b 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -117,7 +117,7 @@ namespace cryptonote * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * - * @return true if the transaction made it to the transaction pool, otherwise false + * @return true if the transaction was accepted, false otherwise */ bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); @@ -133,7 +133,7 @@ namespace cryptonote * @param relayed whether or not the transactions were relayed to us * @param do_not_relay whether to prevent the transactions from being relayed * - * @return true if the transactions made it to the transaction pool, otherwise false + * @return true if the transactions were accepted, false otherwise */ bool handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); @@ -783,13 +783,14 @@ namespace cryptonote * @copydoc add_new_tx(transaction&, tx_verification_context&, bool) * * @param tx_hash the transaction's hash + * @param blob the transaction as a blob * @param tx_prefix_hash the transaction prefix' hash * @param tx_weight the weight of the transaction * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 553a22298..e2900916b 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -111,7 +111,7 @@ namespace cryptonote } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { // this should already be called with that lock, but let's make it explicit for clarity CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -247,9 +247,11 @@ namespace cryptonote memset(meta.padding, 0, sizeof(meta.padding)); try { + if (kept_by_block) + m_parsed_tx_cache.insert(std::make_pair(id, tx)); CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); - m_blockchain.add_txpool_tx(tx, meta); + m_blockchain.add_txpool_tx(id, blob, meta); if (!insert_key_images(tx, id, kept_by_block)) return false; m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); @@ -288,12 +290,13 @@ namespace cryptonote try { + if (kept_by_block) + m_parsed_tx_cache.insert(std::make_pair(id, tx)); CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); - const crypto::hash txid = get_transaction_hash(tx); - m_blockchain.remove_txpool_tx(txid); - m_blockchain.add_txpool_tx(tx, meta); - if (!insert_key_images(tx, txid, kept_by_block)) + m_blockchain.remove_txpool_tx(id); + m_blockchain.add_txpool_tx(id, blob, meta); + if (!insert_key_images(tx, id, kept_by_block)) return false; m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } @@ -324,9 +327,11 @@ namespace cryptonote { crypto::hash h = null_hash; size_t blob_size = 0; - if (!get_transaction_hash(tx, h, blob_size) || blob_size == 0) + cryptonote::blobdata bl; + t_serializable_object_to_blob(tx, bl); + if (bl.size() == 0 || !get_transaction_hash(tx, h)) return false; - return add_tx(tx, h, get_transaction_weight(tx, blob_size), tvc, keeped_by_block, relayed, do_not_relay, version); + return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, keeped_by_block, relayed, do_not_relay, version); } //--------------------------------------------------------------------------------- size_t tx_memory_pool::get_txpool_weight() const @@ -379,11 +384,11 @@ namespace cryptonote return; } // remove first, in case this throws, so key images aren't removed - MINFO("Pruning tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruning tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_weight -= it->first.second; + m_txpool_weight -= meta.weight; remove_transaction_keyimages(tx, txid); - MINFO("Pruned tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_txs_by_fee_and_receive_time.erase(it--); changed = true; } @@ -454,8 +459,6 @@ namespace cryptonote CRITICAL_REGION_LOCAL1(m_blockchain); auto sorted_it = find_tx_in_sorted_container(id); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) - return false; try { @@ -467,7 +470,12 @@ namespace cryptonote return false; } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(id); - if (!parse_and_validate_tx_from_blob(txblob, tx)) + auto ci = m_parsed_tx_cache.find(id); + if (ci != m_parsed_tx_cache.end()) + { + tx = ci->second; + } + else if (!parse_and_validate_tx_from_blob(txblob, tx)) { MERROR("Failed to parse tx from txpool"); return false; @@ -489,7 +497,8 @@ namespace cryptonote return false; } - m_txs_by_fee_and_receive_time.erase(sorted_it); + if (sorted_it != m_txs_by_fee_and_receive_time.end()) + m_txs_by_fee_and_receive_time.erase(sorted_it); ++m_cookie; return true; } @@ -910,6 +919,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); m_input_cache.clear(); + m_parsed_tx_cache.clear(); return true; } //--------------------------------------------------------------------------------- @@ -917,6 +927,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); m_input_cache.clear(); + m_parsed_tx_cache.clear(); return true; } //--------------------------------------------------------------------------------- diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 7a0cc23bf..670d70d77 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -107,7 +107,7 @@ namespace cryptonote * @param id the transaction's hash * @param tx_weight the transaction's weight */ - bool add_tx(transaction &tx, const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -510,7 +510,7 @@ namespace cryptonote * @param txd the transaction to check (and info about it) * @param txid the txid of the transaction to check * @param txblob the transaction blob to check - * @param tx the parsed transaction prefix, if successful + * @param tx the parsed transaction, if successful * * @return true if the transaction is good to go, otherwise false */ @@ -584,6 +584,8 @@ private: size_t m_txpool_weight; mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache; + + std::unordered_map<crypto::hash, transaction> m_parsed_tx_cache; }; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 34d74d642..5ae9851a7 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -567,8 +567,8 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u for (auto & header : res.headers) { if (!first) - std::cout << std::endl; - std::cout + tools::msg_writer() << "" << std::endl; + tools::msg_writer() << "height: " << header.height << ", timestamp: " << header.timestamp << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl @@ -1313,7 +1313,7 @@ bool t_rpc_command_executor::out_peers(uint64_t limit) } } - std::cout << "Max number of out peers set to " << limit << std::endl; + tools::msg_writer() << "Max number of out peers set to " << limit << std::endl; return true; } @@ -1345,7 +1345,7 @@ bool t_rpc_command_executor::in_peers(uint64_t limit) } } - std::cout << "Max number of in peers set to " << limit << std::endl; + tools::msg_writer() << "Max number of in peers set to " << limit << std::endl; return true; } diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index 727134f75..91d670b73 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -44,6 +44,7 @@ set(device_headers device.hpp device_io.hpp device_default.hpp + device_cold.hpp log.hpp ) @@ -72,5 +73,6 @@ target_link_libraries(device cncrypto ringct_basic ${OPENSSL_CRYPTO_LIBRARIES} + ${Boost_SERIALIZATION_LIBRARY} PRIVATE ${EXTRA_LIBRARIES}) diff --git a/src/device/device.hpp b/src/device/device.hpp index 815a0ab93..dd9ad4332 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -47,6 +47,7 @@ #include "crypto/crypto.h" #include "crypto/chacha.h" #include "ringct/rctTypes.h" +#include "cryptonote_config.h" #ifndef USE_DEVICE_LEDGER @@ -99,10 +100,17 @@ namespace hw { enum device_type { SOFTWARE = 0, - LEDGER = 1 + LEDGER = 1, + TREZOR = 2 }; + enum device_protocol_t { + PROTOCOL_DEFAULT, + PROTOCOL_PROXY, // Originally defined by Ledger + PROTOCOL_COLD, // Originally defined by Trezor + }; + /* ======================================================================= */ /* SETUP/TEARDOWN */ /* ======================================================================= */ @@ -120,6 +128,7 @@ namespace hw { virtual device_type get_type() const = 0; + virtual device_protocol_t device_protocol() const { return PROTOCOL_DEFAULT; }; /* ======================================================================= */ /* LOCKER */ @@ -204,6 +213,11 @@ namespace hw { virtual bool close_tx(void) = 0; + virtual bool has_ki_cold_sync(void) const { return false; } + virtual bool has_tx_cold_sign(void) const { return false; } + + virtual void set_network_type(cryptonote::network_type network_type) { } + protected: device_mode mode; } ; diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp new file mode 100644 index 000000000..22128cec1 --- /dev/null +++ b/src/device/device_cold.hpp @@ -0,0 +1,71 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_COLD_H +#define MONERO_DEVICE_COLD_H + +#include "wallet/wallet2.h" +#include <boost/function.hpp> + + +namespace hw { + + typedef struct wallet_shim { + boost::function<crypto::public_key (const tools::wallet2::transfer_details &td)> get_tx_pub_key_from_received_outs; + } wallet_shim; + + class tx_aux_data { + public: + std::vector<std::string> tx_device_aux; // device generated aux data + std::vector<cryptonote::address_parse_info> tx_recipients; // as entered by user + }; + + class device_cold { + public: + + using exported_key_image = std::vector<std::pair<crypto::key_image, crypto::signature>>; + + /** + * Key image sync with the cold protocol. + */ + virtual void ki_sync(wallet_shim * wallet, + const std::vector<::tools::wallet2::transfer_details> & transfers, + exported_key_image & ski) =0; + + /** + * Signs unsigned transaction with the cold protocol. + */ + virtual void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + ::tools::wallet2::signed_tx_set & signed_tx, + tx_aux_data & aux_data) =0; + }; +} + +#endif //MONERO_DEVICE_COLD_H diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 68f40d91e..1e3d80949 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -69,17 +69,17 @@ namespace hw { } bool device_default::init(void) { - dfns(); + return true; } bool device_default::release() { - dfns(); + return true; } bool device_default::connect(void) { - dfns(); + return true; } bool device_default::disconnect() { - dfns(); + return true; } bool device_default::set_mode(device_mode mode) { diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index dde69fbfd..2f5beb044 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -141,6 +141,7 @@ namespace hw { bool set_mode(device_mode mode) override; device_type get_type() const override {return device_type::LEDGER;}; + device_protocol_t device_protocol() const override { return PROTOCOL_PROXY; }; /* ======================================================================= */ /* LOCKER */ diff --git a/src/device_trezor/CMakeLists.txt b/src/device_trezor/CMakeLists.txt new file mode 100644 index 000000000..c555e9fcd --- /dev/null +++ b/src/device_trezor/CMakeLists.txt @@ -0,0 +1,123 @@ +# Copyright (c) 2014-2017, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(TREZOR_PROTOB_H + trezor/messages/messages.pb.h + trezor/messages/messages-common.pb.h + trezor/messages/messages-management.pb.h + trezor/messages/messages-monero.pb.h +) + +set(TREZOR_PROTOB_CPP + trezor/messages/messages.pb.cc + trezor/messages/messages-common.pb.cc + trezor/messages/messages-management.pb.cc + trezor/messages/messages-monero.pb.cc +) + +set(trezor_headers + trezor/exceptions.hpp + trezor/messages_map.hpp + trezor/protocol.hpp + trezor/transport.hpp + device_trezor_base.hpp + device_trezor.hpp + trezor.hpp + ${TREZOR_PROTOB_H} +) + +set(trezor_sources + trezor/messages_map.cpp + trezor/protocol.cpp + trezor/transport.cpp + device_trezor_base.cpp + device_trezor.cpp + ${TREZOR_PROTOB_CPP} +) + +set(trezor_private_headers) + + +include(FindProtobuf) +find_package(Protobuf) # REQUIRED + +# Test for HAVE_PROTOBUF from the parent +if(Protobuf_FOUND AND HAVE_PROTOBUF) + if ("$ENV{PYTHON3}" STREQUAL "") + set(PYTHON3 "python3") + else() + set(PYTHON3 "$ENV{PYTHON3}" CACHE INTERNAL "Copied from environment variable") + endif() + + execute_process(COMMAND ${PYTHON3} tools/build_protob.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) + if(RET) + message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})." + "OUT: ${OUT}, ERR: ${ERR}." + "Please read src/device_trezor/trezor/tools/README.md") + else() + message(STATUS "Trezor protobuf messages regenerated ${OUT}") + set(TREZOR_PROTOBUF_GENERATED 1) + endif() +endif() + + +if(TREZOR_PROTOBUF_GENERATED) + message(STATUS "Trezor support enabled") + + add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0) + + monero_private_headers(device_trezor + ${device_private_headers} + ${PROTOBUF_INCLUDE_DIR}) + + monero_add_library(device_trezor + ${trezor_sources} + ${trezor_headers} + ${trezor_private_headers}) + + target_link_libraries(device_trezor + PUBLIC + device + cncrypto + ringct_basic + cryptonote_core + common + ${SODIUM_LIBRARY} + ${Boost_CHRONO_LIBRARY} + ${PROTOBUF_LIBRARY} + PRIVATE + ${EXTRA_LIBRARIES}) + + # set(WITH_DEVICE_TREZOR 1 PARENT_SCOPE) + # add_definitions(-DWITH_DEVICE_TREZOR=1) + +else() + monero_private_headers(device_trezor) + monero_add_library(device_trezor device_trezor.cpp) + target_link_libraries(device_trezor PUBLIC cncrypto) +endif() diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp new file mode 100644 index 000000000..f55cbb15d --- /dev/null +++ b/src/device_trezor/device_trezor.cpp @@ -0,0 +1,364 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "device_trezor.hpp" + +namespace hw { +namespace trezor { + +#if WITH_DEVICE_TREZOR + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" + +#define HW_TREZOR_NAME "Trezor" +#define HW_TREZOR_NAME_LITE "TrezorLite" + + static device_trezor *trezor_device = nullptr; + static device_trezor *ensure_trezor_device(){ + if (!trezor_device) { + trezor_device = new device_trezor(); + trezor_device->set_name(HW_TREZOR_NAME); + } + return trezor_device; + } + + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry) { + registry.insert(std::make_pair(HW_TREZOR_NAME, std::unique_ptr<device>(ensure_trezor_device()))); + } + + void register_all() { + hw::register_device(HW_TREZOR_NAME, ensure_trezor_device()); + } + + device_trezor::device_trezor() { + + } + + device_trezor::~device_trezor() { + try { + disconnect(); + release(); + } catch(std::exception const& e){ + MWARNING("Could not disconnect and release: " << e.what()); + } + } + + /* ======================================================================= */ + /* WALLET & ADDRESS */ + /* ======================================================================= */ + + bool device_trezor::get_public_address(cryptonote::account_public_address &pubkey) { + try { + auto res = get_address(); + + cryptonote::address_parse_info info{}; + bool r = cryptonote::get_account_address_from_str(info, this->network_type, res->address()); + CHECK_AND_ASSERT_MES(r, false, "Could not parse returned address. Address parse failed: " + res->address()); + CHECK_AND_ASSERT_MES(!info.is_subaddress, false, "Trezor returned a sub address"); + + pubkey = info.address; + return true; + + } catch(std::exception const& e){ + MERROR("Get public address exception: " << e.what()); + return false; + } + } + + bool device_trezor::get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) { + try { + MDEBUG("Loading view-only key from the Trezor. Please check the Trezor for a confirmation."); + auto res = get_view_key(); + CHECK_AND_ASSERT_MES(res->watch_key().size() == 32, false, "Trezor returned invalid view key"); + + spendkey = crypto::null_skey; // not given + memcpy(viewkey.data, res->watch_key().data(), 32); + + return true; + + } catch(std::exception const& e){ + MERROR("Get secret keys exception: " << e.what()); + return false; + } + } + + /* ======================================================================= */ + /* Helpers */ + /* ======================================================================= */ + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + std::shared_ptr<messages::monero::MoneroAddress> device_trezor::get_address( + const boost::optional<std::vector<uint32_t>> & path, + const boost::optional<cryptonote::network_type> & network_type){ + AUTO_LOCK_CMD(); + require_connected(); + test_ping(); + + auto req = std::make_shared<messages::monero::MoneroGetAddress>(); + this->set_msg_addr<messages::monero::MoneroGetAddress>(req.get(), path, network_type); + + auto response = this->client_exchange<messages::monero::MoneroAddress>(req); + MTRACE("Get address response received"); + return response; + } + + std::shared_ptr<messages::monero::MoneroWatchKey> device_trezor::get_view_key( + const boost::optional<std::vector<uint32_t>> & path, + const boost::optional<cryptonote::network_type> & network_type){ + AUTO_LOCK_CMD(); + require_connected(); + test_ping(); + + auto req = std::make_shared<messages::monero::MoneroGetWatchKey>(); + this->set_msg_addr<messages::monero::MoneroGetWatchKey>(req.get(), path, network_type); + + auto response = this->client_exchange<messages::monero::MoneroWatchKey>(req); + MTRACE("Get watch key response received"); + return response; + } + + void device_trezor::ki_sync(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + hw::device_cold::exported_key_image & ski) + { + AUTO_LOCK_CMD(); + require_connected(); + test_ping(); + + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> req; + + std::vector<protocol::ki::MoneroTransferDetails> mtds; + std::vector<protocol::ki::MoneroExportedKeyImage> kis; + protocol::ki::key_image_data(wallet, transfers, mtds); + protocol::ki::generate_commitment(mtds, transfers, req); + + this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get()); + auto ack1 = this->client_exchange<messages::monero::MoneroKeyImageExportInitAck>(req); + + const auto batch_size = 10; + const auto num_batches = (mtds.size() + batch_size - 1) / batch_size; + for(uint64_t cur = 0; cur < num_batches; ++cur){ + auto step_req = std::make_shared<messages::monero::MoneroKeyImageSyncStepRequest>(); + auto idx_finish = std::min(static_cast<uint64_t>((cur + 1) * batch_size), static_cast<uint64_t>(mtds.size())); + for(uint64_t idx = cur * batch_size; idx < idx_finish; ++idx){ + auto added_tdis = step_req->add_tdis(); + CHECK_AND_ASSERT_THROW_MES(idx < mtds.size(), "Invalid transfer detail index"); + *added_tdis = mtds[idx]; + } + + auto step_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncStepAck>(step_req); + auto kis_size = step_ack->kis_size(); + kis.reserve(static_cast<size_t>(kis_size)); + for(int i = 0; i < kis_size; ++i){ + auto ckis = step_ack->kis(i); + kis.push_back(ckis); + } + + MTRACE("Batch " << cur << " / " << num_batches << " batches processed"); + } + + auto final_req = std::make_shared<messages::monero::MoneroKeyImageSyncFinalRequest>(); + auto final_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncFinalAck>(final_req); + ski.reserve(kis.size()); + + for(auto & sub : kis){ + char buff[32*3]; + protocol::crypto::chacha::decrypt(sub.blob().data(), sub.blob().size(), + reinterpret_cast<const uint8_t *>(final_ack->enc_key().data()), + reinterpret_cast<const uint8_t *>(sub.iv().data()), buff); + + ::crypto::signature sig{}; + ::crypto::key_image ki; + memcpy(ki.data, buff, 32); + memcpy(sig.c.data, buff + 32, 32); + memcpy(sig.r.data, buff + 64, 32); + ski.push_back(std::make_pair(ki, sig)); + } + } + + + void device_trezor::tx_sign(wallet_shim * wallet, + const tools::wallet2::unsigned_tx_set & unsigned_tx, + tools::wallet2::signed_tx_set & signed_tx, + hw::tx_aux_data & aux_data) + { + CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); + size_t num_tx = unsigned_tx.txes.size(); + signed_tx.key_images.clear(); + signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); + + for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) { + std::shared_ptr<protocol::tx::Signer> signer; + tx_sign(wallet, unsigned_tx, tx_idx, aux_data, signer); + + auto & cdata = signer->tdata(); + auto aux_info_cur = signer->store_tx_aux_info(); + aux_data.tx_device_aux.emplace_back(aux_info_cur); + + // Pending tx reconstruction + signed_tx.ptx.emplace_back(); + auto & cpend = signed_tx.ptx.back(); + cpend.tx = cdata.tx; + cpend.dust = 0; + cpend.fee = 0; + cpend.dust_added_to_fee = false; + cpend.change_dts = cdata.tx_data.change_dts; + cpend.selected_transfers = cdata.tx_data.selected_transfers; + cpend.key_images = ""; + cpend.dests = cdata.tx_data.dests; + cpend.construction_data = cdata.tx_data; + + // Transaction check + cryptonote::blobdata tx_blob; + cryptonote::transaction tx_deserialized; + bool r = cryptonote::t_serializable_object_to_blob(cpend.tx, tx_blob); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction serialization failed"); + r = cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx_deserialized); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction deserialization failed"); + + std::string key_images; + bool all_are_txin_to_key = std::all_of(cdata.tx.vin.begin(), cdata.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool + { + CHECKED_GET_SPECIFIC_VARIANT(s_e, const cryptonote::txin_to_key, in, false); + key_images += boost::to_string(in.k_image) + " "; + return true; + }); + if(!all_are_txin_to_key) { + throw std::invalid_argument("Not all are txin_to_key"); + } + cpend.key_images = key_images; + + // KI sync + size_t num_sources = cdata.tx_data.sources.size(); + CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.source_permutation.size(), "Invalid permutation size"); + CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.tx.vin.size(), "Invalid tx.vin size"); + for(size_t src_idx = 0; src_idx < num_sources; ++src_idx){ + size_t idx_mapped = cdata.source_permutation[src_idx]; + CHECK_AND_ASSERT_THROW_MES(idx_mapped < cdata.tx_data.selected_transfers.size(), "Invalid idx_mapped"); + CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped"); + + size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped]; + auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); + + CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index"); + signed_tx.key_images[idx_map_src] = vini.k_image; + } + } + } + + void device_trezor::tx_sign(wallet_shim * wallet, + const tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer) + { + AUTO_LOCK_CMD(); + require_connected(); + test_ping(); + + CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index"); + signer = std::make_shared<protocol::tx::Signer>(wallet, &unsigned_tx, idx, &aux_data); + const tools::wallet2::tx_construction_data & cur_tx = unsigned_tx.txes[idx]; + unsigned long num_sources = cur_tx.sources.size(); + unsigned long num_outputs = cur_tx.splitted_dsts.size(); + + // Step: Init + auto init_msg = signer->step_init(); + this->set_msg_addr(init_msg.get()); + + auto response = this->client_exchange<messages::monero::MoneroTransactionInitAck>(init_msg); + signer->step_init_ack(response); + + // Step: Set transaction inputs + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionSetInputAck>(src); + signer->step_set_input_ack(ack); + } + + // Step: sort + auto perm_req = signer->step_permutation(); + if (perm_req){ + auto perm_ack = this->client_exchange<messages::monero::MoneroTransactionInputsPermutationAck>(perm_req); + signer->step_permutation_ack(perm_ack); + } + + // Step: input_vini + if (!signer->in_memory()){ + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_vini_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); + signer->step_set_vini_input_ack(ack); + } + } + + // Step: all inputs set + auto all_inputs_set = signer->step_all_inputs_set(); + auto ack_all_inputs = this->client_exchange<messages::monero::MoneroTransactionAllInputsSetAck>(all_inputs_set); + signer->step_all_inputs_set_ack(ack_all_inputs); + + // Step: outputs + for(size_t cur_dst = 0; cur_dst < num_outputs; ++cur_dst){ + auto src = signer->step_set_output(cur_dst); + auto ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(src); + signer->step_set_output_ack(ack); + } + + // Step: all outs set + auto all_out_set = signer->step_all_outs_set(); + auto ack_all_out_set = this->client_exchange<messages::monero::MoneroTransactionAllOutSetAck>(all_out_set); + signer->step_all_outs_set_ack(ack_all_out_set, *this); + + // Step: sign each input + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_sign_input(cur_src); + auto ack_sign = this->client_exchange<messages::monero::MoneroTransactionSignInputAck>(src); + signer->step_sign_input_ack(ack_sign); + } + + // Step: final + auto final_msg = signer->step_final(); + auto ack_final = this->client_exchange<messages::monero::MoneroTransactionFinalAck>(final_msg); + signer->step_final_ack(ack_final); + } + +#else //WITH_DEVICE_TREZOR + + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry) { + } + + void register_all() { + } + +#endif //WITH_DEVICE_TREZOR +}} diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp new file mode 100644 index 000000000..765c9b82c --- /dev/null +++ b/src/device_trezor/device_trezor.hpp @@ -0,0 +1,132 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_TREZOR_H +#define MONERO_DEVICE_TREZOR_H + + +#include <cstddef> +#include <string> +#include "device/device.hpp" +#include "device/device_default.hpp" +#include "device/device_cold.hpp" +#include <boost/scope_exit.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> +#include "cryptonote_config.h" +#include "trezor.hpp" +#include "device_trezor_base.hpp" + +namespace hw { +namespace trezor { + + void register_all(); + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry); + +#if WITH_DEVICE_TREZOR + class device_trezor; + + /** + * Main device + */ + class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold { + protected: + // To speed up blockchain parsing the view key maybe handle here. + crypto::secret_key viewkey; + bool has_view_key; + + public: + device_trezor(); + virtual ~device_trezor() override; + + device_trezor(const device_trezor &device) = delete ; + device_trezor& operator=(const device_trezor &device) = delete; + + explicit operator bool() const override {return true;} + + device_protocol_t device_protocol() const override { return PROTOCOL_COLD; }; + + bool has_ki_cold_sync() const override { return true; } + bool has_tx_cold_sign() const override { return true; } + void set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; } + + /* ======================================================================= */ + /* WALLET & ADDRESS */ + /* ======================================================================= */ + bool get_public_address(cryptonote::account_public_address &pubkey) override; + bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + /** + * Get address. Throws. + */ + std::shared_ptr<messages::monero::MoneroAddress> get_address( + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none); + + /** + * Get watch key from device. Throws. + */ + std::shared_ptr<messages::monero::MoneroWatchKey> get_view_key( + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none); + + /** + * Key image sync with the Trezor. + */ + void ki_sync(wallet_shim * wallet, + const std::vector<::tools::wallet2::transfer_details> & transfers, + hw::device_cold::exported_key_image & ski) override; + + /** + * Signs particular transaction idx in the unsigned set, keeps state in the signer + */ + void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer); + + /** + * Signs unsigned transaction with the Trezor. + */ + void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + ::tools::wallet2::signed_tx_set & signed_tx, + hw::tx_aux_data & aux_data) override; + }; + +#endif + +} +} +#endif //MONERO_DEVICE_TREZOR_H diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp new file mode 100644 index 000000000..3a98bba5a --- /dev/null +++ b/src/device_trezor/device_trezor_base.cpp @@ -0,0 +1,301 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "device_trezor_base.hpp" + +namespace hw { +namespace trezor { + +#if WITH_DEVICE_TREZOR + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" + + std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_button_request(const messages::common::ButtonRequest * msg){ + MDEBUG("on_button_request"); + device.on_button_request(); + return std::make_shared<messages::common::ButtonAck>(); + } + + std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_pin_matrix_request(const messages::common::PinMatrixRequest * msg){ + MDEBUG("on_pin_request"); + epee::wipeable_string pin; + device.on_pin_request(pin); + auto resp = std::make_shared<messages::common::PinMatrixAck>(); + resp->set_pin(pin.data(), pin.size()); + return resp; + } + + std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_passphrase_request(const messages::common::PassphraseRequest * msg){ + MDEBUG("on_passhprase_request"); + epee::wipeable_string passphrase; + device.on_passphrase_request(msg->on_device(), passphrase); + auto resp = std::make_shared<messages::common::PassphraseAck>(); + if (!msg->on_device()){ + resp->set_passphrase(passphrase.data(), passphrase.size()); + } + return resp; + } + + std::shared_ptr<google::protobuf::Message> trezor_protocol_callback::on_passphrase_state_request(const messages::common::PassphraseStateRequest * msg){ + MDEBUG("on_passhprase_state_request"); + device.on_passphrase_state_request(msg->state()); + return std::make_shared<messages::common::PassphraseStateAck>(); + } + + const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080, 0x80000000}; + + device_trezor_base::device_trezor_base() { + + } + + device_trezor_base::~device_trezor_base() { + try { + disconnect(); + release(); + } catch(std::exception const& e){ + MERROR("Could not disconnect and release: " << e.what()); + } + } + + /* ======================================================================= */ + /* SETUP/TEARDOWN */ + /* ======================================================================= */ + + bool device_trezor_base::reset() { + return false; + } + + bool device_trezor_base::set_name(const std::string & name) { + this->full_name = name; + this->name = ""; + + auto delim = name.find(':'); + if (delim != std::string::npos && delim + 1 < name.length()) { + this->name = name.substr(delim + 1); + } + + return true; + } + + const std::string device_trezor_base::get_name() const { + if (this->full_name.empty()) { + return std::string("<disconnected:").append(this->name).append(">"); + } + return this->full_name; + } + + bool device_trezor_base::init() { + if (!release()){ + MERROR("Release failed"); + return false; + } + + if (!m_protocol_callback){ + m_protocol_callback = std::make_shared<trezor_protocol_callback>(*this); + } + return true; + } + + bool device_trezor_base::release() { + try { + disconnect(); + return true; + + } catch(std::exception const& e){ + MERROR("Release exception: " << e.what()); + return false; + } + } + + bool device_trezor_base::connect() { + disconnect(); + + // Enumerate all available devices + try { + hw::trezor::t_transport_vect trans; + + MDEBUG("Enumerating Trezor devices..."); + enumerate(trans); + + MDEBUG("Enumeration yielded " << trans.size() << " devices"); + for (auto &cur : trans) { + MDEBUG(" device: " << *(cur.get())); + std::string cur_path = cur->get_path(); + if (boost::starts_with(cur_path, this->name)) { + MDEBUG("Device Match: " << cur_path); + m_transport = cur; + break; + } + } + + if (!m_transport) { + MERROR("No matching Trezor device found. Device specifier: \"" + this->name + "\""); + return false; + } + + m_transport->open(); + return true; + + } catch(std::exception const& e){ + MERROR("Open exception: " << e.what()); + return false; + } + } + + bool device_trezor_base::disconnect() { + if (m_transport){ + try { + m_transport->close(); + m_transport = nullptr; + + } catch(std::exception const& e){ + MERROR("Disconnect exception: " << e.what()); + m_transport = nullptr; + return false; + } + } + return true; + } + + /* ======================================================================= */ + /* LOCKER */ + /* ======================================================================= */ + + //lock the device for a long sequence + void device_trezor_base::lock() { + MTRACE("Ask for LOCKING for device " << this->name << " in thread "); + device_locker.lock(); + MTRACE("Device " << this->name << " LOCKed"); + } + + //lock the device for a long sequence + bool device_trezor_base::try_lock() { + MTRACE("Ask for LOCKING(try) for device " << this->name << " in thread "); + bool r = device_locker.try_lock(); + if (r) { + MTRACE("Device " << this->name << " LOCKed(try)"); + } else { + MDEBUG("Device " << this->name << " not LOCKed(try)"); + } + return r; + } + + //unlock the device + void device_trezor_base::unlock() { + MTRACE("Ask for UNLOCKING for device " << this->name << " in thread "); + device_locker.unlock(); + MTRACE("Device " << this->name << " UNLOCKed"); + } + + /* ======================================================================= */ + /* Helpers */ + /* ======================================================================= */ + + void device_trezor_base::require_connected(){ + if (!m_transport){ + throw exc::NotConnectedException(); + } + } + + void device_trezor_base::call_ping_unsafe(){ + auto pingMsg = std::make_shared<messages::management::Ping>(); + pingMsg->set_message("PING"); + + auto success = this->client_exchange<messages::common::Success>(pingMsg); // messages::MessageType_Success + MDEBUG("Ping response " << success->message()); + (void)success; + } + + void device_trezor_base::test_ping(){ + require_connected(); + + try { + this->call_ping_unsafe(); + + } catch(exc::TrezorException const& e){ + MINFO("Trezor does not respond: " << e.what()); + throw exc::DeviceNotResponsiveException(std::string("Trezor not responding: ") + e.what()); + } + } + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + bool device_trezor_base::ping() { + AUTO_LOCK_CMD(); + if (!m_transport){ + MINFO("Ping failed, device not connected"); + return false; + } + + try { + this->call_ping_unsafe(); + return true; + + } catch(std::exception const& e) { + MERROR("Ping failed, exception thrown " << e.what()); + } catch(...){ + MERROR("Ping failed, general exception thrown" << boost::current_exception_diagnostic_information()); + } + + return false; + } + + void device_trezor_base::on_button_request() + { + if (m_callback){ + m_callback->on_button_request(); + } + } + + void device_trezor_base::on_pin_request(epee::wipeable_string & pin) + { + if (m_callback){ + m_callback->on_pin_request(pin); + } + } + + void device_trezor_base::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) + { + if (m_callback){ + m_callback->on_passphrase_request(on_device, passphrase); + } + } + + void device_trezor_base::on_passphrase_state_request(const std::string & state) + { + if (m_callback){ + m_callback->on_passphrase_state_request(state); + } + } + +#endif //WITH_DEVICE_TREZOR +}} diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp new file mode 100644 index 000000000..644a49332 --- /dev/null +++ b/src/device_trezor/device_trezor_base.hpp @@ -0,0 +1,301 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_TREZOR_BASE_H +#define MONERO_DEVICE_TREZOR_BASE_H + + +#include <cstddef> +#include <string> +#include "device/device.hpp" +#include "device/device_default.hpp" +#include "device/device_cold.hpp" +#include <boost/scope_exit.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> +#include "cryptonote_config.h" +#include "trezor.hpp" + +//automatic lock one more level on device ensuring the current thread is allowed to use it +#define AUTO_LOCK_CMD() \ + /* lock both mutexes without deadlock*/ \ + boost::lock(device_locker, command_locker); \ + /* make sure both already-locked mutexes are unlocked at the end of scope */ \ + boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \ + boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock) + + +namespace hw { +namespace trezor { + +#if WITH_DEVICE_TREZOR + class device_trezor_base; + + /** + * Trezor device callbacks + */ + class trezor_callback { + public: + virtual void on_button_request() {}; + virtual void on_pin_request(epee::wipeable_string & pin) {}; + virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {}; + virtual void on_passphrase_state_request(const std::string & state) {}; + }; + + /** + * Default Trezor protocol client callback + */ + class trezor_protocol_callback { + protected: + device_trezor_base & device; + + public: + explicit trezor_protocol_callback(device_trezor_base & device): device(device) {} + + std::shared_ptr<google::protobuf::Message> on_button_request(const messages::common::ButtonRequest * msg); + std::shared_ptr<google::protobuf::Message> on_pin_matrix_request(const messages::common::PinMatrixRequest * msg); + std::shared_ptr<google::protobuf::Message> on_passphrase_request(const messages::common::PassphraseRequest * msg); + std::shared_ptr<google::protobuf::Message> on_passphrase_state_request(const messages::common::PassphraseStateRequest * msg); + + std::shared_ptr<google::protobuf::Message> on_message(const google::protobuf::Message * msg, messages::MessageType message_type){ + MDEBUG("on_general_message"); + return on_message_dispatch(msg, message_type); + } + + std::shared_ptr<google::protobuf::Message> on_message_dispatch(const google::protobuf::Message * msg, messages::MessageType message_type){ + if (message_type == messages::MessageType_ButtonRequest){ + return on_button_request(dynamic_cast<const messages::common::ButtonRequest*>(msg)); + } else if (message_type == messages::MessageType_PassphraseRequest) { + return on_passphrase_request(dynamic_cast<const messages::common::PassphraseRequest*>(msg)); + } else if (message_type == messages::MessageType_PassphraseStateRequest) { + return on_passphrase_state_request(dynamic_cast<const messages::common::PassphraseStateRequest*>(msg)); + } else if (message_type == messages::MessageType_PinMatrixRequest) { + return on_pin_matrix_request(dynamic_cast<const messages::common::PinMatrixRequest*>(msg)); + } else { + return nullptr; + } + } + }; + + /** + * TREZOR device template with basic functions + */ + class device_trezor_base : public hw::core::device_default { + protected: + + // Locker for concurrent access + mutable boost::recursive_mutex device_locker; + mutable boost::mutex command_locker; + + std::shared_ptr<Transport> m_transport; + std::shared_ptr<trezor_protocol_callback> m_protocol_callback; + std::shared_ptr<trezor_callback> m_callback; + + std::string full_name; + + cryptonote::network_type network_type; + + // + // Internal methods + // + + void require_connected(); + void call_ping_unsafe(); + void test_ping(); + + /** + * Client communication wrapper, handles specific Trezor protocol. + * + * @throws UnexpectedMessageException if the response message type is different than expected. + * Exception contains message type and the message itself. + */ + template<class t_message> + std::shared_ptr<t_message> + client_exchange(const std::shared_ptr<const google::protobuf::Message> &req, + const boost::optional<messages::MessageType> & resp_type = boost::none, + const boost::optional<std::vector<messages::MessageType>> & resp_types = boost::none, + const boost::optional<messages::MessageType*> & resp_type_ptr = boost::none, + bool open_session = false, + unsigned depth=0) + { + // Require strictly protocol buffers response in the template. + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + const bool accepting_base = boost::is_same<google::protobuf::Message, t_message>::value; + if (resp_types && !accepting_base){ + throw std::invalid_argument("Cannot specify list of accepted types and not using generic response"); + } + + // Open session if required + if (open_session && depth == 0){ + try { + m_transport->open(); + } catch (const std::exception& e) { + std::throw_with_nested(exc::SessionException("Could not open session")); + } + } + + // Scoped session closer + BOOST_SCOPE_EXIT_ALL(&, this) { + if (open_session && depth == 0){ + this->getTransport()->close(); + } + }; + + // Write the request + CHECK_AND_ASSERT_THROW_MES(req, "Request is null"); + this->getTransport()->write(*req); + + // Read the response + std::shared_ptr<google::protobuf::Message> msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + + // We may have several roundtrips with the handler + this->getTransport()->read(msg_resp, &msg_resp_type); + if (resp_type_ptr){ + *(resp_type_ptr.get()) = msg_resp_type; + } + + // Determine type of expected message response + messages::MessageType required_type = accepting_base ? messages::MessageType_Success : + (resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>()); + + if (msg_resp_type == messages::MessageType_Failure) { + throw_failure_exception(dynamic_cast<messages::common::Failure *>(msg_resp.get())); + + } else if (!accepting_base && msg_resp_type == required_type) { + return message_ptr_retype<t_message>(msg_resp); + + } else { + auto resp = this->getProtocolCallback()->on_message(msg_resp.get(), msg_resp_type); + if (resp) { + return this->client_exchange<t_message>(resp, boost::none, resp_types, resp_type_ptr, false, depth + 1); + + } else if (accepting_base && (!resp_types || + std::find(resp_types.get().begin(), resp_types.get().end(), msg_resp_type) != resp_types.get().end())) { + return message_ptr_retype<t_message>(msg_resp); + + } else { + throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); + } + } + } + + /** + * Utility method to set address_n and network type to the message requets. + */ + template<class t_message> + void set_msg_addr(t_message * msg, + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none) + { + CHECK_AND_ASSERT_THROW_MES(msg, "Message is null"); + msg->clear_address_n(); + if (path){ + for(auto x : path.get()){ + msg->add_address_n(x); + } + } else { + for (unsigned int i : DEFAULT_BIP44_PATH) { + msg->add_address_n(i); + } + } + + if (network_type){ + msg->set_network_type(static_cast<uint32_t>(network_type.get())); + } else { + msg->set_network_type(static_cast<uint32_t>(this->network_type)); + } + } + + public: + device_trezor_base(); + ~device_trezor_base() override; + + device_trezor_base(const device_trezor_base &device) = delete ; + device_trezor_base& operator=(const device_trezor_base &device) = delete; + + explicit operator bool() const override {return true;} + device_type get_type() const override {return device_type::TREZOR;}; + + bool reset(); + + // Default derivation path for Monero + static const uint32_t DEFAULT_BIP44_PATH[3]; + + std::shared_ptr<Transport> getTransport(){ + return m_transport; + } + + std::shared_ptr<trezor_protocol_callback> getProtocolCallback(){ + return m_protocol_callback; + } + + std::shared_ptr<trezor_callback> getCallback(){ + return m_callback; + } + + /* ======================================================================= */ + /* SETUP/TEARDOWN */ + /* ======================================================================= */ + bool set_name(const std::string &name) override; + + const std::string get_name() const override; + bool init() override; + bool release() override; + bool connect() override; + bool disconnect() override; + + /* ======================================================================= */ + /* LOCKER */ + /* ======================================================================= */ + void lock() override; + void unlock() override; + bool try_lock() override; + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + /** + * Device ping, no-throw + */ + bool ping(); + + // Protocol callbacks + void on_button_request(); + void on_pin_request(epee::wipeable_string & pin); + void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + void on_passphrase_state_request(const std::string & state); + }; + +#endif + +} +} +#endif //MONERO_DEVICE_TREZOR_BASE_H diff --git a/src/device_trezor/trezor.hpp b/src/device_trezor/trezor.hpp new file mode 100644 index 000000000..8abdd2c18 --- /dev/null +++ b/src/device_trezor/trezor.hpp @@ -0,0 +1,44 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_TREZOR_HPP +#define MONERO_TREZOR_HPP + +#include "trezor/trezor_defs.hpp" + +#ifdef HAVE_PROTOBUF +#include "trezor/transport.hpp" +#include "trezor/messages/messages.pb.h" +#include "trezor/messages/messages-common.pb.h" +#include "trezor/messages/messages-management.pb.h" +#include "trezor/messages/messages-monero.pb.h" +#include "trezor/protocol.hpp" +#endif + +#endif //MONERO_TREZOR_HPP diff --git a/src/device_trezor/trezor/exceptions.hpp b/src/device_trezor/trezor/exceptions.hpp new file mode 100644 index 000000000..197dc43a4 --- /dev/null +++ b/src/device_trezor/trezor/exceptions.hpp @@ -0,0 +1,193 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_EXCEPTIONS_H +#define MONERO_EXCEPTIONS_H + +#include <exception> +#include <string> +#include <boost/optional.hpp> + +namespace hw { +namespace trezor { +namespace exc { + + class SecurityException : public std::exception { + protected: + boost::optional<std::string> reason; + + public: + SecurityException(): reason("General Security exception"){} + explicit SecurityException(std::string what): reason(what){} + + virtual const char* what() const throw() { + return reason.get().c_str(); + } + }; + + class Poly1305TagInvalid: public SecurityException { + public: + using SecurityException::SecurityException; + Poly1305TagInvalid(): SecurityException("Poly1305 authentication tag invalid"){} + }; + + class TrezorException : public std::exception { + protected: + boost::optional<std::string> reason; + + public: + TrezorException(): reason("General Trezor exception"){} + explicit TrezorException(std::string what): reason(what){} + + virtual const char* what() const throw() { + return reason.get().c_str(); + } + }; + + class CommunicationException: public TrezorException { + public: + using TrezorException::TrezorException; + CommunicationException(): TrezorException("Trezor communication error"){} + }; + + class EncodingException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + EncodingException(): CommunicationException("Trezor message encoding error"){} + }; + + class NotConnectedException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + NotConnectedException(): CommunicationException("Trezor not connected"){} + }; + + class DeviceNotResponsiveException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + DeviceNotResponsiveException(): CommunicationException("Trezor does not respond to ping"){} + }; + + class DeviceAcquireException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + DeviceAcquireException(): CommunicationException("Trezor could not be acquired"){} + }; + + class SessionException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + SessionException(): CommunicationException("Trezor session expired"){} + }; + + class TimeoutException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + TimeoutException(): CommunicationException("Trezor communication timeout"){} + }; + + class ProtocolException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + ProtocolException(): CommunicationException("Trezor protocol error"){} + }; + + // Communication protocol namespace + // Separated to distinguish between client and Trezor side exceptions. +namespace proto { + + class SecurityException : public ProtocolException { + public: + using ProtocolException::ProtocolException; + SecurityException(): ProtocolException("Security assertion violated in the protocol"){} + }; + + class FailureException : public ProtocolException { + private: + boost::optional<uint32_t> code; + boost::optional<std::string> message; + public: + using ProtocolException::ProtocolException; + FailureException(): ProtocolException("Trezor returned failure"){} + FailureException(boost::optional<uint32_t> code, + boost::optional<std::string> message) + : code(code), message(message) { + reason = "Trezor returned failure: code=" + + (code ? std::to_string(code.get()) : "") + + ", message=" + (message ? message.get() : ""); + }; + }; + + class UnexpectedMessageException : public FailureException { + public: + using FailureException::FailureException; + UnexpectedMessageException(): FailureException("Trezor claims unexpected message received"){} + }; + + class CancelledException : public FailureException { + public: + using FailureException::FailureException; + CancelledException(): FailureException("Trezor returned: cancelled operation"){} + }; + + class PinExpectedException : public FailureException { + public: + using FailureException::FailureException; + PinExpectedException(): FailureException("Trezor claims PIN is expected"){} + }; + + class InvalidPinException : public FailureException { + public: + using FailureException::FailureException; + InvalidPinException(): FailureException("Trezor claims PIN is invalid"){} + }; + + class NotEnoughFundsException : public FailureException { + public: + using FailureException::FailureException; + NotEnoughFundsException(): FailureException("Trezor claims not enough funds"){} + }; + + class NotInitializedException : public FailureException { + public: + using FailureException::FailureException; + NotInitializedException(): FailureException("Trezor claims not initialized"){} + }; + + class FirmwareErrorException : public FailureException { + public: + using FailureException::FailureException; + FirmwareErrorException(): FailureException("Trezor returned firmware error"){} + }; + +} +} +} +} +#endif //MONERO_EXCEPTIONS_H diff --git a/src/device_trezor/trezor/messages/.gitignore b/src/device_trezor/trezor/messages/.gitignore new file mode 100644 index 000000000..32f7a77e5 --- /dev/null +++ b/src/device_trezor/trezor/messages/.gitignore @@ -0,0 +1,2 @@ +# protobuf generated code +* diff --git a/src/device_trezor/trezor/messages_map.cpp b/src/device_trezor/trezor/messages_map.cpp new file mode 100644 index 000000000..b0d1aa254 --- /dev/null +++ b/src/device_trezor/trezor/messages_map.cpp @@ -0,0 +1,125 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "messages_map.hpp" +#include "messages/messages.pb.h" +#include "messages/messages-common.pb.h" +#include "messages/messages-management.pb.h" +#include "messages/messages-monero.pb.h" + +using namespace std; +using namespace hw::trezor; + +namespace hw{ +namespace trezor +{ + + const char * TYPE_PREFIX = "MessageType_"; + const char * PACKAGES[] = { + "hw.trezor.messages.", + "hw.trezor.messages.common.", + "hw.trezor.messages.management.", + "hw.trezor.messages.monero." + }; + + google::protobuf::Message * MessageMapper::get_message(int wire_number) { + return MessageMapper::get_message(static_cast<messages::MessageType>(wire_number)); + } + + google::protobuf::Message * MessageMapper::get_message(messages::MessageType wire_number) { + const string &messageTypeName = hw::trezor::messages::MessageType_Name(wire_number); + if (messageTypeName.empty()) { + throw exc::EncodingException(std::string("Message descriptor not found: ") + std::to_string(wire_number)); + } + + string messageName = messageTypeName.substr(strlen(TYPE_PREFIX)); + return MessageMapper::get_message(messageName); + } + + google::protobuf::Message * MessageMapper::get_message(const std::string & msg_name) { + // Each package instantiation so lookup works + hw::trezor::messages::common::Success::default_instance(); + hw::trezor::messages::management::Cancel::default_instance(); + hw::trezor::messages::monero::MoneroGetAddress::default_instance(); + + google::protobuf::Descriptor const * desc = nullptr; + for(const string &text : PACKAGES){ + desc = google::protobuf::DescriptorPool::generated_pool() + ->FindMessageTypeByName(text + msg_name); + if (desc != nullptr){ + break; + } + } + + if (desc == nullptr){ + throw exc::EncodingException(std::string("Message not found: ") + msg_name); + } + + google::protobuf::Message* message = + google::protobuf::MessageFactory::generated_factory() + ->GetPrototype(desc)->New(); + + return message; + +// // CODEGEN way, fast +// switch(wire_number){ +// case 501: +// return new messages::monero::MoneroTransactionSignRequest(); +// default: +// throw std::runtime_error("not implemented"); +// } +// +// // CODEGEN message -> number: specification +// // messages::MessageType get_message_wire_number(const messages::monero::MoneroTransactionSignRequest * msg) { return 501; } +// // messages::MessageType get_message_wire_number(const messages::management::ping * msg) +// + } + + messages::MessageType MessageMapper::get_message_wire_number(const google::protobuf::Message * msg){ + return MessageMapper::get_message_wire_number(msg->GetDescriptor()->name()); + } + + messages::MessageType MessageMapper::get_message_wire_number(const google::protobuf::Message & msg){ + return MessageMapper::get_message_wire_number(msg.GetDescriptor()->name()); + } + + messages::MessageType MessageMapper::get_message_wire_number(const std::string & msg_name){ + string enumMessageName = std::string(TYPE_PREFIX) + msg_name; + + messages::MessageType res; + bool r = hw::trezor::messages::MessageType_Parse(enumMessageName, &res); + if (!r){ + throw exc::EncodingException(std::string("Message ") + msg_name + " not found"); + } + + return res; + } + +} +} diff --git a/src/device_trezor/trezor/messages_map.hpp b/src/device_trezor/trezor/messages_map.hpp new file mode 100644 index 000000000..f61338f09 --- /dev/null +++ b/src/device_trezor/trezor/messages_map.hpp @@ -0,0 +1,94 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_MESSAGES_MAP_H +#define MONERO_MESSAGES_MAP_H + +#include <string> +#include <type_traits> +#include <memory> +#include "exceptions.hpp" + +#include "trezor_defs.hpp" + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/generated_message_util.h> +#include <google/protobuf/repeated_field.h> +#include <google/protobuf/extension_set.h> +#include <google/protobuf/generated_enum_reflection.h> +#include "google/protobuf/descriptor.pb.h" + +#include "messages/messages.pb.h" + +namespace hw { +namespace trezor { + + class MessageMapper{ + public: + MessageMapper() { + + } + + static ::google::protobuf::Message * get_message(int wire_number); + static ::google::protobuf::Message * get_message(messages::MessageType); + static ::google::protobuf::Message * get_message(const std::string & msg_name); + static messages::MessageType get_message_wire_number(const google::protobuf::Message * msg); + static messages::MessageType get_message_wire_number(const google::protobuf::Message & msg); + static messages::MessageType get_message_wire_number(const std::string & msg_name); + + template<class t_message> + static messages::MessageType get_message_wire_number() { + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + return get_message_wire_number(t_message::default_instance().GetDescriptor()->name()); + } + }; + + template<class t_message> + std::shared_ptr<t_message> message_ptr_retype(std::shared_ptr<google::protobuf::Message> & in){ + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + if (!in){ + return nullptr; + } + + return std::dynamic_pointer_cast<t_message>(in); + } + + template<class t_message> + std::shared_ptr<t_message> message_ptr_retype_static(std::shared_ptr<google::protobuf::Message> & in){ + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + if (!in){ + return nullptr; + } + + return std::static_pointer_cast<t_message>(in); + } + +}} + +#endif //MONERO_MESSAGES_MAP_H diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp new file mode 100644 index 000000000..c4a92426c --- /dev/null +++ b/src/device_trezor/trezor/protocol.cpp @@ -0,0 +1,891 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "protocol.hpp" +#include <unordered_map> +#include <set> +#include <utility> +#include <boost/endian/conversion.hpp> +#include <common/apply_permutation.h> +#include <ringct/rctSigs.h> +#include <ringct/bulletproofs.h> +#include "cryptonote_config.h" +#include <sodium.h> +#include <sodium/crypto_verify_32.h> +#include <sodium/crypto_aead_chacha20poly1305.h> + +namespace hw{ +namespace trezor{ +namespace protocol{ + + std::string key_to_string(const ::crypto::ec_point & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::crypto::ec_scalar & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::crypto::hash & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::rct::key & key){ + return std::string(reinterpret_cast<const char*>(key.bytes), sizeof(key.bytes)); + } + + void string_to_key(::crypto::ec_scalar & key, const std::string & str){ + if (str.size() != sizeof(key.data)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.data)) + " B"); + } + memcpy(key.data, str.data(), sizeof(key.data)); + } + + void string_to_key(::crypto::ec_point & key, const std::string & str){ + if (str.size() != sizeof(key.data)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.data)) + " B"); + } + memcpy(key.data, str.data(), sizeof(key.data)); + } + + void string_to_key(::rct::key & key, const std::string & str){ + if (str.size() != sizeof(key.bytes)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.bytes)) + " B"); + } + memcpy(key.bytes, str.data(), sizeof(key.bytes)); + } + +namespace crypto { +namespace chacha { + + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext){ + if (length < 16){ + throw std::invalid_argument("Ciphertext length too small"); + } + + unsigned long long int cip_len = length; + auto r = crypto_aead_chacha20poly1305_ietf_decrypt( + reinterpret_cast<unsigned char *>(plaintext), &cip_len, nullptr, + static_cast<const unsigned char *>(ciphertext), length, nullptr, 0, iv, key); + + if (r != 0){ + throw exc::Poly1305TagInvalid(); + } + } + +} +} + + +// Cold Key image sync +namespace ki { + + bool key_image_data(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::vector<MoneroTransferDetails> & res) + { + for(auto & td : transfers){ + ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td); + const std::vector<::crypto::public_key> additional_tx_pub_keys = cryptonote::get_additional_tx_pub_keys_from_extra(td.m_tx); + + res.emplace_back(); + auto & cres = res.back(); + + cres.set_out_key(key_to_string(boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key)); + cres.set_tx_pub_key(key_to_string(tx_pub_key)); + cres.set_internal_output_index(td.m_internal_output_index); + for(auto & aux : additional_tx_pub_keys){ + cres.add_additional_tx_pub_keys(key_to_string(aux)); + } + } + + return true; + } + + std::string compute_hash(const MoneroTransferDetails & rr){ + KECCAK_CTX kck; + uint8_t md[32]; + + CHECK_AND_ASSERT_THROW_MES(rr.out_key().size() == 32, "Invalid out_key size"); + CHECK_AND_ASSERT_THROW_MES(rr.tx_pub_key().size() == 32, "Invalid tx_pub_key size"); + + keccak_init(&kck); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(rr.out_key().data()), 32); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(rr.tx_pub_key().data()), 32); + for (const auto &aux : rr.additional_tx_pub_keys()){ + CHECK_AND_ASSERT_THROW_MES(aux.size() == 32, "Invalid aux size"); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(aux.data()), 32); + } + + auto index_serialized = tools::get_varint_data(rr.internal_output_index()); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(index_serialized.data()), index_serialized.size()); + keccak_finish(&kck, md); + return std::string(reinterpret_cast<const char*>(md), sizeof(md)); + } + + void generate_commitment(std::vector<MoneroTransferDetails> & mtds, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req) + { + req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>(); + + KECCAK_CTX kck; + uint8_t final_hash[32]; + keccak_init(&kck); + + for(auto &cur : mtds){ + auto hash = compute_hash(cur); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(hash.data()), hash.size()); + } + keccak_finish(&kck, final_hash); + + req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>(); + req->set_hash(std::string(reinterpret_cast<const char*>(final_hash), 32)); + req->set_num(transfers.size()); + + std::unordered_map<uint32_t, std::set<uint32_t>> sub_indices; + for (auto &cur : transfers){ + auto search = sub_indices.emplace(cur.m_subaddr_index.major, std::set<uint32_t>()); + auto & st = search.first->second; + st.insert(cur.m_subaddr_index.minor); + } + + for (auto& x: sub_indices){ + auto subs = req->add_subs(); + subs->set_account(x.first); + for(auto minor : x.second){ + subs->add_minor_indices(minor); + } + } + } + +} + +// Cold transaction signing +namespace tx { + + void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src){ + dst->set_view_public_key(key_to_string(src->m_view_public_key)); + dst->set_spend_public_key(key_to_string(src->m_spend_public_key)); + } + + void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src){ + dst->set_amount(src->amount); + dst->set_is_subaddress(src->is_subaddress); + translate_address(dst->mutable_addr(), &(src->addr)); + } + + void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src){ + for(auto & cur : src->outputs){ + auto out = dst->add_outputs(); + out->set_idx(cur.first); + translate_rct_key(out->mutable_key(), &(cur.second)); + } + + dst->set_real_output(src->real_output); + dst->set_real_out_tx_key(key_to_string(src->real_out_tx_key)); + for(auto & cur : src->real_out_additional_tx_keys){ + dst->add_real_out_additional_tx_keys(key_to_string(cur)); + } + + dst->set_real_output_in_tx_index(src->real_output_in_tx_index); + dst->set_amount(src->amount); + dst->set_rct(src->rct); + dst->set_mask(key_to_string(src->mask)); + translate_klrki(dst->mutable_multisig_klrki(), &(src->multisig_kLRki)); + } + + void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src){ + dst->set_k(key_to_string(src->k)); + dst->set_l(key_to_string(src->L)); + dst->set_r(key_to_string(src->R)); + dst->set_ki(key_to_string(src->ki)); + } + + void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src){ + dst->set_dest(key_to_string(src->dest)); + dst->set_commitment(key_to_string(src->mask)); + } + + std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + return hash_addr(addr->spend_public_key(), addr->view_public_key(), amount, is_subaddr); + } + + std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + ::crypto::public_key spend{}, view{}; + if (spend_key.size() != 32 || view_key.size() != 32){ + throw std::invalid_argument("Public keys have invalid sizes"); + } + + memcpy(spend.data, spend_key.data(), 32); + memcpy(view.data, view_key.data(), 32); + return hash_addr(&spend, &view, amount, is_subaddr); + } + + std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + char buff[64+8+1]; + size_t offset = 0; + + memcpy(buff + offset, spend_key->data, 32); offset += 32; + memcpy(buff + offset, view_key->data, 32); offset += 32; + + if (amount){ + memcpy(buff + offset, (uint8_t*) &(amount.get()), sizeof(amount.get())); offset += sizeof(amount.get()); + } + + if (is_subaddr){ + buff[offset] = is_subaddr.get(); + offset += 1; + } + + return std::string(buff, offset); + } + + TData::TData() { + in_memory = false; + rsig_type = 0; + cur_input_idx = 0; + cur_output_idx = 0; + cur_batch_idx = 0; + cur_output_in_batch_idx = 0; + } + + Signer::Signer(wallet_shim *wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx, hw::tx_aux_data * aux_data) { + m_wallet2 = wallet2; + m_unsigned_tx = unsigned_tx; + m_aux_data = aux_data; + m_tx_idx = tx_idx; + m_ct.tx_data = cur_tx(); + m_multisig = false; + } + + void Signer::extract_payment_id(){ + const std::vector<uint8_t>& tx_extra = cur_tx().extra; + m_ct.tsx_data.set_payment_id(""); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + cryptonote::parse_tx_extra(tx_extra, tx_extra_fields); // ok if partially parsed + cryptonote::tx_extra_nonce extra_nonce; + + ::crypto::hash payment_id{}; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + ::crypto::hash8 payment_id8{}; + if(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + { + m_ct.tsx_data.set_payment_id(std::string(payment_id8.data, 8)); + } + else if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + m_ct.tsx_data.set_payment_id(std::string(payment_id.data, 32)); + } + } + } + + static unsigned get_rsig_type(bool use_bulletproof, size_t num_outputs){ + if (!use_bulletproof){ + return rct::RangeProofBorromean; + } else if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ + return rct::RangeProofMultiOutputBulletproof; + } else { + return rct::RangeProofPaddedBulletproof; + } + } + + static void generate_rsig_batch_sizes(std::vector<uint64_t> &batches, unsigned rsig_type, size_t num_outputs){ + size_t amount_batched = 0; + + while(amount_batched < num_outputs){ + if (rsig_type == rct::RangeProofBorromean || rsig_type == rct::RangeProofBulletproof) { + batches.push_back(1); + amount_batched += 1; + + } else if (rsig_type == rct::RangeProofPaddedBulletproof){ + if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ + throw std::invalid_argument("BP padded can support only BULLETPROOF_MAX_OUTPUTS statements"); + } + batches.push_back(num_outputs); + amount_batched += num_outputs; + + } else if (rsig_type == rct::RangeProofMultiOutputBulletproof){ + size_t batch_size = 1; + while (batch_size * 2 + amount_batched <= num_outputs && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS){ + batch_size *= 2; + } + batch_size = std::min(batch_size, num_outputs - amount_batched); + batches.push_back(batch_size); + amount_batched += batch_size; + + } else { + throw std::invalid_argument("Unknown rsig type"); + } + } + } + + void Signer::compute_integrated_indices(TsxData * tsx_data){ + if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){ + return; + } + + auto & chg = tsx_data->change_dts(); + std::string change_hash = hash_addr(&chg.addr(), chg.amount(), chg.is_subaddress()); + + std::vector<uint32_t> integrated_indices; + std::set<std::string> integrated_hashes; + for (auto & cur : m_aux_data->tx_recipients){ + if (!cur.has_payment_id){ + continue; + } + integrated_hashes.emplace(hash_addr(&cur.address.m_spend_public_key, &cur.address.m_view_public_key)); + } + + ssize_t idx = -1; + for (auto & cur : tsx_data->outputs()){ + idx += 1; + + std::string c_hash = hash_addr(&cur.addr(), cur.amount(), cur.is_subaddress()); + if (c_hash == change_hash || cur.is_subaddress()){ + continue; + } + + c_hash = hash_addr(&cur.addr()); + if (integrated_hashes.find(c_hash) != integrated_hashes.end()){ + integrated_indices.push_back((uint32_t)idx); + } + } + + if (!integrated_indices.empty()){ + assign_to_repeatable(tsx_data->mutable_integrated_indices(), integrated_indices.begin(), integrated_indices.end()); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionInitRequest> Signer::step_init(){ + // extract payment ID from construction data + auto & tsx_data = m_ct.tsx_data; + auto & tx = cur_tx(); + + m_ct.tx.version = 2; + m_ct.tx.unlock_time = tx.unlock_time; + + tsx_data.set_version(1); + tsx_data.set_unlock_time(tx.unlock_time); + tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size())); + tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1)); + tsx_data.set_account(tx.subaddr_account); + assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end()); + + // Rsig decision + auto rsig_data = tsx_data.mutable_rsig_data(); + m_ct.rsig_type = get_rsig_type(tx.use_bulletproofs, tx.splitted_dsts.size()); + rsig_data->set_rsig_type(m_ct.rsig_type); + + generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size()); + assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end()); + + translate_dst_entry(tsx_data.mutable_change_dts(), &(tx.change_dts)); + for(auto & cur : tx.splitted_dsts){ + auto dst = tsx_data.mutable_outputs()->Add(); + translate_dst_entry(dst, &cur); + } + + compute_integrated_indices(&tsx_data); + + int64_t fee = 0; + for(auto & cur_in : tx.sources){ + fee += cur_in.amount; + } + for(auto & cur_out : tx.splitted_dsts){ + fee -= cur_out.amount; + } + if (fee < 0){ + throw std::invalid_argument("Fee cannot be negative"); + } + + tsx_data.set_fee(static_cast<google::protobuf::uint64>(fee)); + this->extract_payment_id(); + + auto init_req = std::make_shared<messages::monero::MoneroTransactionInitRequest>(); + init_req->set_version(0); + init_req->mutable_tsx_data()->CopyFrom(tsx_data); + return init_req; + } + + void Signer::step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack){ + m_ct.in_memory = false; + if (ack->has_rsig_data()){ + m_ct.rsig_param = std::make_shared<MoneroRsigData>(ack->rsig_data()); + } + + assign_from_repeatable(&(m_ct.tx_out_entr_hmacs), ack->hmacs().begin(), ack->hmacs().end()); + } + + std::shared_ptr<messages::monero::MoneroTransactionSetInputRequest> Signer::step_set_input(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < cur_tx().sources.size(), "Invalid source index"); + m_ct.cur_input_idx = idx; + auto res = std::make_shared<messages::monero::MoneroTransactionSetInputRequest>(); + translate_src_entry(res->mutable_src_entr(), &(cur_tx().sources[idx])); + return res; + } + + void Signer::step_set_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetInputAck> ack){ + auto & vini_str = ack->vini(); + + cryptonote::txin_v vini; + if (!cn_deserialize(vini_str.data(), vini_str.size(), vini)){ + throw exc::ProtocolException("Cannot deserialize vin[i]"); + } + + m_ct.tx.vin.emplace_back(vini); + m_ct.tx_in_hmacs.push_back(ack->vini_hmac()); + m_ct.pseudo_outs.push_back(ack->pseudo_out()); + m_ct.pseudo_outs_hmac.push_back(ack->pseudo_out_hmac()); + m_ct.alphas.push_back(ack->pseudo_out_alpha()); + m_ct.spend_encs.push_back(ack->spend_key()); + } + + void Signer::sort_ki(){ + const size_t input_size = cur_tx().sources.size(); + + m_ct.source_permutation.clear(); + for (size_t n = 0; n < input_size; ++n){ + m_ct.source_permutation.push_back(n); + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx.vin.size() == input_size, "Invalid vector size"); + std::sort(m_ct.source_permutation.begin(), m_ct.source_permutation.end(), [&](const size_t i0, const size_t i1) { + const cryptonote::txin_to_key &tk0 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i0]); + const cryptonote::txin_to_key &tk1 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i1]); + return memcmp(&tk0.k_image, &tk1.k_image, sizeof(tk0.k_image)) > 0; + }); + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_in_hmacs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.pseudo_outs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.pseudo_outs_hmac.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.alphas.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.spend_encs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_data.sources.size() == input_size, "Invalid vector size"); + + tools::apply_permutation(m_ct.source_permutation, [&](size_t i0, size_t i1){ + std::swap(m_ct.tx.vin[i0], m_ct.tx.vin[i1]); + std::swap(m_ct.tx_in_hmacs[i0], m_ct.tx_in_hmacs[i1]); + std::swap(m_ct.pseudo_outs[i0], m_ct.pseudo_outs[i1]); + std::swap(m_ct.pseudo_outs_hmac[i0], m_ct.pseudo_outs_hmac[i1]); + std::swap(m_ct.alphas[i0], m_ct.alphas[i1]); + std::swap(m_ct.spend_encs[i0], m_ct.spend_encs[i1]); + std::swap(m_ct.tx_data.sources[i0], m_ct.tx_data.sources[i1]); + }); + } + + std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){ + sort_ki(); + + if (in_memory()){ + return nullptr; + } + + auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>(); + assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); + + return res; + } + + void Signer::step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack){ + if (in_memory()){ + return; + } + } + + std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> Signer::step_set_vini_input(size_t idx){ + if (in_memory()){ + return nullptr; + } + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); + + m_ct.cur_input_idx = idx; + auto tx = m_ct.tx_data; + auto res = std::make_shared<messages::monero::MoneroTransactionInputViniRequest>(); + auto & vini = m_ct.tx.vin[idx]; + translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx])); + res->set_vini(cryptonote::t_serializable_object_to_blob(vini)); + res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); + if (!in_memory()) { + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); + } + + return res; + } + + void Signer::step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack){ + if (in_memory()){ + return; + } + } + + std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> Signer::step_all_inputs_set(){ + return std::make_shared<messages::monero::MoneroTransactionAllInputsSetRequest>(); + } + + void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){ + if (is_offloading()){ + // If offloading, expect rsig configuration. + if (!ack->has_rsig_data()){ + throw exc::ProtocolException("Rsig offloading requires rsig param"); + } + + auto & rsig_data = ack->rsig_data(); + if (!rsig_data.has_mask()){ + throw exc::ProtocolException("Gamma masks not present in offloaded version"); + } + + auto & mask = rsig_data.mask(); + if (mask.size() != 32 * num_outputs()){ + throw exc::ProtocolException("Invalid number of gamma masks"); + } + + m_ct.rsig_gamma.reserve(num_outputs()); + for(size_t c=0; c < num_outputs(); ++c){ + rct::key cmask{}; + memcpy(cmask.bytes, mask.data() + c * 32, 32); + m_ct.rsig_gamma.emplace_back(cmask); + } + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.splitted_dsts.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_out_entr_hmacs.size(), "Invalid transaction index"); + + m_ct.cur_output_idx = idx; + m_ct.cur_output_in_batch_idx += 1; // assumes sequential call to step_set_output() + + auto res = std::make_shared<messages::monero::MoneroTransactionSetOutputRequest>(); + auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; + translate_dst_entry(res->mutable_dst_entr(), &cur_dst); + res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); + + // Range sig offloading to the host + if (!is_offloading()) { + return res; + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + if (m_ct.grouping_vct[m_ct.cur_batch_idx] > m_ct.cur_output_in_batch_idx) { + return res; + } + + auto rsig_data = res->mutable_rsig_data(); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + + if (!is_req_bulletproof()){ + if (batch_size > 1){ + throw std::invalid_argument("Borromean cannot batch outputs"); + } + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.rsig_gamma.size(), "Invalid gamma index"); + rct::key C{}, mask = m_ct.rsig_gamma[idx]; + auto genRsig = rct::proveRange(C, mask, cur_dst.amount); // TODO: rsig with given mask + auto serRsig = cn_serialize(genRsig); + m_ct.tx_out_rsigs.emplace_back(genRsig); + rsig_data->set_rsig(serRsig); + + } else { + std::vector<uint64_t> amounts; + rct::keyV masks; + CHECK_AND_ASSERT_THROW_MES(idx + 1 >= batch_size, "Invalid index for batching"); + + for(size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); + + amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); + masks.push_back(m_ct.rsig_gamma[bidx]); + } + + auto bp = bulletproof_PROVE(amounts, masks); + auto serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + rsig_data->set_rsig(serRsig); + } + + return res; + } + + void Signer::step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ + cryptonote::tx_out tx_out; + rct::rangeSig range_sig{}; + rct::Bulletproof bproof{}; + rct::ctkey out_pk{}; + rct::ecdhTuple ecdh{}; + + bool has_rsig = false; + std::string rsig_buff; + + if (ack->has_rsig_data()){ + auto & rsig_data = ack->rsig_data(); + + if (rsig_data.has_rsig() && !rsig_data.rsig().empty()){ + has_rsig = true; + rsig_buff = rsig_data.rsig(); + + } else if (rsig_data.rsig_parts_size() > 0){ + has_rsig = true; + for (const auto &it : rsig_data.rsig_parts()) { + rsig_buff += it; + } + } + } + + if (!cn_deserialize(ack->tx_out(), tx_out)){ + throw exc::ProtocolException("Cannot deserialize vout[i]"); + } + + if (!cn_deserialize(ack->out_pk(), out_pk)){ + throw exc::ProtocolException("Cannot deserialize out_pk"); + } + + if (!cn_deserialize(ack->ecdh_info(), ecdh)){ + throw exc::ProtocolException("Cannot deserialize ecdhtuple"); + } + + if (has_rsig && !is_req_bulletproof() && !cn_deserialize(rsig_buff, range_sig)){ + throw exc::ProtocolException("Cannot deserialize rangesig"); + } + + if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){ + throw exc::ProtocolException("Cannot deserialize bulletproof rangesig"); + } + + m_ct.tx.vout.emplace_back(tx_out); + m_ct.tx_out_hmacs.push_back(ack->vouti_hmac()); + m_ct.tx_out_pk.emplace_back(out_pk); + m_ct.tx_out_ecdh.emplace_back(ecdh); + + if (!has_rsig){ + return; + } + + if (is_req_bulletproof()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + for (size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + + rct::key commitment = m_ct.tx_out_pk[bidx].mask; + commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); + bproof.V.push_back(commitment); + } + + m_ct.tx_out_rsigs.emplace_back(bproof); + if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + + } else { + m_ct.tx_out_rsigs.emplace_back(range_sig); + + if (!rct::verRange(out_pk.mask, boost::get<rct::rangeSig>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + } + + m_ct.cur_batch_idx += 1; + m_ct.cur_output_in_batch_idx = 0; + } + + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> Signer::step_all_outs_set(){ + return std::make_shared<messages::monero::MoneroTransactionAllOutSetRequest>(); + } + + void Signer::step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev){ + m_ct.rv = std::make_shared<rct::rctSig>(); + m_ct.rv->txnFee = ack->rv().txn_fee(); + m_ct.rv->type = static_cast<uint8_t>(ack->rv().rv_type()); + string_to_key(m_ct.rv->message, ack->rv().message()); + + // Extra copy + m_ct.tx.extra.clear(); + auto extra = ack->extra(); + auto extra_data = extra.data(); + m_ct.tx.extra.reserve(extra.size()); + for(size_t i = 0; i < extra.size(); ++i){ + m_ct.tx.extra.push_back(static_cast<uint8_t>(extra_data[i])); + } + + ::crypto::hash tx_prefix_hash{}; + cryptonote::get_transaction_prefix_hash(m_ct.tx, tx_prefix_hash); + m_ct.tx_prefix_hash = key_to_string(tx_prefix_hash); + if (crypto_verify_32(reinterpret_cast<const unsigned char *>(tx_prefix_hash.data), + reinterpret_cast<const unsigned char *>(ack->tx_prefix_hash().data()))){ + throw exc::proto::SecurityException("Transaction prefix has does not match to the computed value"); + } + + // RctSig + auto num_sources = m_ct.tx_data.sources.size(); + if (is_simple() || is_req_bulletproof()){ + auto dst = &m_ct.rv->pseudoOuts; + if (is_bulletproof()){ + dst = &m_ct.rv->p.pseudoOuts; + } + + dst->clear(); + for (const auto &pseudo_out : m_ct.pseudo_outs) { + dst->emplace_back(); + string_to_key(dst->back(), pseudo_out); + } + + m_ct.rv->mixRing.resize(num_sources); + } else { + m_ct.rv->mixRing.resize(m_ct.tsx_data.mixin()); + m_ct.rv->mixRing[0].resize(num_sources); + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_out_pk.size() == m_ct.tx_out_ecdh.size(), "Invalid vector sizes"); + for(size_t i = 0; i < m_ct.tx_out_ecdh.size(); ++i){ + m_ct.rv->outPk.push_back(m_ct.tx_out_pk[i]); + m_ct.rv->ecdhInfo.push_back(m_ct.tx_out_ecdh[i]); + } + + for(size_t i = 0; i < m_ct.tx_out_rsigs.size(); ++i){ + if (is_bulletproof()){ + m_ct.rv->p.bulletproofs.push_back(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs[i])); + } else { + m_ct.rv->p.rangeSigs.push_back(boost::get<rct::rangeSig>(m_ct.tx_out_rsigs[i])); + } + } + + rct::key hash_computed = rct::get_pre_mlsag_hash(*(m_ct.rv), hwdev); + auto & hash = ack->full_message_hash(); + + if (hash.size() != 32){ + throw exc::ProtocolException("Returned mlsag hash has invalid size"); + } + + if (crypto_verify_32(reinterpret_cast<const unsigned char *>(hash_computed.bytes), + reinterpret_cast<const unsigned char *>(hash.data()))){ + throw exc::proto::SecurityException("Computed MLSAG does not match"); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSignInputRequest> Signer::step_sign_input(size_t idx){ + m_ct.cur_input_idx = idx; + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.alphas.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.spend_encs.size(), "Invalid transaction index"); + + auto res = std::make_shared<messages::monero::MoneroTransactionSignInputRequest>(); + translate_src_entry(res->mutable_src_entr(), &(m_ct.tx_data.sources[idx])); + res->set_vini(cryptonote::t_serializable_object_to_blob(m_ct.tx.vin[idx])); + res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); + res->set_pseudo_out_alpha(m_ct.alphas[idx]); + res->set_spend_key(m_ct.spend_encs[idx]); + if (!in_memory()){ + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); + } + return res; + } + + void Signer::step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack){ + rct::mgSig mg; + if (!cn_deserialize(ack->signature(), mg)){ + throw exc::ProtocolException("Cannot deserialize mg[i]"); + } + + m_ct.rv->p.MGs.push_back(mg); + } + + std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> Signer::step_final(){ + m_ct.tx.rct_signatures = *(m_ct.rv); + return std::make_shared<messages::monero::MoneroTransactionFinalRequest>(); + } + + void Signer::step_final_ack(std::shared_ptr<const messages::monero::MoneroTransactionFinalAck> ack){ + if (m_multisig){ + auto & cout_key = ack->cout_key(); + for(auto & cur : m_ct.couts){ + if (cur.size() != 12 + 32){ + throw std::invalid_argument("Encrypted cout has invalid length"); + } + + char buff[32]; + auto data = cur.data(); + + crypto::chacha::decrypt(data + 12, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); + m_ct.couts_dec.emplace_back(buff, 32); + } + } + + m_ct.enc_salt1 = ack->salt(); + m_ct.enc_salt2 = ack->rand_mult(); + m_ct.enc_keys = ack->tx_enc_keys(); + } + + std::string Signer::store_tx_aux_info(){ + rapidjson::StringBuffer sb; + rapidjson::Writer<rapidjson::StringBuffer> writer(sb); + + rapidjson::Document json; + json.SetObject(); + + rapidjson::Value valueS(rapidjson::kStringType); + rapidjson::Value valueI(rapidjson::kNumberType); + + valueI.SetInt(1); + json.AddMember("version", valueI, json.GetAllocator()); + + valueS.SetString(m_ct.enc_salt1.c_str(), m_ct.enc_salt1.size()); + json.AddMember("salt1", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.enc_salt2.c_str(), m_ct.enc_salt2.size()); + json.AddMember("salt2", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.enc_keys.c_str(), m_ct.enc_keys.size()); + json.AddMember("enc_keys", valueS, json.GetAllocator()); + + json.Accept(writer); + return sb.GetString(); + } + + +} +} +} +} diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp new file mode 100644 index 000000000..99211efed --- /dev/null +++ b/src/device_trezor/trezor/protocol.hpp @@ -0,0 +1,300 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_PROTOCOL_H +#define MONERO_PROTOCOL_H + +#include "trezor_defs.hpp" +#include "device/device_cold.hpp" +#include "messages_map.hpp" +#include "transport.hpp" +#include "wallet/wallet2.h" + +namespace hw{ +namespace trezor{ +namespace protocol{ + + std::string key_to_string(const ::crypto::ec_point & key); + std::string key_to_string(const ::crypto::ec_scalar & key); + std::string key_to_string(const ::crypto::hash & key); + std::string key_to_string(const ::rct::key & key); + + void string_to_key(::crypto::ec_scalar & key, const std::string & str); + void string_to_key(::crypto::ec_point & key, const std::string & str); + void string_to_key(::rct::key & key, const std::string & str); + + template<class sub_t, class InputIterator> + void assign_to_repeatable(::google::protobuf::RepeatedField<sub_t> * dst, const InputIterator begin, const InputIterator end){ + for (InputIterator it = begin; it != end; it++) { + auto s = dst->Add(); + *s = *it; + } + } + + template<class sub_t, class InputIterator> + void assign_from_repeatable(std::vector<sub_t> * dst, const InputIterator begin, const InputIterator end){ + for (InputIterator it = begin; it != end; it++) { + dst->push_back(*it); + } + }; + + template<typename T> + bool cn_deserialize(const void * buff, size_t len, T & dst){ + std::stringstream ss; + ss.write(static_cast<const char *>(buff), len); //ss << tx_blob; + binary_archive<false> ba(ss); + bool r = ::serialization::serialize(ba, dst); + return r; + } + + template<typename T> + bool cn_deserialize(const std::string & str, T & dst){ + return cn_deserialize(str.data(), str.size(), dst); + } + + template<typename T> + std::string cn_serialize(T & obj){ + std::ostringstream oss; + binary_archive<true> oar(oss); + bool success = ::serialization::serialize(oar, obj); + if (!success){ + throw exc::EncodingException("Could not CN serialize given object"); + } + return oss.str(); + } + +// Crypto / encryption +namespace crypto { +namespace chacha { + + /** + * Chacha20Poly1305 decryption with tag verification. RFC 7539. + */ + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext); + +} +} + + +// Cold Key image sync +namespace ki { + + using MoneroTransferDetails = messages::monero::MoneroKeyImageSyncStepRequest_MoneroTransferDetails; + using MoneroSubAddressIndicesList = messages::monero::MoneroKeyImageExportInitRequest_MoneroSubAddressIndicesList; + using MoneroExportedKeyImage = messages::monero::MoneroKeyImageSyncStepAck_MoneroExportedKeyImage; + using exported_key_image = hw::device_cold::exported_key_image; + + /** + * Converts transfer details to the MoneroTransferDetails required for KI sync + */ + bool key_image_data(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::vector<MoneroTransferDetails> & res); + + /** + * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync. + */ + std::string compute_hash(const MoneroTransferDetails & rr); + + /** + * Generates KI sync request with commitments computed. + */ + void generate_commitment(std::vector<MoneroTransferDetails> & mtds, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req); + +} + +// Cold transaction signing +namespace tx { + using TsxData = messages::monero::MoneroTransactionInitRequest_MoneroTransactionData; + using MoneroTransactionDestinationEntry = messages::monero::MoneroTransactionDestinationEntry; + using MoneroAccountPublicAddress = messages::monero::MoneroTransactionDestinationEntry_MoneroAccountPublicAddress; + using MoneroTransactionSourceEntry = messages::monero::MoneroTransactionSourceEntry; + using MoneroMultisigKLRki = messages::monero::MoneroTransactionSourceEntry_MoneroMultisigKLRki; + using MoneroOutputEntry = messages::monero::MoneroTransactionSourceEntry_MoneroOutputEntry; + using MoneroRctKey = messages::monero::MoneroTransactionSourceEntry_MoneroOutputEntry_MoneroRctKeyPublic; + using MoneroRsigData = messages::monero::MoneroTransactionRsigData; + + using tx_construction_data = tools::wallet2::tx_construction_data; + using unsigned_tx_set = tools::wallet2::unsigned_tx_set; + + void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src); + void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src); + void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src); + void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src); + void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src); + std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + + typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v; + + /** + * Transaction signer state holder. + */ + class TData { + public: + TsxData tsx_data; + tx_construction_data tx_data; + cryptonote::transaction tx; + bool in_memory; + unsigned rsig_type; + std::vector<uint64_t> grouping_vct; + std::shared_ptr<MoneroRsigData> rsig_param; + size_t cur_input_idx; + size_t cur_output_idx; + size_t cur_batch_idx; + size_t cur_output_in_batch_idx; + + std::vector<std::string> tx_in_hmacs; + std::vector<std::string> tx_out_entr_hmacs; + std::vector<std::string> tx_out_hmacs; + std::vector<rsig_v> tx_out_rsigs; + std::vector<rct::ctkey> tx_out_pk; + std::vector<rct::ecdhTuple> tx_out_ecdh; + std::vector<size_t> source_permutation; + std::vector<std::string> alphas; + std::vector<std::string> spend_encs; + std::vector<std::string> pseudo_outs; + std::vector<std::string> pseudo_outs_hmac; + std::vector<std::string> couts; + std::vector<std::string> couts_dec; + std::vector<rct::key> rsig_gamma; + std::string tx_prefix_hash; + std::string enc_salt1; + std::string enc_salt2; + std::string enc_keys; + + std::shared_ptr<rct::rctSig> rv; + + TData(); + }; + + class Signer { + private: + TData m_ct; + wallet_shim * m_wallet2; + + size_t m_tx_idx; + const unsigned_tx_set * m_unsigned_tx; + hw::tx_aux_data * m_aux_data; + + bool m_multisig; + + const tx_construction_data & cur_tx(){ + CHECK_AND_ASSERT_THROW_MES(m_tx_idx < m_unsigned_tx->txes.size(), "Invalid transaction index"); + return m_unsigned_tx->txes[m_tx_idx]; + } + + void extract_payment_id(); + void compute_integrated_indices(TsxData * tsx_data); + + public: + Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr); + + std::shared_ptr<messages::monero::MoneroTransactionInitRequest> step_init(); + void step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionSetInputRequest> step_set_input(size_t idx); + void step_set_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetInputAck> ack); + + void sort_ki(); + std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> step_permutation(); + void step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> step_set_vini_input(size_t idx); + void step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> step_all_inputs_set(); + void step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_set_output(size_t idx); + void step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> step_all_outs_set(); + void step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev); + + std::shared_ptr<messages::monero::MoneroTransactionSignInputRequest> step_sign_input(size_t idx); + void step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> step_final(); + void step_final_ack(std::shared_ptr<const messages::monero::MoneroTransactionFinalAck> ack); + + std::string store_tx_aux_info(); + + bool in_memory() const { + return m_ct.in_memory; + } + + bool is_simple() const { + if (!m_ct.rv){ + throw std::invalid_argument("RV not initialized"); + } + auto tp = m_ct.rv->type; + return tp == rct::RCTTypeSimple; + } + + bool is_req_bulletproof() const { + return m_ct.tx_data.use_bulletproofs; + } + + bool is_bulletproof() const { + if (!m_ct.rv){ + throw std::invalid_argument("RV not initialized"); + } + auto tp = m_ct.rv->type; + return tp == rct::RCTTypeBulletproof; + } + + bool is_offloading() const { + return m_ct.rsig_param && m_ct.rsig_param->offload_type() != 0; + } + + size_t num_outputs() const { + return m_ct.tx_data.splitted_dsts.size(); + } + + size_t num_inputs() const { + return m_ct.tx_data.sources.size(); + } + + const TData & tdata() const { + return m_ct; + } + }; + +} + +} +} +} + + +#endif //MONERO_PROTOCOL_H diff --git a/src/device_trezor/trezor/tools/README.md b/src/device_trezor/trezor/tools/README.md new file mode 100644 index 000000000..91a8fb3f0 --- /dev/null +++ b/src/device_trezor/trezor/tools/README.md @@ -0,0 +1,36 @@ +# Trezor + +## Messages rebuild + +Install `protoc` for your distribution. + +- `protobuf-compiler` +- `libprotobuf-dev` +- `libprotoc-dev` +- `python-protobuf` + +Python 3 is required. If you don't have python 3 quite an easy way is +to use [pyenv]. + +It is also advised to create own python virtual environment so dependencies +are installed in this project-related virtual environment. + +```bash +python -m venv / +``` + +Make sure your python has `protobuf` package installed + +```bash +pip install protobuf +``` + +Regenerate messages: + +``` +./venv/bin/python3 src/device_trezor/trezor/tools/build_protob.py +``` + +The messages regeneration is done also automatically via cmake. + +[pyenv]: https://github.com/pyenv/pyenv diff --git a/src/device_trezor/trezor/tools/build_protob.py b/src/device_trezor/trezor/tools/build_protob.py new file mode 100644 index 000000000..2611f3296 --- /dev/null +++ b/src/device_trezor/trezor/tools/build_protob.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +import os +import subprocess +import sys + +CWD = os.path.dirname(os.path.realpath(__file__)) +ROOT_DIR = os.path.abspath(os.path.join(CWD, "..", "..", "..", "..")) +TREZOR_COMMON = os.path.join(ROOT_DIR, "external", "trezor-common") +TREZOR_MESSAGES = os.path.join(CWD, "..", "messages") + +# check for existence of the submodule directory +common_defs = os.path.join(TREZOR_COMMON, "defs") +if not os.path.exists(common_defs): + raise ValueError( + "trezor-common submodule seems to be missing.\n" + + 'Use "git submodule update --init --recursive" to retrieve it.' + ) + +# regenerate messages +try: + selected = [ + "messages.proto", + "messages-common.proto", + "messages-management.proto", + "messages-monero.proto", + ] + proto_srcs = [os.path.join(TREZOR_COMMON, "protob", x) for x in selected] + exec_args = [ + sys.executable, + os.path.join(CWD, "pb2cpp.py"), + "-o", + TREZOR_MESSAGES, + ] + proto_srcs + + subprocess.check_call(exec_args) + +except Exception as e: + raise diff --git a/src/device_trezor/trezor/tools/pb2cpp.py b/src/device_trezor/trezor/tools/pb2cpp.py new file mode 100644 index 000000000..eaa8a90ed --- /dev/null +++ b/src/device_trezor/trezor/tools/pb2cpp.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# Converts Google's protobuf python definitions of TREZOR wire messages +# to plain-python objects as used in TREZOR Core and python-trezor + +import argparse +import logging +import os +import re +import shutil +import subprocess +import sys +import glob +import tempfile +import hashlib + + +AUTO_HEADER = "# Automatically generated by pb2cpp\n" + +# Fixing GCC7 compilation error +UNDEF_STATEMENT = """ +#ifdef minor +#undef minor +#endif +""" + + +def which(pgm): + path = os.getenv('PATH') + for p in path.split(os.path.pathsep): + p = os.path.join(p, pgm) + if os.path.exists(p) and os.access(p, os.X_OK): + return p + + +PROTOC = which("protoc") +if not PROTOC: + print("protoc command not found") + sys.exit(1) + +PROTOC_PREFIX = os.path.dirname(os.path.dirname(PROTOC)) +PROTOC_INCLUDE = os.path.join(PROTOC_PREFIX, "include") + + +def namespace_file(fpath, package): + """Adds / replaces package name. Simple regex parsing, may use https://github.com/ph4r05/plyprotobuf later""" + with open(fpath) as fh: + fdata = fh.read() + + re_syntax = re.compile(r"^syntax\s*=") + re_package = re.compile(r"^package\s+([^;]+?)\s*;\s*$") + lines = fdata.split("\n") + + line_syntax = None + line_package = None + for idx, line in enumerate(lines): + if line_syntax is None and re_syntax.match(line): + line_syntax = idx + if line_package is None and re_package.match(line): + line_package = idx + + if package is None: + if line_package is None: + return + else: + lines.pop(line_package) + + else: + new_package = "package %s;" % package + if line_package is None: + lines.insert(line_syntax + 1 if line_syntax is not None else 0, new_package) + else: + lines[line_package] = new_package + + new_fdat = "\n".join(lines) + with open(fpath, "w+") as fh: + fh.write(new_fdat) + return new_fdat + + +def protoc(files, out_dir, additional_includes=(), package=None, force=False): + """Compile code with protoc and return the data.""" + + include_dirs = set() + include_dirs.add(PROTOC_INCLUDE) + include_dirs.update(additional_includes) + + with tempfile.TemporaryDirectory() as tmpdir_protob, tempfile.TemporaryDirectory() as tmpdir_out: + include_dirs.add(tmpdir_protob) + + new_files = [] + for file in files: + bname = os.path.basename(file) + tmp_file = os.path.join(tmpdir_protob, bname) + + shutil.copy(file, tmp_file) + if package is not None: + namespace_file(tmp_file, package) + new_files.append(tmp_file) + + protoc_includes = ["-I" + dir for dir in include_dirs if dir] + + exec_args = ( + [ + PROTOC, + "--cpp_out", + tmpdir_out, + ] + + protoc_includes + + new_files + ) + + subprocess.check_call(exec_args) + + # Fixing gcc compilation and clashes with "minor" field name + add_undef(tmpdir_out) + + # Scan output dir, check file differences + update_message_files(tmpdir_out, out_dir, force) + + +def update_message_files(tmpdir_out, out_dir, force=False): + files = glob.glob(os.path.join(tmpdir_out, '*.pb.*')) + for fname in files: + bname = os.path.basename(fname) + dest_file = os.path.join(out_dir, bname) + if not force and os.path.exists(dest_file): + data = open(fname, 'rb').read() + data_hash = hashlib.sha3_256(data).digest() + data_dest = open(dest_file, 'rb').read() + data_dest_hash = hashlib.sha3_256(data_dest).digest() + if data_hash == data_dest_hash: + continue + + shutil.copy(fname, dest_file) + + +def add_undef(out_dir): + files = glob.glob(os.path.join(out_dir, '*.pb.*')) + for fname in files: + with open(fname) as fh: + lines = fh.readlines() + + idx_insertion = None + for idx in range(len(lines)): + if '@@protoc_insertion_point(includes)' in lines[idx]: + idx_insertion = idx + break + + if idx_insertion is None: + pass + + lines.insert(idx_insertion + 1, UNDEF_STATEMENT) + with open(fname, 'w') as fh: + fh.write("".join(lines)) + + +def strip_leader(s, prefix): + """Remove given prefix from underscored name.""" + leader = prefix + "_" + if s.startswith(leader): + return s[len(leader) :] + else: + return s + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + + parser = argparse.ArgumentParser() + # fmt: off + parser.add_argument("proto", nargs="+", help="Protobuf definition files") + parser.add_argument("-o", "--out-dir", help="Directory for generated source code") + parser.add_argument("-n", "--namespace", default=None, help="Message namespace") + parser.add_argument("-I", "--protoc-include", action="append", help="protoc include path") + parser.add_argument("-P", "--protobuf-module", default="protobuf", help="Name of protobuf module") + parser.add_argument("-f", "--force", default=False, help="Overwrite existing files") + # fmt: on + args = parser.parse_args() + + protoc_includes = args.protoc_include or (os.environ.get("PROTOC_INCLUDE"),) + + protoc( + args.proto, args.out_dir, protoc_includes, package=args.namespace, force=args.force + ) + + diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp new file mode 100644 index 000000000..fc86177e1 --- /dev/null +++ b/src/device_trezor/trezor/transport.cpp @@ -0,0 +1,651 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include <boost/endian/conversion.hpp> +#include <boost/asio/io_service.hpp> +#include <boost/asio/ip/udp.hpp> +#include <boost/date_time/posix_time/posix_time_types.hpp> +#include "transport.hpp" +#include "messages/messages-common.pb.h" + +using namespace std; +using json = rapidjson::Document; + + +namespace hw{ +namespace trezor{ + + bool t_serialize(const std::string & in, std::string & out){ + out = in; + return true; + } + + bool t_serialize(const json_val & in, std::string & out){ + rapidjson::StringBuffer sb; + rapidjson::Writer<rapidjson::StringBuffer> writer(sb); + in.Accept(writer); + out = sb.GetString(); + return true; + } + + std::string t_serialize(const json_val & in){ + std::string ret; + t_serialize(in, ret); + return ret; + } + + bool t_deserialize(const std::string & in, std::string & out){ + out = in; + return true; + } + + bool t_deserialize(const std::string & in, json & out){ + if (out.Parse(in.c_str()).HasParseError()) { + throw exc::CommunicationException("JSON parse error"); + } + return true; + } + + static std::string json_get_string(const rapidjson::Value & in){ + return std::string(in.GetString()); + } + + // + // Helpers + // + +#define PROTO_HEADER_SIZE 6 + + static size_t message_size(const google::protobuf::Message &req){ + return static_cast<size_t>(req.ByteSize()); + } + + static size_t serialize_message_buffer_size(size_t msg_size) { + return PROTO_HEADER_SIZE + msg_size; // tag 2B + len 4B + } + + static void serialize_message_header(void * buff, uint16_t tag, uint32_t len){ + uint16_t wire_tag = boost::endian::native_to_big(static_cast<uint16_t>(tag)); + uint32_t wire_len = boost::endian::native_to_big(static_cast<uint32_t>(len)); + memcpy(buff, (void *) &wire_tag, 2); + memcpy((uint8_t*)buff + 2, (void *) &wire_len, 4); + } + + static void deserialize_message_header(const void * buff, uint16_t & tag, uint32_t & len){ + uint16_t wire_tag; + uint32_t wire_len; + memcpy(&wire_tag, buff, 2); + memcpy(&wire_len, (uint8_t*)buff + 2, 4); + + tag = boost::endian::big_to_native(wire_tag); + len = boost::endian::big_to_native(wire_len); + } + + static void serialize_message(const google::protobuf::Message &req, size_t msg_size, uint8_t * buff, size_t buff_size) { + auto msg_wire_num = MessageMapper::get_message_wire_number(req); + const auto req_buffer_size = serialize_message_buffer_size(msg_size); + if (req_buffer_size > buff_size){ + throw std::invalid_argument("Buffer too small"); + } + + serialize_message_header(buff, msg_wire_num, msg_size); + if (!req.SerializeToArray(buff + 6, msg_size)){ + throw exc::EncodingException("Message serialization error"); + } + } + + // + // Communication protocol + // + +#define REPLEN 64 + + void ProtocolV1::write(Transport & transport, const google::protobuf::Message & req){ + const auto msg_size = message_size(req); + const auto buff_size = serialize_message_buffer_size(msg_size) + 2; + + std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]); + uint8_t * req_buff_raw = req_buff.get(); + req_buff_raw[0] = '#'; + req_buff_raw[1] = '#'; + + serialize_message(req, msg_size, req_buff_raw + 2, buff_size - 2); + + size_t offset = 0; + uint8_t chunk_buff[REPLEN]; + + // Chunk by chunk upload + while(offset < buff_size){ + auto to_copy = std::min((size_t)(buff_size - offset), (size_t)(REPLEN - 1)); + + chunk_buff[0] = '?'; + memcpy(chunk_buff + 1, req_buff_raw + offset, to_copy); + + // Pad with zeros + if (to_copy < REPLEN - 1){ + memset(chunk_buff + 1 + to_copy, 0, REPLEN - 1 - to_copy); + } + + transport.write_chunk(chunk_buff, REPLEN); + offset += REPLEN - 1; + } + } + + void ProtocolV1::read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type){ + char chunk[REPLEN]; + + // Initial chunk read + size_t nread = transport.read_chunk(chunk, REPLEN); + if (nread != REPLEN){ + throw exc::CommunicationException("Read chunk has invalid size"); + } + + if (strncmp(chunk, "?##", 3) != 0){ + throw exc::CommunicationException("Malformed chunk"); + } + + uint16_t tag; + uint32_t len; + nread -= 3 + 6; + deserialize_message_header(chunk + 3, tag, len); + + std::string data_acc(chunk + 3 + 6, nread); + data_acc.reserve(len); + + while(nread < len){ + const size_t cur = transport.read_chunk(chunk, REPLEN); + if (chunk[0] != '?'){ + throw exc::CommunicationException("Chunk malformed"); + } + + data_acc.append(chunk + 1, cur - 1); + nread += cur - 1; + } + + if (msg_type){ + *msg_type = static_cast<messages::MessageType>(tag); + } + + if (nread < len){ + throw exc::CommunicationException("Response incomplete"); + } + + std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(tag)); + if (!msg_wrap->ParseFromArray(data_acc.c_str(), len)){ + throw exc::CommunicationException("Message could not be parsed"); + } + + msg = msg_wrap; + } + + // + // Bridge transport + // + + const char * BridgeTransport::PATH_PREFIX = "bridge:"; + + std::string BridgeTransport::get_path() const { + if (!m_device_path){ + return ""; + } + + std::string path(PATH_PREFIX); + return path + m_device_path.get(); + } + + void BridgeTransport::enumerate(t_transport_vect & res) { + json bridge_res; + std::string req; + + bool req_status = invoke_bridge_http("/enumerate", req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Bridge enumeration failed"); + } + + for(rapidjson::Value::ConstValueIterator itr = bridge_res.Begin(); itr != bridge_res.End(); ++itr){ + auto element = itr->GetObject(); + auto t = std::make_shared<BridgeTransport>(boost::make_optional(json_get_string(element["path"]))); + t->m_device_info.emplace(); + t->m_device_info->CopyFrom(*itr, t->m_device_info->GetAllocator()); + res.push_back(t); + } + } + + void BridgeTransport::open() { + if (!m_device_path){ + throw exc::CommunicationException("Coud not open, empty device path"); + } + + std::string uri = "/acquire/" + m_device_path.get() + "/null"; + std::string req; + json bridge_res; + bool req_status = invoke_bridge_http(uri, req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Failed to acquire device"); + } + + m_session = boost::make_optional(json_get_string(bridge_res["session"])); + } + + void BridgeTransport::close() { + if (!m_device_path || !m_session){ + throw exc::CommunicationException("Device not open"); + } + + std::string uri = "/release/" + m_session.get(); + std::string req; + json bridge_res; + bool req_status = invoke_bridge_http(uri, req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Failed to release device"); + } + + m_session = boost::none; + } + + void BridgeTransport::write(const google::protobuf::Message &req) { + m_response = boost::none; + + const auto msg_size = message_size(req); + const auto buff_size = serialize_message_buffer_size(msg_size); + + std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]); + uint8_t * req_buff_raw = req_buff.get(); + + serialize_message(req, msg_size, req_buff_raw, buff_size); + + std::string uri = "/call/" + m_session.get(); + std::string req_hex = epee::to_hex::string(epee::span<const std::uint8_t>(req_buff_raw, buff_size)); + std::string res_hex; + + bool req_status = invoke_bridge_http(uri, req_hex, res_hex, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Call method failed"); + } + + m_response = res_hex; + } + + void BridgeTransport::read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type) { + if (!m_response){ + throw exc::CommunicationException("Could not read, no response stored"); + } + + std::string bin_data; + if (!epee::string_tools::parse_hexstr_to_binbuff(m_response.get(), bin_data)){ + throw exc::CommunicationException("Response is not well hexcoded"); + } + + uint16_t msg_tag; + uint32_t msg_len; + deserialize_message_header(bin_data.c_str(), msg_tag, msg_len); + if (bin_data.size() != msg_len + 6){ + throw exc::CommunicationException("Response is not well hexcoded"); + } + + if (msg_type){ + *msg_type = static_cast<messages::MessageType>(msg_tag); + } + + std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(msg_tag)); + if (!msg_wrap->ParseFromArray(bin_data.c_str() + 6, msg_len)){ + throw exc::EncodingException("Response is not well hexcoded"); + } + msg = msg_wrap; + } + + const boost::optional<json> & BridgeTransport::device_info() const { + return m_device_info; + } + + std::ostream& BridgeTransport::dump(std::ostream& o) const { + return o << "BridgeTransport<path=" << (m_device_path ? get_path() : "None") + << ", info=" << (m_device_info ? t_serialize(m_device_info.get()) : "None") + << ", session=" << (m_session ? m_session.get() : "None") + << ">"; + } + + // + // UdpTransport + // + const char * UdpTransport::PATH_PREFIX = "udp:"; + const char * UdpTransport::DEFAULT_HOST = "127.0.0.1"; + const int UdpTransport::DEFAULT_PORT = 21324; + + UdpTransport::UdpTransport(boost::optional<std::string> device_path, + boost::optional<std::shared_ptr<Protocol>> proto) : + m_io_service(), m_deadline(m_io_service) + { + m_device_port = DEFAULT_PORT; + if (device_path) { + const std::string device_str = device_path.get(); + auto delim = device_str.find(':'); + if (delim == std::string::npos) { + m_device_host = device_str; + } else { + m_device_host = device_str.substr(0, delim); + m_device_port = std::stoi(device_str.substr(delim + 1)); + } + } else { + m_device_host = DEFAULT_HOST; + } + + if (m_device_port <= 1024 || m_device_port > 65535){ + throw std::invalid_argument("Port number invalid"); + } + + if (m_device_host != "localhost" && m_device_host != DEFAULT_HOST){ + throw std::invalid_argument("Local endpoint allowed only"); + } + + m_proto = proto ? proto.get() : std::make_shared<ProtocolV1>(); + } + + std::string UdpTransport::get_path() const { + std::string path(PATH_PREFIX); + return path + m_device_host + ":" + std::to_string(m_device_port); + } + + void UdpTransport::require_socket(){ + if (!m_socket){ + throw exc::NotConnectedException("Socket not connected"); + } + } + + bool UdpTransport::ping(){ + return ping_int(); + } + + bool UdpTransport::ping_int(boost::posix_time::time_duration timeout){ + require_socket(); + try { + std::string req = "PINGPING"; + char res[8]; + + m_socket->send_to(boost::asio::buffer(req.c_str(), req.size()), m_endpoint); + receive(res, 8, nullptr, false, timeout); + + return memcmp(res, "PONGPONG", 8) == 0; + + } catch(...){ + return false; + } + } + + void UdpTransport::enumerate(t_transport_vect & res) { + std::shared_ptr<UdpTransport> t = std::make_shared<UdpTransport>(); + bool t_works = false; + + try{ + t->open(); + t_works = t->ping(); + } catch(...) { + + } + t->close(); + if (t_works){ + res.push_back(t); + } + } + + void UdpTransport::open() { + udp::resolver resolver(m_io_service); + udp::resolver::query query(udp::v4(), m_device_host, std::to_string(m_device_port)); + m_endpoint = *resolver.resolve(query); + + m_socket.reset(new udp::socket(m_io_service)); + m_socket->open(udp::v4()); + + m_deadline.expires_at(boost::posix_time::pos_infin); + check_deadline(); + + m_proto->session_begin(*this); + } + + void UdpTransport::close() { + if (!m_socket){ + throw exc::CommunicationException("Socket is already closed"); + } + + m_proto->session_end(*this); + m_socket->close(); + m_socket = nullptr; + } + + void UdpTransport::write_chunk(const void * buff, size_t size){ + require_socket(); + + if (size != 64){ + throw exc::CommunicationException("Invalid chunk size"); + } + + auto written = m_socket->send_to(boost::asio::buffer(buff, size), m_endpoint); + if (size != written){ + throw exc::CommunicationException("Could not send the whole chunk"); + } + } + + size_t UdpTransport::read_chunk(void * buff, size_t size){ + require_socket(); + if (size < 64){ + throw std::invalid_argument("Buffer too small"); + } + + ssize_t len; + while(true) { + try { + boost::system::error_code ec; + len = receive(buff, size, &ec, true); + if (ec == boost::asio::error::operation_aborted) { + continue; + } else if (ec) { + throw exc::CommunicationException(std::string("Comm error: ") + ec.message()); + } + + if (len != 64) { + throw exc::CommunicationException("Invalid chunk size"); + } + + break; + + } catch(exc::CommunicationException const& e){ + throw; + } catch(std::exception const& e){ + MWARNING("Error reading chunk, reason: " << e.what()); + throw exc::CommunicationException(std::string("Chunk read error: ") + std::string(e.what())); + } + } + + return static_cast<size_t>(len); + } + + ssize_t UdpTransport::receive(void * buff, size_t size, boost::system::error_code * error_code, bool no_throw, boost::posix_time::time_duration timeout){ + boost::system::error_code ec; + boost::asio::mutable_buffer buffer = boost::asio::buffer(buff, size); + + require_socket(); + + // Set a deadline for the asynchronous operation. + m_deadline.expires_from_now(timeout); + + // Set up the variables that receive the result of the asynchronous + // operation. The error code is set to would_block to signal that the + // operation is incomplete. Asio guarantees that its asynchronous + // operations will never fail with would_block, so any other value in + // ec indicates completion. + ec = boost::asio::error::would_block; + std::size_t length = 0; + + // Start the asynchronous operation itself. The handle_receive function + // used as a callback will update the ec and length variables. + m_socket->async_receive_from(boost::asio::buffer(buffer), m_endpoint, + boost::bind(&UdpTransport::handle_receive, _1, _2, &ec, &length)); + + // Block until the asynchronous operation has completed. + do { + m_io_service.run_one(); + } + while (ec == boost::asio::error::would_block); + + if (error_code){ + *error_code = ec; + } + + if (no_throw){ + return length; + } + + // Operation result + if (ec == boost::asio::error::operation_aborted){ + throw exc::TimeoutException(); + + } else if (ec) { + MWARNING("Reading from UDP socket failed: " << ec.message()); + throw exc::CommunicationException(); + + } + + return length; + } + + void UdpTransport::write(const google::protobuf::Message &req) { + m_proto->write(*this, req); + } + + void UdpTransport::read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type) { + m_proto->read(*this, msg, msg_type); + } + + void UdpTransport::check_deadline(){ + if (!m_socket){ + return; // no active socket. + } + + // Check whether the deadline has passed. We compare the deadline against + // the current time since a new asynchronous operation may have moved the + // deadline before this actor had a chance to run. + if (m_deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) + { + // The deadline has passed. The outstanding asynchronous operation needs + // to be cancelled so that the blocked receive() function will return. + // + // Please note that cancel() has portability issues on some versions of + // Microsoft Windows, and it may be necessary to use close() instead. + // Consult the documentation for cancel() for further information. + m_socket->cancel(); + + // There is no longer an active deadline. The expiry is set to positive + // infinity so that the actor takes no action until a new deadline is set. + m_deadline.expires_at(boost::posix_time::pos_infin); + } + + // Put the actor back to sleep. + m_deadline.async_wait(boost::bind(&UdpTransport::check_deadline, this)); + } + + void UdpTransport::handle_receive(const boost::system::error_code &ec, std::size_t length, + boost::system::error_code *out_ec, std::size_t *out_length) { + *out_ec = ec; + *out_length = length; + } + + std::ostream& UdpTransport::dump(std::ostream& o) const { + return o << "UdpTransport<path=" << get_path() + << ", socket_alive=" << (m_socket ? "true" : "false") + << ">"; + } + + void enumerate(t_transport_vect & res){ + BridgeTransport bt; + bt.enumerate(res); + + hw::trezor::UdpTransport btu; + btu.enumerate(res); + } + + std::shared_ptr<Transport> transport(const std::string & path){ + if (boost::starts_with(path, BridgeTransport::PATH_PREFIX)){ + return std::make_shared<BridgeTransport>(path.substr(strlen(BridgeTransport::PATH_PREFIX))); + + } else if (boost::starts_with(path, UdpTransport::PATH_PREFIX)){ + return std::make_shared<UdpTransport>(path.substr(strlen(UdpTransport::PATH_PREFIX))); + + } else { + throw std::invalid_argument("Unknown Trezor device path: " + path); + + } + } + + void throw_failure_exception(const messages::common::Failure * failure) { + if (failure == nullptr){ + throw std::invalid_argument("Failure message cannot be null"); + } + + boost::optional<std::string> message = failure->has_message() ? boost::make_optional(failure->message()) : boost::none; + boost::optional<uint32_t> code = failure->has_code() ? boost::make_optional(static_cast<uint32_t>(failure->code())) : boost::none; + if (!code){ + throw exc::proto::FailureException(code, message); + } + + auto ecode = failure->code(); + if (ecode == messages::common::Failure_FailureType_Failure_UnexpectedMessage){ + throw exc::proto::UnexpectedMessageException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_ActionCancelled){ + throw exc::proto::CancelledException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_PinExpected){ + throw exc::proto::PinExpectedException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_PinInvalid){ + throw exc::proto::InvalidPinException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_NotEnoughFunds){ + throw exc::proto::NotEnoughFundsException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_NotInitialized){ + throw exc::proto::NotInitializedException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_FirmwareError){ + throw exc::proto::FirmwareErrorException(code, message); + } else { + throw exc::proto::FailureException(code, message); + } + } + + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t){ + return t.dump(o); + } + + std::ostream& operator<<(std::ostream& o, std::shared_ptr<hw::trezor::Transport> const& t){ + if (!t){ + return o << "None"; + } + + return t->dump(o); + } + +} +} + + diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp new file mode 100644 index 000000000..7b82fd06f --- /dev/null +++ b/src/device_trezor/trezor/transport.hpp @@ -0,0 +1,331 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_TRANSPORT_H +#define MONERO_TRANSPORT_H + + +#include <boost/asio.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/array.hpp> +#include <boost/utility/string_ref.hpp> + +#include <typeinfo> +#include <type_traits> +#include "net/http_client.h" + +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" + +#include "exceptions.hpp" +#include "trezor_defs.hpp" +#include "messages_map.hpp" + +#include "messages/messages.pb.h" +#include "messages/messages-common.pb.h" +#include "messages/messages-management.pb.h" +#include "messages/messages-monero.pb.h" + +namespace hw { +namespace trezor { + + using json = rapidjson::Document; + using json_val = rapidjson::Value; + namespace http = epee::net_utils::http; + + const std::string DEFAULT_BRIDGE = "127.0.0.1:21325"; + + // Base HTTP comm serialization. + bool t_serialize(const std::string & in, std::string & out); + bool t_serialize(const json_val & in, std::string & out); + std::string t_serialize(const json_val & in); + + bool t_deserialize(const std::string & in, std::string & out); + bool t_deserialize(const std::string & in, json & out); + + // Flexible json serialization. HTTP client tailored for bridge API + template<class t_req, class t_res, class t_transport> + bool invoke_bridge_http(const boost::string_ref uri, const t_req & out_struct, t_res & result_struct, t_transport& transport, const boost::string_ref method = "POST", std::chrono::milliseconds timeout = std::chrono::seconds(180)) + { + std::string req_param; + t_serialize(out_struct, req_param); + + http::fields_list additional_params; + additional_params.push_back(std::make_pair("Origin","https://monero.trezor.io")); + additional_params.push_back(std::make_pair("Content-Type","application/json; charset=utf-8")); + + const http::http_response_info* pri = nullptr; + if(!transport.invoke(uri, method, req_param, timeout, &pri, std::move(additional_params))) + { + MERROR("Failed to invoke http request to " << uri); + return false; + } + + if(!pri) + { + MERROR("Failed to invoke http request to " << uri << ", internal error (null response ptr)"); + return false; + } + + if(pri->m_response_code != 200) + { + MERROR("Failed to invoke http request to " << uri << ", wrong response code: " << pri->m_response_code + << " Response Body: " << pri->m_body); + return false; + } + + return t_deserialize(pri->m_body, result_struct); + } + + // Forward decl + class Transport; + class Protocol; + + // Communication protocol + class Protocol { + public: + Protocol() = default; + virtual ~Protocol() = default; + virtual void session_begin(Transport & transport){ }; + virtual void session_end(Transport & transport){ }; + virtual void write(Transport & transport, const google::protobuf::Message & req)= 0; + virtual void read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr)= 0; + }; + + class ProtocolV1 : public Protocol { + public: + ProtocolV1() = default; + virtual ~ProtocolV1() = default; + + void write(Transport & transport, const google::protobuf::Message & req) override; + void read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + }; + + + // Base transport + typedef std::vector<std::shared_ptr<Transport>> t_transport_vect; + + class Transport { + public: + Transport() = default; + virtual ~Transport() = default; + + virtual bool ping() { return false; }; + virtual std::string get_path() const { return ""; }; + virtual void enumerate(t_transport_vect & res){}; + virtual void open(){}; + virtual void close(){}; + virtual void write(const google::protobuf::Message & req) =0; + virtual void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) =0; + + virtual void write_chunk(const void * buff, size_t size) { }; + virtual size_t read_chunk(void * buff, size_t size) { return 0; }; + virtual std::ostream& dump(std::ostream& o) const { return o << "Transport<>"; } + }; + + // Bridge transport + class BridgeTransport : public Transport { + public: + BridgeTransport( + boost::optional<std::string> device_path = boost::none, + boost::optional<std::string> bridge_host = boost::none): + m_device_path(device_path), + m_bridge_host(bridge_host ? bridge_host.get() : DEFAULT_BRIDGE), + m_response(boost::none), + m_session(boost::none), + m_device_info(boost::none) + { + m_http_client.set_server(m_bridge_host, boost::none, false); + } + + virtual ~BridgeTransport() = default; + + static const char * PATH_PREFIX; + + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + + const boost::optional<json> & device_info() const; + std::ostream& dump(std::ostream& o) const override; + + private: + epee::net_utils::http::http_simple_client m_http_client; + std::string m_bridge_host; + boost::optional<std::string> m_device_path; + boost::optional<std::string> m_session; + boost::optional<std::string> m_response; + boost::optional<json> m_device_info; + }; + + // UdpTransport transport + using boost::asio::ip::udp; + + class UdpTransport : public Transport { + public: + + explicit UdpTransport( + boost::optional<std::string> device_path=boost::none, + boost::optional<std::shared_ptr<Protocol>> proto=boost::none); + + virtual ~UdpTransport() = default; + + static const char * PATH_PREFIX; + static const char * DEFAULT_HOST; + static const int DEFAULT_PORT; + + bool ping() override; + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + + void write_chunk(const void * buff, size_t size) override; + size_t read_chunk(void * buff, size_t size) override; + + std::ostream& dump(std::ostream& o) const override; + + private: + void require_socket(); + ssize_t receive(void * buff, size_t size, boost::system::error_code * error_code=nullptr, bool no_throw=false, boost::posix_time::time_duration timeout=boost::posix_time::seconds(10)); + void check_deadline(); + static void handle_receive(const boost::system::error_code& ec, std::size_t length, + boost::system::error_code* out_ec, std::size_t* out_length); + bool ping_int(boost::posix_time::time_duration timeout=boost::posix_time::milliseconds(1500)); + + std::shared_ptr<Protocol> m_proto; + std::string m_device_host; + int m_device_port; + + std::unique_ptr<udp::socket> m_socket; + boost::asio::io_service m_io_service; + boost::asio::deadline_timer m_deadline; + udp::endpoint m_endpoint; + }; + + // + // General helpers + // + + /** + * Enumerates all transports + */ + void enumerate(t_transport_vect & res); + + /** + * Transforms path to the transport + */ + std::shared_ptr<Transport> transport(const std::string & path); + + /** + * Transforms path to the particular transport + */ + template<class t_transport> + std::shared_ptr<t_transport> transport_typed(const std::string & path){ + auto t = transport(path); + if (!t){ + return nullptr; + } + + return std::dynamic_pointer_cast<t_transport>(t); + } + + // Exception carries unexpected message being received + namespace exc { + class UnexpectedMessageException: public ProtocolException { + protected: + hw::trezor::messages::MessageType recvType; + std::shared_ptr<google::protobuf::Message> recvMsg; + + public: + using ProtocolException::ProtocolException; + UnexpectedMessageException(): ProtocolException("Trezor returned unexpected message") {}; + UnexpectedMessageException(hw::trezor::messages::MessageType recvType, + const std::shared_ptr<google::protobuf::Message> & recvMsg) + : recvType(recvType), recvMsg(recvMsg) { + reason = std::string("Trezor returned unexpected message: ") + std::to_string(recvType); + } + }; + } + + /** + * Throws corresponding failure exception. + */ + [[ noreturn ]] void throw_failure_exception(const messages::common::Failure * failure); + + /** + * Simple wrapper for write-read message exchange with expected message response type. + * + * @throws UnexpectedMessageException if the response message type is different than expected. + * Exception contains message type and the message itself. + */ + template<class t_message> + std::shared_ptr<t_message> + exchange_message(Transport & transport, const google::protobuf::Message & req, + boost::optional<messages::MessageType> resp_type = boost::none) + { + // Require strictly protocol buffers response in the template. + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + + // Write the request + transport.write(req); + + // Read the response + std::shared_ptr<google::protobuf::Message> msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + transport.read(msg_resp, &msg_resp_type); + + // Determine type of expected message response + messages::MessageType required_type = resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>(); + + if (msg_resp_type == required_type) { + return message_ptr_retype<t_message>(msg_resp); + } else if (msg_resp_type == messages::MessageType_Failure){ + throw_failure_exception(dynamic_cast<messages::common::Failure*>(msg_resp.get())); + } else { + throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); + } + } + + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t); + std::ostream& operator<<(std::ostream& o, std::shared_ptr<hw::trezor::Transport> const& t); +}} + + +#endif //MONERO_TRANSPORT_H diff --git a/src/device_trezor/trezor/trezor_defs.hpp b/src/device_trezor/trezor/trezor_defs.hpp new file mode 100644 index 000000000..951a8f802 --- /dev/null +++ b/src/device_trezor/trezor/trezor_defs.hpp @@ -0,0 +1,48 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#if defined(HAVE_PROTOBUF) && !defined(WITHOUT_TREZOR) + #define WITH_DEVICE_TREZOR 1 +#else + #define WITH_DEVICE_TREZOR 0 +#endif + +#ifndef WITH_DEVICE_TREZOR_LITE +#define WITH_DEVICE_TREZOR_LITE 0 +#endif + +// Avoids protobuf undefined macro warning +#ifndef PROTOBUF_INLINE_NOT_IN_HEADERS +#define PROTOBUF_INLINE_NOT_IN_HEADERS 0 +#endif + +// Fixes gcc7 problem with minor macro defined clashing with minor() field. +#ifdef minor +#undef minor +#endif diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 3d6338856..94071c23c 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -215,7 +215,7 @@ namespace } boost::crc_32_type result; result.process_bytes(trimmed_words.data(), trimmed_words.length()); - return result.checksum() % crypto::ElectrumWords::seed_length; + return result.checksum() % word_list.size(); } /*! @@ -335,6 +335,7 @@ namespace crypto return false; } + w[0] = SWAP32LE(w[0]); dst.append((const char*)&w[0], 4); // copy 4 bytes to position memwipe(w, sizeof(w)); } diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index e9d2061e8..7cad6e077 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -62,8 +62,8 @@ namespace nodetool const command_line::arg_descriptor<int64_t> arg_in_peers = {"in-peers", "set max number of in peers", -1}; const command_line::arg_descriptor<int> arg_tos_flag = {"tos-flag", "set TOS flag", -1}; - const command_line::arg_descriptor<int64_t> arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1}; - const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1}; + const command_line::arg_descriptor<int64_t> arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", P2P_DEFAULT_LIMIT_RATE_UP}; + const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", P2P_DEFAULT_LIMIT_RATE_DOWN}; const command_line::arg_descriptor<int64_t> arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", -1}; const command_line::arg_descriptor<bool> arg_save_graph = {"save-graph", "Save data for dr monero", false}; diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 90e2f78b1..808945393 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -308,7 +308,7 @@ namespace nodetool epee::math_helper::once_a_time_seconds<1> m_connections_maker_interval; epee::math_helper::once_a_time_seconds<60*30, false> m_peerlist_store_interval; epee::math_helper::once_a_time_seconds<60> m_gray_peerlist_housekeeping_interval; - epee::math_helper::once_a_time_seconds<900, false> m_incoming_connections_interval; + epee::math_helper::once_a_time_seconds<3600, false> m_incoming_connections_interval; std::string m_bind_ip; std::string m_port; @@ -337,8 +337,8 @@ namespace nodetool cryptonote::network_type m_nettype; }; - const int64_t default_limit_up = 2048; // kB/s - const int64_t default_limit_down = 8192; // kB/s + const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP; // kB/s + const int64_t default_limit_down = P2P_DEFAULT_LIMIT_RATE_DOWN; // kB/s extern const command_line::arg_descriptor<std::string> arg_p2p_bind_ip; extern const command_line::arg_descriptor<std::string, false, true, 2> arg_p2p_bind_port; extern const command_line::arg_descriptor<uint32_t> arg_p2p_external_port; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index a61b6107f..fbf265fc9 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1334,12 +1334,19 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::check_incoming_connections() { - if (m_offline || m_hide_my_port) + if (m_offline) return true; if (get_incoming_connections_count() == 0) { - const el::Level level = el::Level::Warning; - MCLOG_RED(level, "global", "No incoming connections - check firewalls/routers allow port " << get_this_peer_port()); + if (m_hide_my_port || m_config.m_net_config.max_in_connection_count == 0) + { + MGINFO("Incoming connections disabled, enable them for full connectivity"); + } + else + { + const el::Level level = el::Level::Warning; + MCLOG_RED(level, "global", "No incoming connections - check firewalls/routers allow port " << get_this_peer_port()); + } } return true; } diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index 381f50872..bed48769a 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -29,8 +29,6 @@ // Adapted from Java code by Sarang Noether #include <stdlib.h> -#include <openssl/ssl.h> -#include <openssl/bn.h> #include <boost/thread/mutex.hpp> #include "misc_log_ex.h" #include "common/perf_timer.h" @@ -48,9 +46,15 @@ extern "C" //#define DEBUG_BP +#if 1 #define PERF_TIMER_START_BP(x) PERF_TIMER_START_UNIT(x, 1000000) +#define PERF_TIMER_STOP_BP(x) PERF_TIMER_STOP(x) +#else +#define PERF_TIMER_START_BP(x) ((void*)0) +#define PERF_TIMER_STOP_BP(x) ((void*)0) +#endif -#define STRAUS_SIZE_LIMIT 128 +#define STRAUS_SIZE_LIMIT 232 #define PIPPENGER_SIZE_LIMIT 0 namespace rct @@ -75,65 +79,20 @@ static const rct::keyV twoN = vector_powers(TWO, maxN); static const rct::key ip12 = inner_product(oneN, twoN); static boost::mutex init_mutex; -static inline rct::key multiexp(const std::vector<MultiexpData> &data, bool HiGi) +static inline rct::key multiexp(const std::vector<MultiexpData> &data, size_t HiGi_size) { - if (HiGi) + if (HiGi_size > 0) { - static_assert(128 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); - return data.size() <= 128 ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, get_pippenger_c(data.size())); + static_assert(232 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); + return HiGi_size <= 232 && data.size() == HiGi_size ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, HiGi_size, get_pippenger_c(data.size())); } else - return data.size() <= 64 ? straus(data, NULL, 0) : pippenger(data, NULL, get_pippenger_c(data.size())); -} - -static bool is_reduced(const rct::key &scalar) -{ - rct::key reduced = scalar; - sc_reduce32(reduced.bytes); - return scalar == reduced; -} - -static void addKeys_acc_p3(ge_p3 *acc_p3, const rct::key &a, const rct::key &point) -{ - ge_p3 p3; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); - ge_scalarmult_p3(&p3, a.bytes, &p3); - ge_cached cached; - ge_p3_to_cached(&cached, acc_p3); - ge_p1p1 p1; - ge_add(&p1, &p3, &cached); - ge_p1p1_to_p3(acc_p3, &p1); -} - -static void add_acc_p3(ge_p3 *acc_p3, const rct::key &point) -{ - ge_p3 p3; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); - ge_cached cached; - ge_p3_to_cached(&cached, &p3); - ge_p1p1 p1; - ge_add(&p1, acc_p3, &cached); - ge_p1p1_to_p3(acc_p3, &p1); + return data.size() <= 95 ? straus(data, NULL, 0) : pippenger(data, NULL, 0, get_pippenger_c(data.size())); } -static void sub_acc_p3(ge_p3 *acc_p3, const rct::key &point) +static inline bool is_reduced(const rct::key &scalar) { - ge_p3 p3; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); - ge_cached cached; - ge_p3_to_cached(&cached, &p3); - ge_p1p1 p1; - ge_sub(&p1, acc_p3, &cached); - ge_p1p1_to_p3(acc_p3, &p1); -} - -static rct::key scalarmultKey(const ge_p3 &P, const rct::key &a) -{ - ge_p2 R; - ge_scalarmult(&R, a.bytes, &P); - rct::key aP; - ge_tobytes(aP.bytes, &R); - return aP; + return sc_check(scalar.bytes) == 0; } static rct::key get_exponent(const rct::key &base, size_t idx) @@ -160,12 +119,12 @@ static void init_exponents() Gi[i] = get_exponent(rct::H, i * 2 + 1); CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Gi_p3[i], Gi[i].bytes) == 0, "ge_frombytes_vartime failed"); - data.push_back({rct::zero(), Gi[i]}); - data.push_back({rct::zero(), Hi[i]}); + data.push_back({rct::zero(), Gi_p3[i]}); + data.push_back({rct::zero(), Hi_p3[i]}); } straus_HiGi_cache = straus_init_cache(data, STRAUS_SIZE_LIMIT); - pippenger_HiGi_cache = pippenger_init_cache(data, PIPPENGER_SIZE_LIMIT); + pippenger_HiGi_cache = pippenger_init_cache(data, 0, PIPPENGER_SIZE_LIMIT); MINFO("Hi/Gi cache size: " << (sizeof(Hi)+sizeof(Gi))/1024 << " kB"); MINFO("Hi_p3/Gi_p3 cache size: " << (sizeof(Hi_p3)+sizeof(Gi_p3))/1024 << " kB"); @@ -189,29 +148,37 @@ static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b) multiexp_data.emplace_back(a[i], Gi_p3[i]); multiexp_data.emplace_back(b[i], Hi_p3[i]); } - return multiexp(multiexp_data, true); + return multiexp(multiexp_data, 2 * a.size()); } /* Compute a custom vector-scalar commitment */ -static rct::key vector_exponent_custom(const rct::keyV &A, const rct::keyV &B, const rct::keyV &a, const rct::keyV &b) +static rct::key cross_vector_exponent8(size_t size, const std::vector<ge_p3> &A, size_t Ao, const std::vector<ge_p3> &B, size_t Bo, const rct::keyV &a, size_t ao, const rct::keyV &b, size_t bo, const rct::keyV *scale, const ge_p3 *extra_point, const rct::key *extra_scalar) { - CHECK_AND_ASSERT_THROW_MES(A.size() == B.size(), "Incompatible sizes of A and B"); - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() == A.size(), "Incompatible sizes of a and A"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + CHECK_AND_ASSERT_THROW_MES(size + Ao <= A.size(), "Incompatible size for A"); + CHECK_AND_ASSERT_THROW_MES(size + Bo <= B.size(), "Incompatible size for B"); + CHECK_AND_ASSERT_THROW_MES(size + ao <= a.size(), "Incompatible size for a"); + CHECK_AND_ASSERT_THROW_MES(size + bo <= b.size(), "Incompatible size for b"); + CHECK_AND_ASSERT_THROW_MES(size <= maxN*maxM, "size is too large"); + CHECK_AND_ASSERT_THROW_MES(!scale || size == scale->size() / 2, "Incompatible size for scale"); + CHECK_AND_ASSERT_THROW_MES(!!extra_point == !!extra_scalar, "only one of extra point/scalar present"); std::vector<MultiexpData> multiexp_data; - multiexp_data.reserve(a.size()*2); - for (size_t i = 0; i < a.size(); ++i) + multiexp_data.resize(size*2 + (!!extra_point)); + for (size_t i = 0; i < size; ++i) { - multiexp_data.resize(multiexp_data.size() + 1); - multiexp_data.back().scalar = a[i]; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, A[i].bytes) == 0, "ge_frombytes_vartime failed"); - multiexp_data.resize(multiexp_data.size() + 1); - multiexp_data.back().scalar = b[i]; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, B[i].bytes) == 0, "ge_frombytes_vartime failed"); + sc_mul(multiexp_data[i*2].scalar.bytes, a[ao+i].bytes, INV_EIGHT.bytes);; + multiexp_data[i*2].point = A[Ao+i]; + sc_mul(multiexp_data[i*2+1].scalar.bytes, b[bo+i].bytes, INV_EIGHT.bytes); + if (scale) + sc_mul(multiexp_data[i*2+1].scalar.bytes, multiexp_data[i*2+1].scalar.bytes, (*scale)[Bo+i].bytes); + multiexp_data[i*2+1].point = B[Bo+i]; } - return multiexp(multiexp_data, false); + if (extra_point) + { + sc_mul(multiexp_data.back().scalar.bytes, extra_scalar->bytes, INV_EIGHT.bytes); + multiexp_data.back().point = *extra_point; + } + return multiexp(multiexp_data, 0); } /* Given a scalar, construct a vector of powers */ @@ -273,16 +240,22 @@ static rct::keyV hadamard(const rct::keyV &a, const rct::keyV &b) return res; } -/* Given two curvepoint arrays, construct the Hadamard product */ -static rct::keyV hadamard2(const rct::keyV &a, const rct::keyV &b) +/* folds a curvepoint array using a two way scaled Hadamard product */ +static void hadamard_fold(std::vector<ge_p3> &v, const rct::keyV *scale, const rct::key &a, const rct::key &b) { - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - rct::keyV res(a.size()); - for (size_t i = 0; i < a.size(); ++i) + CHECK_AND_ASSERT_THROW_MES((v.size() & 1) == 0, "Vector size should be even"); + const size_t sz = v.size() / 2; + for (size_t n = 0; n < sz; ++n) { - rct::addKeys(res[i], a[i], b[i]); + ge_dsmp c[2]; + ge_dsm_precomp(c[0], &v[n]); + ge_dsm_precomp(c[1], &v[sz + n]); + rct::key sa, sb; + if (scale) sc_mul(sa.bytes, a.bytes, (*scale)[n].bytes); else sa = a; + if (scale) sc_mul(sb.bytes, b.bytes, (*scale)[sz + n].bytes); else sb = b; + ge_double_scalarmult_precomp_vartime2_p3(&v[n], sa.bytes, c[0], sb.bytes, c[1]); } - return res; + v.resize(sz); } /* Add two vectors */ @@ -297,88 +270,98 @@ static rct::keyV vector_add(const rct::keyV &a, const rct::keyV &b) return res; } -/* Subtract two vectors */ -static rct::keyV vector_subtract(const rct::keyV &a, const rct::keyV &b) +/* Add a scalar to all elements of a vector */ +static rct::keyV vector_add(const rct::keyV &a, const rct::key &b) { - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - sc_sub(res[i].bytes, a[i].bytes, b[i].bytes); + sc_add(res[i].bytes, a[i].bytes, b.bytes); } return res; } -/* Multiply a scalar and a vector */ -static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) +/* Subtract a scalar from all elements of a vector */ +static rct::keyV vector_subtract(const rct::keyV &a, const rct::key &b) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - sc_mul(res[i].bytes, a[i].bytes, x.bytes); + sc_sub(res[i].bytes, a[i].bytes, b.bytes); } return res; } -/* Create a vector from copies of a single value */ -static rct::keyV vector_dup(const rct::key &x, size_t N) -{ - return rct::keyV(N, x); -} - -/* Exponentiate a curve vector by a scalar */ -static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) +/* Multiply a scalar and a vector */ +static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - rct::scalarmultKey(res[i], a[i], x); + sc_mul(res[i].bytes, a[i].bytes, x.bytes); } return res; } -/* Get the sum of a vector's elements */ -static rct::key vector_sum(const rct::keyV &a) +/* Create a vector from copies of a single value */ +static rct::keyV vector_dup(const rct::key &x, size_t N) { - rct::key res = rct::zero(); - for (size_t i = 0; i < a.size(); ++i) - { - sc_add(res.bytes, res.bytes, a[i].bytes); - } - return res; + return rct::keyV(N, x); } -static rct::key switch_endianness(rct::key k) +static rct::key sm(rct::key y, int n, const rct::key &x) { - std::reverse(k.bytes, k.bytes + sizeof(k)); - return k; + while (n--) + sc_mul(y.bytes, y.bytes, y.bytes); + sc_mul(y.bytes, y.bytes, x.bytes); + return y; } -/* Compute the inverse of a scalar, the stupid way */ +/* Compute the inverse of a scalar, the clever way */ static rct::key invert(const rct::key &x) { - rct::key inv; - - BN_CTX *ctx = BN_CTX_new(); - BIGNUM *X = BN_new(); - BIGNUM *L = BN_new(); - BIGNUM *I = BN_new(); - - BN_bin2bn(switch_endianness(x).bytes, sizeof(rct::key), X); - BN_bin2bn(switch_endianness(rct::curveOrder()).bytes, sizeof(rct::key), L); - - CHECK_AND_ASSERT_THROW_MES(BN_mod_inverse(I, X, L, ctx), "Failed to invert"); - - const int len = BN_num_bytes(I); - CHECK_AND_ASSERT_THROW_MES((size_t)len <= sizeof(rct::key), "Invalid number length"); - inv = rct::zero(); - BN_bn2bin(I, inv.bytes); - std::reverse(inv.bytes, inv.bytes + len); + rct::key _1, _10, _100, _11, _101, _111, _1001, _1011, _1111; + + _1 = x; + sc_mul(_10.bytes, _1.bytes, _1.bytes); + sc_mul(_100.bytes, _10.bytes, _10.bytes); + sc_mul(_11.bytes, _10.bytes, _1.bytes); + sc_mul(_101.bytes, _10.bytes, _11.bytes); + sc_mul(_111.bytes, _10.bytes, _101.bytes); + sc_mul(_1001.bytes, _10.bytes, _111.bytes); + sc_mul(_1011.bytes, _10.bytes, _1001.bytes); + sc_mul(_1111.bytes, _100.bytes, _1011.bytes); - BN_free(I); - BN_free(L); - BN_free(X); - BN_CTX_free(ctx); + rct::key inv; + sc_mul(inv.bytes, _1111.bytes, _1.bytes); + + inv = sm(inv, 123 + 3, _101); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 4, _1001); + inv = sm(inv, 2, _11); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 1 + 3, _101); + inv = sm(inv, 3 + 3, _101); + inv = sm(inv, 3, _111); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 2 + 3, _111); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 1 + 4, _1011); + inv = sm(inv, 2 + 4, _1011); + inv = sm(inv, 6 + 4, _1001); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 3 + 2, _11); + inv = sm(inv, 3 + 2, _11); + inv = sm(inv, 1 + 4, _1001); + inv = sm(inv, 1 + 3, _111); + inv = sm(inv, 2 + 4, _1111); + inv = sm(inv, 1 + 4, _1011); + inv = sm(inv, 3, _101); + inv = sm(inv, 2 + 4, _1111); + inv = sm(inv, 3, _101); + inv = sm(inv, 1 + 2, _11); #ifdef DEBUG_BP rct::key tmp; @@ -388,6 +371,34 @@ static rct::key invert(const rct::key &x) return inv; } +static rct::keyV invert(rct::keyV x) +{ + rct::keyV scratch; + scratch.reserve(x.size()); + + rct::key acc = rct::identity(); + for (size_t n = 0; n < x.size(); ++n) + { + scratch.push_back(acc); + if (n == 0) + acc = x[0]; + else + sc_mul(acc.bytes, acc.bytes, x[n].bytes); + } + + acc = invert(acc); + + rct::key tmp; + for (int i = x.size(); i-- > 0; ) + { + sc_mul(tmp.bytes, acc.bytes, x[i].bytes); + sc_mul(x[i].bytes, acc.bytes, scratch[i].bytes); + acc = tmp; + } + + return x; +} + /* Compute the slice of a vector */ static rct::keyV slice(const rct::keyV &a, size_t start, size_t stop) { @@ -438,270 +449,12 @@ static rct::key hash_cache_mash(rct::key &hash_cache, const rct::key &mash0, con /* Given a value v (0..2^N-1) and a mask gamma, construct a range proof */ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) { - init_exponents(); - - PERF_TIMER_UNIT(PROVE, 1000000); - - constexpr size_t logN = 6; // log2(64) - constexpr size_t N = 1<<logN; - - rct::key V; - rct::keyV aL(N), aR(N); - - PERF_TIMER_START_BP(PROVE_v); - rct::addKeys2(V, gamma, sv, rct::H); - V = rct::scalarmultKey(V, INV_EIGHT); - PERF_TIMER_STOP(PROVE_v); - - PERF_TIMER_START_BP(PROVE_aLaR); - for (size_t i = N; i-- > 0; ) - { - if (sv[i/8] & (((uint64_t)1)<<(i%8))) - { - aL[i] = rct::identity(); - } - else - { - aL[i] = rct::zero(); - } - sc_sub(aR[i].bytes, aL[i].bytes, rct::identity().bytes); - } - PERF_TIMER_STOP(PROVE_aLaR); - - rct::key hash_cache = rct::hash_to_scalar(V); - - // DEBUG: Test to ensure this recovers the value -#ifdef DEBUG_BP - uint64_t test_aL = 0, test_aR = 0; - for (size_t i = 0; i < N; ++i) - { - if (aL[i] == rct::identity()) - test_aL += ((uint64_t)1)<<i; - if (aR[i] == rct::zero()) - test_aR += ((uint64_t)1)<<i; - } - uint64_t v_test = 0; - for (int n = 0; n < 8; ++n) v_test |= (((uint64_t)sv[n]) << (8*n)); - CHECK_AND_ASSERT_THROW_MES(test_aL == v_test, "test_aL failed"); - CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); -#endif - -try_again: - PERF_TIMER_START_BP(PROVE_step1); - // PAPER LINES 38-39 - rct::key alpha = rct::skGen(); - rct::key ve = vector_exponent(aL, aR); - rct::key A; - rct::addKeys(A, ve, rct::scalarmultBase(alpha)); - A = rct::scalarmultKey(A, INV_EIGHT); - - // PAPER LINES 40-42 - rct::keyV sL = rct::skvGen(N), sR = rct::skvGen(N); - rct::key rho = rct::skGen(); - ve = vector_exponent(sL, sR); - rct::key S; - rct::addKeys(S, ve, rct::scalarmultBase(rho)); - S = rct::scalarmultKey(S, INV_EIGHT); - - // PAPER LINES 43-45 - rct::key y = hash_cache_mash(hash_cache, A, S); - if (y == rct::zero()) - { - PERF_TIMER_STOP(PROVE_step1); - MINFO("y is 0, trying again"); - goto try_again; - } - rct::key z = hash_cache = rct::hash_to_scalar(y); - if (z == rct::zero()) - { - PERF_TIMER_STOP(PROVE_step1); - MINFO("z is 0, trying again"); - goto try_again; - } - - // Polynomial construction before PAPER LINE 46 - rct::key t0 = rct::zero(); - rct::key t1 = rct::zero(); - rct::key t2 = rct::zero(); - - const auto yN = vector_powers(y, N); - - rct::key ip1y = vector_sum(yN); - rct::key tmp; - sc_muladd(t0.bytes, z.bytes, ip1y.bytes, t0.bytes); - - rct::key zsq; - sc_mul(zsq.bytes, z.bytes, z.bytes); - sc_muladd(t0.bytes, zsq.bytes, sv.bytes, t0.bytes); - - rct::key k = rct::zero(); - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); - - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - sc_add(t0.bytes, t0.bytes, k.bytes); - - // DEBUG: Test the value of t0 has the correct form -#ifdef DEBUG_BP - rct::key test_t0 = rct::zero(); - rct::key iph = inner_product(aL, hadamard(aR, yN)); - sc_add(test_t0.bytes, test_t0.bytes, iph.bytes); - rct::key ips = inner_product(vector_subtract(aL, aR), yN); - sc_muladd(test_t0.bytes, z.bytes, ips.bytes, test_t0.bytes); - rct::key ipt = inner_product(twoN, aL); - sc_muladd(test_t0.bytes, zsq.bytes, ipt.bytes, test_t0.bytes); - sc_add(test_t0.bytes, test_t0.bytes, k.bytes); - CHECK_AND_ASSERT_THROW_MES(t0 == test_t0, "t0 check failed"); -#endif - PERF_TIMER_STOP(PROVE_step1); - - PERF_TIMER_START_BP(PROVE_step2); - const auto HyNsR = hadamard(yN, sR); - const auto vpIz = vector_dup(z, N); - const auto vp2zsq = vector_scalar(twoN, zsq); - const auto aL_vpIz = vector_subtract(aL, vpIz); - const auto aR_vpIz = vector_add(aR, vpIz); - - rct::key ip1 = inner_product(aL_vpIz, HyNsR); - sc_add(t1.bytes, t1.bytes, ip1.bytes); - - rct::key ip2 = inner_product(sL, vector_add(hadamard(yN, aR_vpIz), vp2zsq)); - sc_add(t1.bytes, t1.bytes, ip2.bytes); - - rct::key ip3 = inner_product(sL, HyNsR); - sc_add(t2.bytes, t2.bytes, ip3.bytes); - - // PAPER LINES 47-48 - rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - - rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); - T1 = rct::scalarmultKey(T1, INV_EIGHT); - rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); - T2 = rct::scalarmultKey(T2, INV_EIGHT); - - // PAPER LINES 49-51 - rct::key x = hash_cache_mash(hash_cache, z, T1, T2); - if (x == rct::zero()) - { - PERF_TIMER_STOP(PROVE_step2); - MINFO("x is 0, trying again"); - goto try_again; - } - - // PAPER LINES 52-53 - rct::key taux = rct::zero(); - sc_mul(taux.bytes, tau1.bytes, x.bytes); - rct::key xsq; - sc_mul(xsq.bytes, x.bytes, x.bytes); - sc_muladd(taux.bytes, tau2.bytes, xsq.bytes, taux.bytes); - sc_muladd(taux.bytes, gamma.bytes, zsq.bytes, taux.bytes); - rct::key mu; - sc_muladd(mu.bytes, x.bytes, rho.bytes, alpha.bytes); - - // PAPER LINES 54-57 - rct::keyV l = vector_add(aL_vpIz, vector_scalar(sL, x)); - rct::keyV r = vector_add(hadamard(yN, vector_add(aR_vpIz, vector_scalar(sR, x))), vp2zsq); - PERF_TIMER_STOP(PROVE_step2); - - PERF_TIMER_START_BP(PROVE_step3); - rct::key t = inner_product(l, r); - - // DEBUG: Test if the l and r vectors match the polynomial forms -#ifdef DEBUG_BP - rct::key test_t; - sc_muladd(test_t.bytes, t1.bytes, x.bytes, t0.bytes); - sc_muladd(test_t.bytes, t2.bytes, xsq.bytes, test_t.bytes); - CHECK_AND_ASSERT_THROW_MES(test_t == t, "test_t check failed"); -#endif - - // PAPER LINES 32-33 - rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); - - // These are used in the inner product rounds - size_t nprime = N; - rct::keyV Gprime(N); - rct::keyV Hprime(N); - rct::keyV aprime(N); - rct::keyV bprime(N); - const rct::key yinv = invert(y); - rct::key yinvpow = rct::identity(); - for (size_t i = 0; i < N; ++i) - { - Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); - aprime[i] = l[i]; - bprime[i] = r[i]; - } - rct::keyV L(logN); - rct::keyV R(logN); - int round = 0; - rct::keyV w(logN); // this is the challenge x in the inner product protocol - PERF_TIMER_STOP(PROVE_step3); - - PERF_TIMER_START_BP(PROVE_step4); - // PAPER LINE 13 - while (nprime > 1) - { - // PAPER LINE 15 - nprime /= 2; - - // PAPER LINES 16-17 - rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); - rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); - - // PAPER LINES 18-19 - L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); - sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); - L[round] = rct::scalarmultKey(L[round], INV_EIGHT); - R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); - sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); - R[round] = rct::scalarmultKey(R[round], INV_EIGHT); - - // PAPER LINES 21-22 - w[round] = hash_cache_mash(hash_cache, L[round], R[round]); - if (w[round] == rct::zero()) - { - PERF_TIMER_STOP(PROVE_step4); - MINFO("w[round] is 0, trying again"); - goto try_again; - } - - // PAPER LINES 24-25 - const rct::key winv = invert(w[round]); - Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); - Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); - - // PAPER LINES 28-29 - aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); - bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); - - ++round; - } - PERF_TIMER_STOP(PROVE_step4); - - // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) - return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); + return bulletproof_PROVE(rct::keyV(1, sv), rct::keyV(1, gamma)); } Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) { - // vG + gammaH - PERF_TIMER_START_BP(PROVE_v); - rct::key sv = rct::zero(); - sv.bytes[0] = v & 255; - sv.bytes[1] = (v >> 8) & 255; - sv.bytes[2] = (v >> 16) & 255; - sv.bytes[3] = (v >> 24) & 255; - sv.bytes[4] = (v >> 32) & 255; - sv.bytes[5] = (v >> 40) & 255; - sv.bytes[6] = (v >> 48) & 255; - sv.bytes[7] = (v >> 56) & 255; - PERF_TIMER_STOP(PROVE_v); - return bulletproof_PROVE(sv, gamma); + return bulletproof_PROVE(std::vector<uint64_t>(1, v), rct::keyV(1, gamma)); } /* Given a set of values v (0..2^N-1) and masks gamma, construct a range proof */ @@ -728,37 +481,39 @@ Bulletproof bulletproof_PROVE(const rct::keyV &sv, const rct::keyV &gamma) rct::keyV V(sv.size()); rct::keyV aL(MN), aR(MN); - rct::key tmp; + rct::keyV aL8(MN), aR8(MN); + rct::key tmp, tmp2; PERF_TIMER_START_BP(PROVE_v); for (size_t i = 0; i < sv.size(); ++i) { - rct::addKeys2(V[i], gamma[i], sv[i], rct::H); - V[i] = rct::scalarmultKey(V[i], INV_EIGHT); + rct::key gamma8, sv8; + sc_mul(gamma8.bytes, gamma[i].bytes, INV_EIGHT.bytes); + sc_mul(sv8.bytes, sv[i].bytes, INV_EIGHT.bytes); + rct::addKeys2(V[i], gamma8, sv8, rct::H); } - PERF_TIMER_STOP(PROVE_v); + PERF_TIMER_STOP_BP(PROVE_v); PERF_TIMER_START_BP(PROVE_aLaR); for (size_t j = 0; j < M; ++j) { for (size_t i = N; i-- > 0; ) { - if (j >= sv.size()) - { - aL[j*N+i] = rct::zero(); - } - else if (sv[j][i/8] & (((uint64_t)1)<<(i%8))) + if (j < sv.size() && (sv[j][i/8] & (((uint64_t)1)<<(i%8)))) { aL[j*N+i] = rct::identity(); + aL8[j*N+i] = INV_EIGHT; + aR[j*N+i] = aR8[j*N+i] = rct::zero(); } else { - aL[j*N+i] = rct::zero(); + aL[j*N+i] = aL8[j*N+i] = rct::zero(); + aR[j*N+i] = MINUS_ONE; + aR8[j*N+i] = MINUS_INV_EIGHT; } - sc_sub(aR[j*N+i].bytes, aL[j*N+i].bytes, rct::identity().bytes); } } - PERF_TIMER_STOP(PROVE_aLaR); + PERF_TIMER_STOP_BP(PROVE_aLaR); // DEBUG: Test to ensure this recovers the value #ifdef DEBUG_BP @@ -786,10 +541,10 @@ try_again: PERF_TIMER_START_BP(PROVE_step1); // PAPER LINES 38-39 rct::key alpha = rct::skGen(); - rct::key ve = vector_exponent(aL, aR); + rct::key ve = vector_exponent(aL8, aR8); rct::key A; - rct::addKeys(A, ve, rct::scalarmultBase(alpha)); - A = rct::scalarmultKey(A, INV_EIGHT); + sc_mul(tmp.bytes, alpha.bytes, INV_EIGHT.bytes); + rct::addKeys(A, ve, rct::scalarmultBase(tmp)); // PAPER LINES 40-42 rct::keyV sL = rct::skvGen(MN), sR = rct::skvGen(MN); @@ -803,21 +558,20 @@ try_again: rct::key y = hash_cache_mash(hash_cache, A, S); if (y == rct::zero()) { - PERF_TIMER_STOP(PROVE_step1); + PERF_TIMER_STOP_BP(PROVE_step1); MINFO("y is 0, trying again"); goto try_again; } rct::key z = hash_cache = rct::hash_to_scalar(y); if (z == rct::zero()) { - PERF_TIMER_STOP(PROVE_step1); + PERF_TIMER_STOP_BP(PROVE_step1); MINFO("z is 0, trying again"); goto try_again; } // Polynomial construction by coefficients - const auto zMN = vector_dup(z, MN); - rct::keyV l0 = vector_subtract(aL, zMN); + rct::keyV l0 = vector_subtract(aL, z); const rct::keyV &l1 = sL; // This computes the ugly sum/concatenation from PAPER LINE 65 @@ -837,7 +591,7 @@ try_again: } } - rct::keyV r0 = vector_add(aR, zMN); + rct::keyV r0 = vector_add(aR, z); const auto yMN = vector_powers(y, MN); r0 = hadamard(r0, yMN); r0 = vector_add(r0, zero_twos); @@ -850,22 +604,28 @@ try_again: sc_add(t1.bytes, t1_1.bytes, t1_2.bytes); rct::key t2 = inner_product(l1, r1); - PERF_TIMER_STOP(PROVE_step1); + PERF_TIMER_STOP_BP(PROVE_step1); PERF_TIMER_START_BP(PROVE_step2); // PAPER LINES 47-48 rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); - T1 = rct::scalarmultKey(T1, INV_EIGHT); - rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); - T2 = rct::scalarmultKey(T2, INV_EIGHT); + rct::key T1, T2; + ge_p3 p3; + sc_mul(tmp.bytes, t1.bytes, INV_EIGHT.bytes); + sc_mul(tmp2.bytes, tau1.bytes, INV_EIGHT.bytes); + ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); + ge_p3_tobytes(T1.bytes, &p3); + sc_mul(tmp.bytes, t2.bytes, INV_EIGHT.bytes); + sc_mul(tmp2.bytes, tau2.bytes, INV_EIGHT.bytes); + ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); + ge_p3_tobytes(T2.bytes, &p3); // PAPER LINES 49-51 rct::key x = hash_cache_mash(hash_cache, z, T1, T2); if (x == rct::zero()) { - PERF_TIMER_STOP(PROVE_step2); + PERF_TIMER_STOP_BP(PROVE_step2); MINFO("x is 0, trying again"); goto try_again; } @@ -889,7 +649,7 @@ try_again: l = vector_add(l, vector_scalar(l1, x)); rct::keyV r = r0; r = vector_add(r, vector_scalar(r1, x)); - PERF_TIMER_STOP(PROVE_step2); + PERF_TIMER_STOP_BP(PROVE_step2); PERF_TIMER_START_BP(PROVE_step3); rct::key t = inner_product(l, r); @@ -907,24 +667,27 @@ try_again: rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); if (x_ip == rct::zero()) { - PERF_TIMER_STOP(PROVE_step3); + PERF_TIMER_STOP_BP(PROVE_step3); MINFO("x_ip is 0, trying again"); goto try_again; } // These are used in the inner product rounds size_t nprime = MN; - rct::keyV Gprime(MN); - rct::keyV Hprime(MN); + std::vector<ge_p3> Gprime(MN); + std::vector<ge_p3> Hprime(MN); rct::keyV aprime(MN); rct::keyV bprime(MN); const rct::key yinv = invert(y); - rct::key yinvpow = rct::identity(); + rct::keyV yinvpow(MN); + yinvpow[0] = rct::identity(); + yinvpow[1] = yinv; for (size_t i = 0; i < MN; ++i) { - Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + Gprime[i] = Gi_p3[i]; + Hprime[i] = Hi_p3[i]; + if (i > 1) + sc_mul(yinvpow[i].bytes, yinvpow[i-1].bytes, yinv.bytes); aprime[i] = l[i]; bprime[i] = r[i]; } @@ -932,53 +695,62 @@ try_again: rct::keyV R(logMN); int round = 0; rct::keyV w(logMN); // this is the challenge x in the inner product protocol - PERF_TIMER_STOP(PROVE_step3); + PERF_TIMER_STOP_BP(PROVE_step3); PERF_TIMER_START_BP(PROVE_step4); // PAPER LINE 13 + const rct::keyV *scale = &yinvpow; while (nprime > 1) { // PAPER LINE 15 nprime /= 2; // PAPER LINES 16-17 + PERF_TIMER_START_BP(PROVE_inner_product); rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + PERF_TIMER_STOP_BP(PROVE_inner_product); // PAPER LINES 18-19 - L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + PERF_TIMER_START_BP(PROVE_LR); sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); - L[round] = rct::scalarmultKey(L[round], INV_EIGHT); - R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + L[round] = cross_vector_exponent8(nprime, Gprime, nprime, Hprime, 0, aprime, 0, bprime, nprime, scale, &ge_p3_H, &tmp); sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); - R[round] = rct::scalarmultKey(R[round], INV_EIGHT); + R[round] = cross_vector_exponent8(nprime, Gprime, 0, Hprime, nprime, aprime, nprime, bprime, 0, scale, &ge_p3_H, &tmp); + PERF_TIMER_STOP_BP(PROVE_LR); // PAPER LINES 21-22 w[round] = hash_cache_mash(hash_cache, L[round], R[round]); if (w[round] == rct::zero()) { - PERF_TIMER_STOP(PROVE_step4); + PERF_TIMER_STOP_BP(PROVE_step4); MINFO("w[round] is 0, trying again"); goto try_again; } // PAPER LINES 24-25 const rct::key winv = invert(w[round]); - Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); - Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); + if (nprime > 1) + { + PERF_TIMER_START_BP(PROVE_hadamard2); + hadamard_fold(Gprime, NULL, winv, w[round]); + hadamard_fold(Hprime, scale, w[round], winv); + PERF_TIMER_STOP_BP(PROVE_hadamard2); + } // PAPER LINES 28-29 + PERF_TIMER_START_BP(PROVE_prime); aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); + PERF_TIMER_STOP_BP(PROVE_prime); + scale = NULL; ++round; } - PERF_TIMER_STOP(PROVE_step4); + PERF_TIMER_STOP_BP(PROVE_step4); // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) - return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); + return Bulletproof(std::move(V), A, S, T1, T2, taux, mu, std::move(L), std::move(R), aprime[0], bprime[0], t); } Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma) @@ -1000,10 +772,17 @@ Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &g sv[i].bytes[6] = (v[i] >> 48) & 255; sv[i].bytes[7] = (v[i] >> 56) & 255; } - PERF_TIMER_STOP(PROVE_v); + PERF_TIMER_STOP_BP(PROVE_v); return bulletproof_PROVE(sv, gamma); } +struct proof_data_t +{ + rct::key x, y, z, x_ip; + std::vector<rct::key> w; + size_t logM, inv_offset; +}; + /* Given a range proof, determine if it is valid */ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) { @@ -1011,8 +790,17 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) PERF_TIMER_START_BP(VERIFY); + const size_t logN = 6; + const size_t N = 1 << logN; + // sanity and figure out which proof is longest size_t max_length = 0; + size_t nV = 0; + std::vector<proof_data_t> proof_data; + proof_data.reserve(proofs.size()); + size_t inv_offset = 0; + std::vector<rct::key> to_invert; + to_invert.reserve(11 * sizeof(proofs)); for (const Bulletproof *p: proofs) { const Bulletproof &proof = *p; @@ -1029,44 +817,76 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); max_length = std::max(max_length, proof.L.size()); + nV += proof.V.size(); + + // Reconstruct the challenges + PERF_TIMER_START_BP(VERIFY_start); + proof_data.resize(proof_data.size() + 1); + proof_data_t &pd = proof_data.back(); + rct::key hash_cache = rct::hash_to_scalar(proof.V); + pd.y = hash_cache_mash(hash_cache, proof.A, proof.S); + CHECK_AND_ASSERT_MES(!(pd.y == rct::zero()), false, "y == 0"); + pd.z = hash_cache = rct::hash_to_scalar(pd.y); + CHECK_AND_ASSERT_MES(!(pd.z == rct::zero()), false, "z == 0"); + pd.x = hash_cache_mash(hash_cache, pd.z, proof.T1, proof.T2); + CHECK_AND_ASSERT_MES(!(pd.x == rct::zero()), false, "x == 0"); + pd.x_ip = hash_cache_mash(hash_cache, pd.x, proof.taux, proof.mu, proof.t); + CHECK_AND_ASSERT_MES(!(pd.x_ip == rct::zero()), false, "x_ip == 0"); + PERF_TIMER_STOP_BP(VERIFY_start); + + size_t M; + for (pd.logM = 0; (M = 1<<pd.logM) <= maxM && M < proof.V.size(); ++pd.logM); + CHECK_AND_ASSERT_MES(proof.L.size() == 6+pd.logM, false, "Proof is not the expected size"); + + const size_t rounds = pd.logM+logN; + CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_21_22); + // PAPER LINES 21-22 + // The inner product challenges are computed per round + pd.w.resize(rounds); + for (size_t i = 0; i < rounds; ++i) + { + pd.w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + CHECK_AND_ASSERT_MES(!(pd.w[i] == rct::zero()), false, "w[i] == 0"); + } + PERF_TIMER_STOP_BP(VERIFY_line_21_22); + + pd.inv_offset = inv_offset; + for (size_t i = 0; i < rounds; ++i) + to_invert.push_back(pd.w[i]); + to_invert.push_back(pd.y); + inv_offset += rounds + 1; } CHECK_AND_ASSERT_MES(max_length < 32, false, "At least one proof is too large"); size_t maxMN = 1u << max_length; - const size_t logN = 6; - const size_t N = 1 << logN; rct::key tmp; + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(nV + (2 * (10/*logM*/ + logN) + 4) * proofs.size() + 2 * maxMN); + multiexp_data.resize(2 * maxMN); + + PERF_TIMER_START_BP(VERIFY_line_24_25_invert); + const std::vector<rct::key> inverses = invert(to_invert); + PERF_TIMER_STOP_BP(VERIFY_line_24_25_invert); + // setup weighted aggregates - rct::key Z0 = rct::identity(); rct::key z1 = rct::zero(); - rct::key Z2 = rct::identity(); rct::key z3 = rct::zero(); - rct::keyV z4(maxMN, rct::zero()), z5(maxMN, rct::zero()); - rct::key Y2 = rct::identity(), Y3 = rct::identity(), Y4 = rct::identity(); - rct::key y0 = rct::zero(), y1 = rct::zero(); + rct::keyV m_z4(maxMN, rct::zero()), m_z5(maxMN, rct::zero()); + rct::key m_y0 = rct::zero(), y1 = rct::zero(); + int proof_data_index = 0; for (const Bulletproof *p: proofs) { const Bulletproof &proof = *p; + const proof_data_t &pd = proof_data[proof_data_index++]; - size_t M, logM; - for (logM = 0; (M = 1<<logM) <= maxM && M < proof.V.size(); ++logM); - CHECK_AND_ASSERT_MES(proof.L.size() == 6+logM, false, "Proof is not the expected size"); + CHECK_AND_ASSERT_MES(proof.L.size() == 6+pd.logM, false, "Proof is not the expected size"); + const size_t M = 1 << pd.logM; const size_t MN = M*N; - rct::key weight = rct::skGen(); - - // Reconstruct the challenges - PERF_TIMER_START_BP(VERIFY_start); - rct::key hash_cache = rct::hash_to_scalar(proof.V); - rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); - CHECK_AND_ASSERT_MES(!(y == rct::zero()), false, "y == 0"); - rct::key z = hash_cache = rct::hash_to_scalar(y); - CHECK_AND_ASSERT_MES(!(z == rct::zero()), false, "z == 0"); - rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); - CHECK_AND_ASSERT_MES(!(x == rct::zero()), false, "x == 0"); - rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); - CHECK_AND_ASSERT_MES(!(x_ip == rct::zero()), false, "x_ip == 0"); - PERF_TIMER_STOP(VERIFY_start); + const rct::key weight_y = rct::skGen(); + const rct::key weight_z = rct::skGen(); // pre-multiply some points by 8 rct::keyV proof8_V = proof.V; for (rct::key &k: proof8_V) k = rct::scalarmult8(k); @@ -1075,177 +895,161 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) rct::key proof8_T1 = rct::scalarmult8(proof.T1); rct::key proof8_T2 = rct::scalarmult8(proof.T2); rct::key proof8_S = rct::scalarmult8(proof.S); + rct::key proof8_A = rct::scalarmult8(proof.A); PERF_TIMER_START_BP(VERIFY_line_61); // PAPER LINE 61 - sc_muladd(y0.bytes, proof.taux.bytes, weight.bytes, y0.bytes); + sc_mulsub(m_y0.bytes, proof.taux.bytes, weight_y.bytes, m_y0.bytes); - const rct::keyV zpow = vector_powers(z, M+3); + const rct::keyV zpow = vector_powers(pd.z, M+3); rct::key k; - const rct::key ip1y = vector_power_sum(y, MN); + const rct::key ip1y = vector_power_sum(pd.y, MN); sc_mulsub(k.bytes, zpow[2].bytes, ip1y.bytes, rct::zero().bytes); for (size_t j = 1; j <= M; ++j) { CHECK_AND_ASSERT_MES(j+2 < zpow.size(), false, "invalid zpow index"); sc_mulsub(k.bytes, zpow[j+2].bytes, ip12.bytes, k.bytes); } - PERF_TIMER_STOP(VERIFY_line_61); + PERF_TIMER_STOP_BP(VERIFY_line_61); PERF_TIMER_START_BP(VERIFY_line_61rl_new); - sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); - std::vector<MultiexpData> multiexp_data; - multiexp_data.reserve(proof.V.size()); + sc_muladd(tmp.bytes, pd.z.bytes, ip1y.bytes, k.bytes); sc_sub(tmp.bytes, proof.t.bytes, tmp.bytes); - sc_muladd(y1.bytes, tmp.bytes, weight.bytes, y1.bytes); + sc_muladd(y1.bytes, tmp.bytes, weight_y.bytes, y1.bytes); for (size_t j = 0; j < proof8_V.size(); j++) { - multiexp_data.emplace_back(zpow[j+2], proof8_V[j]); + sc_mul(tmp.bytes, zpow[j+2].bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_V[j]); } - rct::addKeys(Y2, Y2, rct::scalarmultKey(multiexp(multiexp_data, false), weight)); - sc_mul(tmp.bytes, x.bytes, weight.bytes); - rct::addKeys(Y3, Y3, rct::scalarmultKey(proof8_T1, tmp)); + sc_mul(tmp.bytes, pd.x.bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_T1); rct::key xsq; - sc_mul(xsq.bytes, x.bytes, x.bytes); - sc_mul(tmp.bytes, xsq.bytes, weight.bytes); - rct::addKeys(Y4, Y4, rct::scalarmultKey(proof8_T2, tmp)); - PERF_TIMER_STOP(VERIFY_line_61rl_new); + sc_mul(xsq.bytes, pd.x.bytes, pd.x.bytes); + sc_mul(tmp.bytes, xsq.bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_T2); + PERF_TIMER_STOP_BP(VERIFY_line_61rl_new); PERF_TIMER_START_BP(VERIFY_line_62); // PAPER LINE 62 - rct::addKeys(Z0, Z0, rct::scalarmultKey(rct::addKeys(rct::scalarmult8(proof.A), rct::scalarmultKey(proof8_S, x)), weight)); - PERF_TIMER_STOP(VERIFY_line_62); + multiexp_data.emplace_back(weight_z, proof8_A); + sc_mul(tmp.bytes, pd.x.bytes, weight_z.bytes); + multiexp_data.emplace_back(tmp, proof8_S); + PERF_TIMER_STOP_BP(VERIFY_line_62); // Compute the number of rounds for the inner product - const size_t rounds = logM+logN; + const size_t rounds = pd.logM+logN; CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); - PERF_TIMER_START_BP(VERIFY_line_21_22); - // PAPER LINES 21-22 - // The inner product challenges are computed per round - rct::keyV w(rounds); - for (size_t i = 0; i < rounds; ++i) - { - w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); - CHECK_AND_ASSERT_MES(!(w[i] == rct::zero()), false, "w[i] == 0"); - } - PERF_TIMER_STOP(VERIFY_line_21_22); - PERF_TIMER_START_BP(VERIFY_line_24_25); // Basically PAPER LINES 24-25 // Compute the curvepoints from G[i] and H[i] rct::key yinvpow = rct::identity(); rct::key ypow = rct::identity(); - PERF_TIMER_START_BP(VERIFY_line_24_25_invert); - const rct::key yinv = invert(y); - rct::keyV winv(rounds); - for (size_t i = 0; i < rounds; ++i) - winv[i] = invert(w[i]); - PERF_TIMER_STOP(VERIFY_line_24_25_invert); + const rct::key *winv = &inverses[pd.inv_offset]; + const rct::key yinv = inverses[pd.inv_offset + rounds]; + + // precalc + PERF_TIMER_START_BP(VERIFY_line_24_25_precalc); + rct::keyV w_cache(1<<rounds); + w_cache[0] = winv[0]; + w_cache[1] = pd.w[0]; + for (size_t j = 1; j < rounds; ++j) + { + const size_t slots = 1<<(j+1); + for (size_t s = slots; s-- > 0; --s) + { + sc_mul(w_cache[s].bytes, w_cache[s/2].bytes, pd.w[j].bytes); + sc_mul(w_cache[s-1].bytes, w_cache[s/2].bytes, winv[j].bytes); + } + } + PERF_TIMER_STOP_BP(VERIFY_line_24_25_precalc); for (size_t i = 0; i < MN; ++i) { - // Convert the index to binary IN REVERSE and construct the scalar exponent rct::key g_scalar = proof.a; rct::key h_scalar; - sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); + if (i == 0) + h_scalar = proof.b; + else + sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); - for (size_t j = rounds; j-- > 0; ) - { - size_t J = w.size() - j - 1; - - if ((i & (((size_t)1)<<j)) == 0) - { - sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); - } - else - { - sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); - } - } + // Convert the index to binary IN REVERSE and construct the scalar exponent + sc_mul(g_scalar.bytes, g_scalar.bytes, w_cache[i].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, w_cache[(~i) & (MN-1)].bytes); // Adjust the scalars using the exponents from PAPER LINE 62 - sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); + sc_add(g_scalar.bytes, g_scalar.bytes, pd.z.bytes); CHECK_AND_ASSERT_MES(2+i/N < zpow.size(), false, "invalid zpow index"); CHECK_AND_ASSERT_MES(i%N < twoN.size(), false, "invalid twoN index"); sc_mul(tmp.bytes, zpow[2+i/N].bytes, twoN[i%N].bytes); - sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); - sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + if (i == 0) + { + sc_add(tmp.bytes, tmp.bytes, pd.z.bytes); + sc_sub(h_scalar.bytes, h_scalar.bytes, tmp.bytes); + } + else + { + sc_muladd(tmp.bytes, pd.z.bytes, ypow.bytes, tmp.bytes); + sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + } - sc_muladd(z4[i].bytes, g_scalar.bytes, weight.bytes, z4[i].bytes); - sc_muladd(z5[i].bytes, h_scalar.bytes, weight.bytes, z5[i].bytes); + sc_mulsub(m_z4[i].bytes, g_scalar.bytes, weight_z.bytes, m_z4[i].bytes); + sc_mulsub(m_z5[i].bytes, h_scalar.bytes, weight_z.bytes, m_z5[i].bytes); - if (i != MN-1) + if (i == 0) + { + yinvpow = yinv; + ypow = pd.y; + } + else if (i != MN-1) { sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); - sc_mul(ypow.bytes, ypow.bytes, y.bytes); + sc_mul(ypow.bytes, ypow.bytes, pd.y.bytes); } } - PERF_TIMER_STOP(VERIFY_line_24_25); + PERF_TIMER_STOP_BP(VERIFY_line_24_25); // PAPER LINE 26 PERF_TIMER_START_BP(VERIFY_line_26_new); - multiexp_data.clear(); - multiexp_data.reserve(2*rounds); - - sc_muladd(z1.bytes, proof.mu.bytes, weight.bytes, z1.bytes); + sc_muladd(z1.bytes, proof.mu.bytes, weight_z.bytes, z1.bytes); for (size_t i = 0; i < rounds; ++i) { - sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); + sc_mul(tmp.bytes, pd.w[i].bytes, pd.w[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, weight_z.bytes); multiexp_data.emplace_back(tmp, proof8_L[i]); sc_mul(tmp.bytes, winv[i].bytes, winv[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, weight_z.bytes); multiexp_data.emplace_back(tmp, proof8_R[i]); } - rct::key acc = multiexp(multiexp_data, false); - rct::addKeys(Z2, Z2, rct::scalarmultKey(acc, weight)); sc_mulsub(tmp.bytes, proof.a.bytes, proof.b.bytes, proof.t.bytes); - sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); - sc_muladd(z3.bytes, tmp.bytes, weight.bytes, z3.bytes); - PERF_TIMER_STOP(VERIFY_line_26_new); + sc_mul(tmp.bytes, tmp.bytes, pd.x_ip.bytes); + sc_muladd(z3.bytes, tmp.bytes, weight_z.bytes, z3.bytes); + PERF_TIMER_STOP_BP(VERIFY_line_26_new); } // now check all proofs at once PERF_TIMER_START_BP(VERIFY_step2_check); - ge_p3 check1; - ge_scalarmult_base(&check1, y0.bytes); - addKeys_acc_p3(&check1, y1, rct::H); - sub_acc_p3(&check1, Y2); - sub_acc_p3(&check1, Y3); - sub_acc_p3(&check1, Y4); - if (!ge_p3_is_point_at_infinity(&check1)) - { - MERROR("Verification failure at step 1"); - return false; - } - ge_p3 check2; - sc_sub(tmp.bytes, rct::zero().bytes, z1.bytes); - ge_double_scalarmult_base_vartime_p3(&check2, z3.bytes, &ge_p3_H, tmp.bytes); - add_acc_p3(&check2, Z0); - add_acc_p3(&check2, Z2); - - std::vector<MultiexpData> multiexp_data; - multiexp_data.reserve(2 * maxMN); + sc_sub(tmp.bytes, m_y0.bytes, z1.bytes); + multiexp_data.emplace_back(tmp, rct::G); + sc_sub(tmp.bytes, z3.bytes, y1.bytes); + multiexp_data.emplace_back(tmp, rct::H); for (size_t i = 0; i < maxMN; ++i) { - sc_sub(tmp.bytes, rct::zero().bytes, z4[i].bytes); - multiexp_data.emplace_back(tmp, Gi_p3[i]); - sc_sub(tmp.bytes, rct::zero().bytes, z5[i].bytes); - multiexp_data.emplace_back(tmp, Hi_p3[i]); + multiexp_data[i * 2] = {m_z4[i], Gi_p3[i]}; + multiexp_data[i * 2 + 1] = {m_z5[i], Hi_p3[i]}; } - add_acc_p3(&check2, multiexp(multiexp_data, true)); - PERF_TIMER_STOP(VERIFY_step2_check); - - if (!ge_p3_is_point_at_infinity(&check2)) + if (!(multiexp(multiexp_data, 2 * maxMN) == rct::identity())) { - MERROR("Verification failure at step 2"); + PERF_TIMER_STOP_BP(VERIFY_step2_check); + MERROR("Verification failure"); return false; } + PERF_TIMER_STOP_BP(VERIFY_step2_check); - PERF_TIMER_STOP(VERIFY); + PERF_TIMER_STOP_BP(VERIFY); return true; } diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index 21957b94c..6f77fed34 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -79,6 +79,25 @@ extern "C" // Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip // Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip +// New timings: +// Pippenger: +// 2/1 always +// 3/2 at ~13 +// 4/3 at ~29 +// 5/4 at ~83 +// 6/5 < 200 +// 7/6 at ~470 +// 8/7 at ~1180 +// 9/8 at ~2290 +// Cached Pippenger: +// 6/5 < 200 +// 7/6 at 460 +// 8/7 at 1180 +// 9/8 at 2300 +// +// Cached Straus/Pippenger cross at 232 +// + namespace rct { @@ -320,7 +339,7 @@ rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data) return res; } -static constexpr unsigned int STRAUS_C = 4; +#define STRAUS_C 4 struct straus_cached_data { @@ -447,28 +466,26 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str #endif MULTIEXP_PERF(PERF_TIMER_START_UNIT(digits, 1000000)); +#if STRAUS_C==4 + std::unique_ptr<uint8_t[]> digits{new uint8_t[64 * data.size()]}; +#else std::unique_ptr<uint8_t[]> digits{new uint8_t[256 * data.size()]}; +#endif for (size_t j = 0; j < data.size(); ++j) { - unsigned char bytes33[33]; - memcpy(bytes33, data[j].scalar.bytes, 32); - bytes33[32] = 0; - const unsigned char *bytes = bytes33; -#if 1 - static_assert(STRAUS_C == 4, "optimized version needs STRAUS_C == 4"); + const unsigned char *bytes = data[j].scalar.bytes; +#if STRAUS_C==4 unsigned int i; - for (i = 0; i < 256; i += 8, bytes++) + for (i = 0; i < 64; i += 2, bytes++) { - digits[j*256+i] = bytes[0] & 0xf; - digits[j*256+i+1] = (bytes[0] >> 1) & 0xf; - digits[j*256+i+2] = (bytes[0] >> 2) & 0xf; - digits[j*256+i+3] = (bytes[0] >> 3) & 0xf; - digits[j*256+i+4] = ((bytes[0] >> 4) | (bytes[1]<<4)) & 0xf; - digits[j*256+i+5] = ((bytes[0] >> 5) | (bytes[1]<<3)) & 0xf; - digits[j*256+i+6] = ((bytes[0] >> 6) | (bytes[1]<<2)) & 0xf; - digits[j*256+i+7] = ((bytes[0] >> 7) | (bytes[1]<<1)) & 0xf; + digits[j*64+i] = bytes[0] & 0xf; + digits[j*64+i+1] = bytes[0] >> 4; } #elif 1 + unsigned char bytes33[33]; + memcpy(bytes33, data[j].scalar.bytes, 32); + bytes33[32] = 0; + bytes = bytes33; for (size_t i = 0; i < 256; ++i) digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; #else @@ -521,7 +538,11 @@ skipfirst: if (skip[j]) continue; #endif +#if STRAUS_C==4 + const uint8_t digit = digits[j*64+i/4]; +#else const uint8_t digit = digits[j*256+i]; +#endif if (digit) { ge_add(&p1, &band_p3, &CACHE_OFFSET(local_cache, j, digit)); @@ -542,16 +563,13 @@ skipfirst: size_t get_pippenger_c(size_t N) { -// uncached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 -// cached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 - if (N <= 2) return 1; - if (N <= 8) return 2; - if (N <= 16) return 3; - if (N <= 64) return 4; - if (N <= 128) return 5; - if (N <= 256) return 6; - if (N <= 1024) return 7; - if (N <= 2048) return 8; + if (N <= 13) return 2; + if (N <= 29) return 3; + if (N <= 83) return 4; + if (N <= 185) return 5; + if (N <= 465) return 6; + if (N <= 1180) return 7; + if (N <= 2295) return 8; return 9; } @@ -563,12 +581,13 @@ struct pippenger_cached_data ~pippenger_cached_data() { aligned_free(cached); } }; -std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N) +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t start_offset, size_t N) { MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); + CHECK_AND_ASSERT_THROW_MES(start_offset <= data.size(), "Bad cache base data"); if (N == 0) - N = data.size(); - CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + N = data.size() - start_offset; + CHECK_AND_ASSERT_THROW_MES(N <= data.size() - start_offset, "Bad cache base data"); ge_cached cached; std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); @@ -576,7 +595,7 @@ std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<Mu cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); for (size_t i = 0; i < N; ++i) - ge_p3_to_cached(&cache->cached[i], &data[i].point); + ge_p3_to_cached(&cache->cached[i], &data[i+start_offset].point); MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); return cache; @@ -587,16 +606,21 @@ size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &ca return cache->size * sizeof(*cache->cached); } -rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t c) +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t cache_size, size_t c) { - CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + if (cache != NULL && cache_size == 0) + cache_size = cache->size; + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache_size <= cache->size, "Cache is too small"); if (c == 0) c = get_pippenger_c(data.size()); CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); ge_p3 result = ge_p3_identity; + bool result_init = false; std::unique_ptr<ge_p3[]> buckets{new ge_p3[1<<c]}; + bool buckets_init[1<<9]; std::shared_ptr<pippenger_cached_data> local_cache = cache == NULL ? pippenger_init_cache(data) : cache; + std::shared_ptr<pippenger_cached_data> local_cache_2 = data.size() > cache_size ? pippenger_init_cache(data, cache_size) : NULL; rct::key maxscalar = rct::zero(); for (size_t i = 0; i < data.size(); ++i) @@ -611,7 +635,7 @@ rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr< for (size_t k = groups; k-- > 0; ) { - if (!ge_p3_is_point_at_infinity(&result)) + if (result_init) { ge_p2 p2; ge_p3_to_p2(&p2, &result); @@ -625,8 +649,7 @@ rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr< ge_p1p1_to_p2(&p2, &p1); } } - for (size_t i = 0; i < (1u<<c); ++i) - buckets[i] = ge_p3_identity; + memset(buckets_init, 0, 1u<<c); // partition scalars into buckets for (size_t i = 0; i < data.size(); ++i) @@ -638,22 +661,45 @@ rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr< if (bucket == 0) continue; CHECK_AND_ASSERT_THROW_MES(bucket < (1u<<c), "bucket overflow"); - if (!ge_p3_is_point_at_infinity(&buckets[bucket])) + if (buckets_init[bucket]) { - add(buckets[bucket], local_cache->cached[i]); + if (i < cache_size) + add(buckets[bucket], local_cache->cached[i]); + else + add(buckets[bucket], local_cache_2->cached[i - cache_size]); } else + { buckets[bucket] = data[i].point; + buckets_init[bucket] = true; + } } // sum the buckets - ge_p3 pail = ge_p3_identity; + ge_p3 pail; + bool pail_init = false; for (size_t i = (1<<c)-1; i > 0; --i) { - if (!ge_p3_is_point_at_infinity(&buckets[i])) - add(pail, buckets[i]); - if (!ge_p3_is_point_at_infinity(&pail)) - add(result, pail); + if (buckets_init[i]) + { + if (pail_init) + add(pail, buckets[i]); + else + { + pail = buckets[i]; + pail_init = true; + } + } + if (pail_init) + { + if (result_init) + add(result, pail); + else + { + result = pail; + result_init = true; + } + } } } diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h index 559ab664a..b52707933 100644 --- a/src/ringct/multiexp.h +++ b/src/ringct/multiexp.h @@ -61,10 +61,10 @@ rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data); std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N =0); size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache); rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache = NULL, size_t STEP = 0); -std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t start_offset = 0, size_t N =0); size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache); size_t get_pippenger_c(size_t N); -rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t c = 0); +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t cache_size = 0, size_t c = 0); } diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 181e89c45..dccd18867 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -58,15 +58,6 @@ namespace } namespace rct { - Bulletproof proveRangeBulletproof(key &C, key &mask, uint64_t amount) - { - mask = rct::skGen(); - Bulletproof proof = bulletproof_PROVE(amount, mask); - CHECK_AND_ASSERT_THROW_MES(proof.V.size() == 1, "V has not exactly one element"); - C = proof.V[0]; - return proof; - } - Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts) { masks = rct::skvGen(amounts.size()); diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index ae8bb91d7..b67a0b992 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -133,7 +133,7 @@ namespace rct { xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); - + key get_pre_mlsag_hash(const rctSig &rv, hw::device &hwdev); bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key); } #endif /* RCTSIGS_H */ diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 4f8f96524..8fc42b7e3 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -31,6 +31,7 @@ set(rpc_base_sources set(rpc_sources core_rpc_server.cpp + rpc_handler.cpp instanciations) set(daemon_messages_sources @@ -45,7 +46,8 @@ set(daemon_rpc_server_sources set(rpc_base_headers rpc_args.h) -set(rpc_headers) +set(rpc_headers + rpc_handler.cpp) set(daemon_rpc_server_headers) @@ -63,7 +65,6 @@ set(daemon_rpc_server_private_headers message.h daemon_messages.h daemon_handler.h - rpc_handler.h zmq_server.h) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index aa9d3d64b..c2e71bef8 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -45,6 +45,7 @@ using namespace epee; #include "storages/http_abstract_invoke.h" #include "crypto/hash.h" #include "rpc/rpc_args.h" +#include "rpc/rpc_handler.h" #include "core_rpc_server_error_codes.h" #include "p2p/net_node.h" #include "version.h" @@ -700,7 +701,7 @@ namespace cryptonote res.status = "Failed"; res.reason = ""; if ((res.low_mixin = tvc.m_low_mixin)) - add_reason(res.reason, "ring size too small"); + add_reason(res.reason, "bad ring size"); if ((res.double_spend = tvc.m_double_spend)) add_reason(res.reason, "double spend"); if ((res.invalid_input = tvc.m_invalid_input)) @@ -801,7 +802,14 @@ namespace cryptonote bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res) { PERF_TIMER(on_stop_mining); - if(!m_core.get_miner().stop()) + cryptonote::miner &miner= m_core.get_miner(); + if(!miner.is_mining()) + { + res.status = "Mining never started"; + LOG_PRINT_L0(res.status); + return true; + } + if(!miner.stop()) { res.status = "Failed, mining not stopped"; LOG_PRINT_L0(res.status); @@ -1010,7 +1018,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= h) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(h) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(h) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); } res = string_tools::pod_to_hex(m_core.get_block_id_by_height(h)); return true; @@ -1457,7 +1465,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(req.height) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } crypto::hash block_hash = m_core.get_block_id_by_height(req.height); @@ -1502,7 +1510,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(req.height) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } block_hash = m_core.get_block_id_by_height(req.height); @@ -2117,62 +2125,15 @@ namespace cryptonote const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (uint64_t amount: req.amounts) { - static struct D - { - boost::mutex mutex; - std::vector<uint64_t> cached_distribution; - uint64_t cached_from, cached_to, cached_start_height, cached_base; - bool cached; - D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} - } d; - boost::unique_lock<boost::mutex> lock(d.mutex); - - if (d.cached && amount == 0 && d.cached_from == req.from_height && d.cached_to == req_to_height) - { - res.distributions.push_back({amount, d.cached_start_height, req.binary, d.cached_distribution, d.cached_base}); - if (!req.cumulative) - { - auto &distribution = res.distributions.back().distribution; - for (size_t n = distribution.size() - 1; n > 0; --n) - distribution[n] -= distribution[n-1]; - distribution[0] -= d.cached_base; - } - continue; - } - - std::vector<uint64_t> distribution; - uint64_t start_height, base; - if (!m_core.get_output_distribution(amount, req.from_height, req_to_height, start_height, distribution, base)) + auto data = rpc::RpcHandler::get_output_distribution(m_core, amount, req.from_height, req_to_height, req.cumulative); + if (!data) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Failed to get rct distribution"; + error_resp.message = "Failed to get output distribution"; return false; } - if (req_to_height > 0 && req_to_height >= req.from_height) - { - uint64_t offset = std::max(req.from_height, start_height); - if (offset <= req_to_height && req_to_height - offset + 1 < distribution.size()) - distribution.resize(req_to_height - offset + 1); - } - - if (amount == 0) - { - d.cached_from = req.from_height; - d.cached_to = req_to_height; - d.cached_distribution = distribution; - d.cached_start_height = start_height; - d.cached_base = base; - d.cached = true; - } - - if (!req.cumulative) - { - for (size_t n = distribution.size() - 1; n > 0; --n) - distribution[n] -= distribution[n-1]; - distribution[0] -= base; - } - res.distributions.push_back({amount, start_height, req.binary, std::move(distribution), base}); + res.distributions.push_back({std::move(*data), amount, req.binary}); } } catch (const std::exception &e) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index b229841d6..8e8df7a52 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -33,6 +33,7 @@ #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/difficulty.h" #include "crypto/hash.h" +#include "rpc/rpc_handler.h" namespace cryptonote { @@ -2233,21 +2234,19 @@ namespace cryptonote struct distribution { + rpc::output_distribution_data data; uint64_t amount; - uint64_t start_height; bool binary; - std::vector<uint64_t> distribution; - uint64_t base; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) - KV_SERIALIZE(start_height) + KV_SERIALIZE_N(data.start_height, "start_height") KV_SERIALIZE(binary) if (this_ref.binary) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") else - KV_SERIALIZE(distribution) - KV_SERIALIZE(base) + KV_SERIALIZE_N(data.distribution, "distribution") + KV_SERIALIZE_N(data.base, "base") END_KV_SERIALIZE_MAP() }; diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 9d3b09b68..8822bd378 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -724,12 +724,53 @@ namespace rpc res.status = Message::STATUS_OK; } - void DaemonHandler::handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res) + void DaemonHandler::handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res) { - res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); + res.hard_fork_version = m_core.get_blockchain_storage().get_current_hard_fork_version(); + res.estimated_base_fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); + + if (res.hard_fork_version < HF_VERSION_PER_BYTE_FEE) + { + res.size_scale = 1024; // per KiB fee + res.fee_mask = 1; + } + else + { + res.size_scale = 1; // per byte fee + res.fee_mask = Blockchain::get_fee_quantization_mask(); + } res.status = Message::STATUS_OK; } + void DaemonHandler::handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res) + { + try + { + res.distributions.reserve(req.amounts.size()); + + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); + for (std::uint64_t amount : req.amounts) + { + auto data = get_output_distribution(m_core, amount, req.from_height, req_to_height, req.cumulative); + if (!data) + { + res.distributions.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = "Failed to get output distribution"; + return; + } + res.distributions.push_back(output_distribution{std::move(*data), amount, req.cumulative}); + } + res.status = Message::STATUS_OK; + } + catch (const std::exception& e) + { + res.distributions.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = e.what(); + } + } + bool DaemonHandler::getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& header) { block b; @@ -804,7 +845,8 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetOutputHistogram, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetOutputKeys, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetRPCVersion, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetPerKBFeeEstimate, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, GetFeeEstimate, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, GetOutputDistribution, req_json, resp_message, handle); // if none of the request types matches if (resp_message == NULL) diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index 5f9687511..2c8ac3867 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -126,7 +126,9 @@ class DaemonHandler : public RpcHandler void handle(const GetRPCVersion::Request& req, GetRPCVersion::Response& res); - void handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res); + void handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res); + + void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res); std::string handle(const std::string& request); diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index 56f6f6a8c..fa848cff4 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -59,7 +59,8 @@ const char* const HardForkInfo::name = "hard_fork_info"; const char* const GetOutputHistogram::name = "get_output_histogram"; const char* const GetOutputKeys::name = "get_output_keys"; const char* const GetRPCVersion::name = "get_rpc_version"; -const char* const GetPerKBFeeEstimate::name = "get_dynamic_per_kb_fee_estimate"; +const char* const GetFeeEstimate::name = "get_dynamic_fee_estimate"; +const char* const GetOutputDistribution::name = "get_output_distribution"; @@ -825,7 +826,7 @@ void GetRPCVersion::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, version, version); } -rapidjson::Value GetPerKBFeeEstimate::Request::toJson(rapidjson::Document& doc) const +rapidjson::Value GetFeeEstimate::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); @@ -836,27 +837,70 @@ rapidjson::Value GetPerKBFeeEstimate::Request::toJson(rapidjson::Document& doc) return val; } -void GetPerKBFeeEstimate::Request::fromJson(rapidjson::Value& val) +void GetFeeEstimate::Request::fromJson(rapidjson::Value& val) { GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks); } -rapidjson::Value GetPerKBFeeEstimate::Response::toJson(rapidjson::Document& doc) const +rapidjson::Value GetFeeEstimate::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, estimated_fee_per_kb, estimated_fee_per_kb); + INSERT_INTO_JSON_OBJECT(val, doc, estimated_base_fee, estimated_base_fee); + INSERT_INTO_JSON_OBJECT(val, doc, fee_mask, fee_mask); + INSERT_INTO_JSON_OBJECT(val, doc, size_scale, size_scale); + INSERT_INTO_JSON_OBJECT(val, doc, hard_fork_version, hard_fork_version); return val; } -void GetPerKBFeeEstimate::Response::fromJson(rapidjson::Value& val) +void GetFeeEstimate::Response::fromJson(rapidjson::Value& val) { - GET_FROM_JSON_OBJECT(val, estimated_fee_per_kb, estimated_fee_per_kb); + GET_FROM_JSON_OBJECT(val, estimated_base_fee, estimated_base_fee); + GET_FROM_JSON_OBJECT(val, fee_mask, fee_mask); + GET_FROM_JSON_OBJECT(val, size_scale, size_scale); + GET_FROM_JSON_OBJECT(val, hard_fork_version, hard_fork_version); } +rapidjson::Value GetOutputDistribution::Request::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + auto& al = doc.GetAllocator(); + + INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); + INSERT_INTO_JSON_OBJECT(val, doc, from_height, from_height); + INSERT_INTO_JSON_OBJECT(val, doc, to_height, to_height); + INSERT_INTO_JSON_OBJECT(val, doc, cumulative, cumulative); + + return val; +} + +void GetOutputDistribution::Request::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, amounts, amounts); + GET_FROM_JSON_OBJECT(val, from_height, from_height); + GET_FROM_JSON_OBJECT(val, to_height, to_height); + GET_FROM_JSON_OBJECT(val, cumulative, cumulative); +} + +rapidjson::Value GetOutputDistribution::Response::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + auto& al = doc.GetAllocator(); + + INSERT_INTO_JSON_OBJECT(val, doc, status, status); + INSERT_INTO_JSON_OBJECT(val, doc, distributions, distributions); + + return val; +} + +void GetOutputDistribution::Response::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, status, status); + GET_FROM_JSON_OBJECT(val, distributions, distributions); +} } // namespace rpc diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index 8488bab0b..d2014247c 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -28,6 +28,9 @@ #pragma once +#include <unordered_map> +#include <vector> + #include "message.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" #include "rpc/message_data_structs.h" @@ -62,8 +65,6 @@ class classname \ #define END_RPC_MESSAGE_RESPONSE }; #define END_RPC_MESSAGE_CLASS }; -#define COMMA() , - // NOTE: when using a type with multiple template parameters, // replace any comma in the template specifier with the macro // above, or the preprocessor will eat the comma in a bad way. @@ -118,7 +119,8 @@ BEGIN_RPC_MESSAGE_CLASS(GetTransactions); RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, tx_hashes); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::unordered_map<crypto::hash COMMA() cryptonote::rpc::transaction_info>, txs); + using txes_map = std::unordered_map<crypto::hash, transaction_info>; + RPC_MESSAGE_MEMBER(txes_map, txs); RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, missed_hashes); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; @@ -407,12 +409,27 @@ BEGIN_RPC_MESSAGE_CLASS(GetRPCVersion); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; -BEGIN_RPC_MESSAGE_CLASS(GetPerKBFeeEstimate); +BEGIN_RPC_MESSAGE_CLASS(GetFeeEstimate); BEGIN_RPC_MESSAGE_REQUEST; RPC_MESSAGE_MEMBER(uint64_t, num_grace_blocks); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(uint64_t, estimated_fee_per_kb); + RPC_MESSAGE_MEMBER(uint64_t, estimated_base_fee); + RPC_MESSAGE_MEMBER(uint64_t, fee_mask); + RPC_MESSAGE_MEMBER(uint32_t, size_scale); + RPC_MESSAGE_MEMBER(uint8_t, hard_fork_version); + END_RPC_MESSAGE_RESPONSE; +END_RPC_MESSAGE_CLASS; + +BEGIN_RPC_MESSAGE_CLASS(GetOutputDistribution); + BEGIN_RPC_MESSAGE_REQUEST; + RPC_MESSAGE_MEMBER(std::vector<uint64_t>, amounts); + RPC_MESSAGE_MEMBER(uint64_t, from_height); + RPC_MESSAGE_MEMBER(uint64_t, to_height); + RPC_MESSAGE_MEMBER(bool, cumulative); + END_RPC_MESSAGE_REQUEST; + BEGIN_RPC_MESSAGE_RESPONSE; + RPC_MESSAGE_MEMBER(std::vector<output_distribution>, distributions); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index cf15ade1c..3b56aff15 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -31,6 +31,7 @@ #include "crypto/hash.h" #include "cryptonote_basic/cryptonote_basic.h" #include "ringct/rctSigs.h" +#include "rpc/rpc_handler.h" #include <unordered_map> #include <vector> @@ -192,6 +193,12 @@ namespace rpc uint64_t start_time; }; + struct output_distribution + { + output_distribution_data data; + uint64_t amount; + bool cumulative; + }; } // namespace rpc } // namespace cryptonote diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp new file mode 100644 index 000000000..d4beb1928 --- /dev/null +++ b/src/rpc/rpc_handler.cpp @@ -0,0 +1,69 @@ + +#include <algorithm> +#include <boost/thread/locks.hpp> +#include <boost/thread/mutex.hpp> + +#include "cryptonote_core/cryptonote_core.h" + +namespace cryptonote +{ +namespace rpc +{ + namespace + { + output_distribution_data + process_distribution(bool cumulative, std::uint64_t start_height, std::vector<std::uint64_t> distribution, std::uint64_t base) + { + if (!cumulative && !distribution.empty()) + { + for (std::size_t n = distribution.size() - 1; 0 < n; --n) + distribution[n] -= distribution[n - 1]; + distribution[0] -= base; + } + + return {std::move(distribution), start_height, base}; + } + } + + boost::optional<output_distribution_data> + RpcHandler::get_output_distribution(core& src, std::uint64_t amount, std::uint64_t from_height, std::uint64_t to_height, bool cumulative) + { + static struct D + { + boost::mutex mutex; + std::vector<std::uint64_t> cached_distribution; + std::uint64_t cached_from, cached_to, cached_start_height, cached_base; + bool cached; + D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} + } d; + const boost::unique_lock<boost::mutex> lock(d.mutex); + + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height) + return process_distribution(cumulative, d.cached_start_height, d.cached_distribution, d.cached_base); + + std::vector<std::uint64_t> distribution; + std::uint64_t start_height, base; + if (!src.get_output_distribution(amount, from_height, to_height, start_height, distribution, base)) + return boost::none; + + if (to_height > 0 && to_height >= from_height) + { + const std::uint64_t offset = std::max(from_height, start_height); + if (offset <= to_height && to_height - offset + 1 < distribution.size()) + distribution.resize(to_height - offset + 1); + } + + if (amount == 0) + { + d.cached_from = from_height; + d.cached_to = to_height; + d.cached_distribution = distribution; + d.cached_start_height = start_height; + d.cached_base = base; + d.cached = true; + } + + return process_distribution(cumulative, start_height, std::move(distribution), base); + } +} // rpc +} // cryptonote diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index 64bade5a8..3cccef78a 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -28,24 +28,35 @@ #pragma once +#include <boost/optional/optional.hpp> +#include <cstdint> #include <string> +#include <vector> namespace cryptonote { +class core; namespace rpc { +struct output_distribution_data +{ + std::vector<std::uint64_t> distribution; + std::uint64_t start_height; + std::uint64_t base; +}; class RpcHandler { public: + RpcHandler() { } + virtual ~RpcHandler() { } virtual std::string handle(const std::string& request) = 0; - RpcHandler() { } - - virtual ~RpcHandler() { } + static boost::optional<output_distribution_data> + get_output_distribution(core& src, std::uint64_t amount, std::uint64_t from_height, std::uint64_t to_height, bool cumulative); }; diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 89a1dbd23..8b1af9c12 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -1192,7 +1192,9 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& in INSERT_INTO_JSON_OBJECT(val, doc, incoming_connections_count, info.incoming_connections_count); INSERT_INTO_JSON_OBJECT(val, doc, white_peerlist_size, info.white_peerlist_size); INSERT_INTO_JSON_OBJECT(val, doc, grey_peerlist_size, info.grey_peerlist_size); + INSERT_INTO_JSON_OBJECT(val, doc, mainnet, info.mainnet); INSERT_INTO_JSON_OBJECT(val, doc, testnet, info.testnet); + INSERT_INTO_JSON_OBJECT(val, doc, stagenet, info.stagenet); INSERT_INTO_JSON_OBJECT(val, doc, nettype, info.nettype); INSERT_INTO_JSON_OBJECT(val, doc, top_block_hash, info.top_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, cumulative_difficulty, info.cumulative_difficulty); @@ -1221,7 +1223,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.incoming_connections_count, incoming_connections_count); GET_FROM_JSON_OBJECT(val, info.white_peerlist_size, white_peerlist_size); GET_FROM_JSON_OBJECT(val, info.grey_peerlist_size, grey_peerlist_size); + GET_FROM_JSON_OBJECT(val, info.mainnet, mainnet); GET_FROM_JSON_OBJECT(val, info.testnet, testnet); + GET_FROM_JSON_OBJECT(val, info.stagenet, stagenet); GET_FROM_JSON_OBJECT(val, info.nettype, nettype); GET_FROM_JSON_OBJECT(val, info.top_block_hash, top_block_hash); GET_FROM_JSON_OBJECT(val, info.cumulative_difficulty, cumulative_difficulty); @@ -1232,6 +1236,29 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.start_time, start_time); } +void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::output_distribution& dist, rapidjson::Value& val) +{ + val.SetObject(); + + INSERT_INTO_JSON_OBJECT(val, doc, distribution, dist.data.distribution); + INSERT_INTO_JSON_OBJECT(val, doc, amount, dist.amount); + INSERT_INTO_JSON_OBJECT(val, doc, start_height, dist.data.start_height); + INSERT_INTO_JSON_OBJECT(val, doc, base, dist.data.base); +} + +void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist) +{ + if (!val.IsObject()) + { + throw WRONG_TYPE("json object"); + } + + GET_FROM_JSON_OBJECT(val, dist.data.distribution, distribution); + GET_FROM_JSON_OBJECT(val, dist.amount, amount); + GET_FROM_JSON_OBJECT(val, dist.data.start_height, start_height); + GET_FROM_JSON_OBJECT(val, dist.data.base, base); +} + } // namespace json } // namespace cryptonote diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index da3351fe3..b6384ca0d 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -281,6 +281,9 @@ void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig); void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& info, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info); +void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::output_distribution& dist, rapidjson::Value& val); +void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist); + template <typename Map> typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Document& doc, const Map& map, rapidjson::Value& val); diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index c31cdebde..e292f85dd 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -53,6 +53,7 @@ target_link_libraries(simplewallet ${Boost_CHRONO_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} + ${Boost_LOCALE_LIBRARY} ${ICU_LIBRARIES} ${Boost_THREAD_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index f171d35b6..ce3fc3b09 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -43,6 +43,7 @@ #include <boost/algorithm/string.hpp> #include <boost/format.hpp> #include <boost/regex.hpp> +#include <boost/range/adaptor/transformed.hpp> #include "include_base_utils.h" #include "common/i18n.h" #include "common/command_line.h" @@ -780,10 +781,7 @@ bool simple_wallet::payment_id(const std::vector<std::string> &args/* = std::vec bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("Cannot connect to daemon"); return true; - } const bool per_byte = m_wallet->use_fork_rules(HF_VERSION_PER_BYTE_FEE); const uint64_t base_fee = m_wallet->get_base_fee(); const char *base = per_byte ? "byte" : "kB"; @@ -1801,6 +1799,27 @@ bool simple_wallet::version(const std::vector<std::string> &args) return true; } +bool simple_wallet::cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& ptx_vector, tools::wallet2::signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::function<bool(const tools::wallet2::signed_tx_set &)> accept_func) +{ + std::vector<std::string> tx_aux; + + message_writer(console_color_white, false) << tr("Please confirm the transaction on the device"); + + m_wallet->cold_sign_tx(ptx_vector, exported_txs, dsts_info, tx_aux); + + if (accept_func && !accept_func(exported_txs)) + { + MERROR("Transactions rejected by callback"); + return false; + } + + // aux info + m_wallet->cold_tx_aux_import(exported_txs.ptx, tx_aux); + + // import key images + return m_wallet->import_key_images(exported_txs.key_images); +} + bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { const auto pwd_container = get_and_verify_password(); @@ -2253,6 +2272,33 @@ bool simple_wallet::set_ignore_fractional_outputs(const std::vector<std::string> return true; } +bool simple_wallet::set_device_name(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + if (args.size() == 0){ + fail_msg_writer() << tr("Device name not specified"); + return true; + } + + m_wallet->device_name(args[0]); + bool r = false; + try { + r = m_wallet->reconnect_device(); + if (!r){ + fail_msg_writer() << tr("Device reconnect failed"); + } + + } catch(const std::exception & e){ + MWARNING("Device reconnect failed: " << e.what()); + fail_msg_writer() << tr("Device reconnect failed: ") << e.what(); + } + + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if(args.empty()) @@ -2493,12 +2539,17 @@ simple_wallet::simple_wallet() "Pending or Failed: \"failed\"|\"pending\", \"out\", Time, Amount*, Transaction Hash, Payment ID, Fee, Input addresses**, \"-\", Note\n\n" "* Excluding change and fee.\n" "** Set of address indices used as inputs in this transfer.")); + m_cmd_binder.set_handler("export_transfers", + boost::bind(&simple_wallet::export_transfers, this, _1), + tr("export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<filepath>]"), + tr("Export to CSV the incoming/outgoing transfers within an optional height range.")); m_cmd_binder.set_handler("unspent_outputs", boost::bind(&simple_wallet::unspent_outputs, this, _1), tr("unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]"), tr("Show the unspent outputs of a specified address within an optional amount range.")); m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), + tr("rescan_bc [hard]"), tr("Rescan the blockchain from scratch, losing any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), @@ -2537,6 +2588,10 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::import_key_images, this, _1), tr("import_key_images <file>"), tr("Import a signed key images list and verify their spent status.")); + m_cmd_binder.set_handler("hw_key_images_sync", + boost::bind(&simple_wallet::hw_key_images_sync, this, _1), + tr("hw_key_images_sync"), + tr("Synchronizes key images with the hw wallet.")); m_cmd_binder.set_handler("hw_reconnect", boost::bind(&simple_wallet::hw_reconnect, this, _1), tr("hw_reconnect"), @@ -2728,6 +2783,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("subaddress-lookahead", set_subaddress_lookahead, tr("<major>:<minor>")); CHECK_SIMPLE_VARIABLE("segregation-height", set_segregation_height, tr("unsigned integer")); CHECK_SIMPLE_VARIABLE("ignore-fractional-outputs", set_ignore_fractional_outputs, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("device-name", set_device_name, tr("<device_name[:device_spec]>")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; @@ -4245,15 +4301,15 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init) +bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bool is_init) { if (!try_connect_to_daemon(is_init)) return true; LOCK_IDLE_SCOPE(); - if (reset) - m_wallet->rescan_blockchain(false); + if (reset != ResetNone) + m_wallet->rescan_blockchain(reset == ResetHard, false); #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; @@ -4332,7 +4388,7 @@ bool simple_wallet::refresh(const std::vector<std::string>& args) start_height = 0; } } - return refresh_main(start_height, false); + return refresh_main(start_height, ResetNone); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::show_balance_unlocked(bool detailed) @@ -4430,7 +4486,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args tools::wallet2::transfer_container transfers; m_wallet->get_transfers(transfers); - bool transfers_found = false; + size_t transfers_found = 0; for (const auto& td : transfers) { if (!filter || available != td.m_spent) @@ -4443,7 +4499,6 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args if (verbose) verbose_string = (boost::format("%68s%68s") % tr("pubkey") % tr("key image")).str(); message_writer() << boost::format("%21s%8s%12s%8s%16s%68s%16s%s") % tr("amount") % tr("spent") % tr("unlocked") % tr("ringct") % tr("global index") % tr("tx id") % tr("addr index") % verbose_string; - transfers_found = true; } std::string verbose_string; if (verbose) @@ -4458,6 +4513,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args td.m_txid % td.m_subaddr_index.minor % verbose_string; + ++transfers_found; } } @@ -4476,6 +4532,10 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args success_msg_writer() << tr("No incoming unavailable transfers"); } } + else + { + success_msg_writer() << boost::format("Found %u/%u transfers") % transfers_found % transfers.size(); + } return true; } @@ -4608,10 +4668,7 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending { uint32_t version; if (!try_connect_to_daemon(false, &version)) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return false; - } // available for RPC version 1.4 or higher if (version < MAKE_CORE_RPC_VERSION(1, 4)) return true; @@ -4834,12 +4891,14 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri local_args.pop_back(); } + vector<cryptonote::address_parse_info> dsts_info; vector<cryptonote::tx_destination_entry> dsts; size_t num_subaddresses = 0; for (size_t i = 0; i < local_args.size(); ) { + dsts_info.emplace_back(); + cryptonote::address_parse_info & info = dsts_info.back(); cryptonote::tx_destination_entry de; - cryptonote::address_parse_info info; bool r = true; // check for a URI @@ -5123,6 +5182,28 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to file: ") << "multisig_monero_tx"; } } + else if (m_wallet->get_account().get_device().has_tx_cold_sign()) + { + try + { + tools::wallet2::signed_tx_set signed_tx; + if (!cold_sign_tx(ptx_vector, signed_tx, dsts_info, [&](const tools::wallet2::signed_tx_set &tx){ return accept_loaded_tx(tx); })){ + fail_msg_writer() << tr("Failed to cold sign transaction with HW wallet"); + return true; + } + + commit_or_save(signed_tx.ptx, m_do_not_relay); + } + catch (const std::exception& e) + { + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); + } + catch (...) + { + LOG_ERROR("Unknown error"); + fail_msg_writer() << tr("unknown error"); + } + } else if (m_wallet->watch_only()) { bool r = m_wallet->save_tx(ptx_vector, "unsigned_monero_tx"); @@ -5545,6 +5626,31 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to file: ") << "multisig_monero_tx"; } } + else if (m_wallet->get_account().get_device().has_tx_cold_sign()) + { + try + { + tools::wallet2::signed_tx_set signed_tx; + std::vector<cryptonote::address_parse_info> dsts_info; + dsts_info.push_back(info); + + if (!cold_sign_tx(ptx_vector, signed_tx, dsts_info, [&](const tools::wallet2::signed_tx_set &tx){ return accept_loaded_tx(tx); })){ + fail_msg_writer() << tr("Failed to cold sign transaction with HW wallet"); + return true; + } + + commit_or_save(signed_tx.ptx, m_do_not_relay); + } + catch (const std::exception& e) + { + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); + } + catch (...) + { + LOG_ERROR("Unknown error"); + fail_msg_writer() << tr("unknown error"); + } + } else if (m_wallet->watch_only()) { bool r = m_wallet->save_tx(ptx_vector, "unsigned_monero_tx"); @@ -6003,8 +6109,8 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, bool simple_wallet::accept_loaded_tx(const tools::wallet2::unsigned_tx_set &txs) { std::string extra_message; - if (!txs.transfers.empty()) - extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.size()).str(); + if (!txs.transfers.second.empty()) + extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.second.size()).str(); return accept_loaded_tx([&txs](){return txs.txes.size();}, [&txs](size_t n)->const tools::wallet2::tx_construction_data&{return txs.txes[n];}, extra_message); } //---------------------------------------------------------------------------------------------------- @@ -6451,10 +6557,7 @@ bool simple_wallet::get_spend_proof(const std::vector<std::string> &args) } if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return true; - } SCOPED_WALLET_UNLOCK(); @@ -6489,10 +6592,7 @@ bool simple_wallet::check_spend_proof(const std::vector<std::string> &args) } if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return true; - } std::string sig_str; if (!epee::file_io_utils::load_file_to_string(args[1], sig_str)) @@ -6546,10 +6646,7 @@ bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args) } if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return true; - } SCOPED_WALLET_UNLOCK(); @@ -6577,10 +6674,7 @@ bool simple_wallet::check_reserve_proof(const std::vector<std::string> &args) } if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return true; - } cryptonote::address_parse_info info; if(!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), args[0], oa_prompter)) @@ -6656,9 +6750,9 @@ static std::string get_human_readable_timespan(std::chrono::seconds seconds) return sw::tr("a long time"); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::show_transfers(const std::vector<std::string> &args_) +// mutates local_args as it parses and consumes arguments +bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vector<transfer_view>& transfers) { - std::vector<std::string> local_args = args_; bool in = true; bool out = true; bool pending = true; @@ -6667,15 +6761,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) bool coinbase = true; uint64_t min_height = 0; uint64_t max_height = (uint64_t)-1; - boost::optional<uint32_t> subaddr_index; - if(local_args.size() > 4) { - fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); - return true; - } - - LOCK_IDLE_SCOPE(); - // optional in/out selector if (local_args.size() > 0) { if (local_args[0] == "in" || local_args[0] == "incoming") { @@ -6713,38 +6799,34 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (local_args.size() > 0 && local_args[0].substr(0, 6) == "index=") { if (!parse_subaddress_indices(local_args[0], subaddr_indices)) - return true; + return false; local_args.erase(local_args.begin()); } // min height - if (local_args.size() > 0) { + if (local_args.size() > 0 && local_args[0].find('=') == std::string::npos) { try { min_height = boost::lexical_cast<uint64_t>(local_args[0]); } catch (const boost::bad_lexical_cast &) { fail_msg_writer() << tr("bad min_height parameter:") << " " << local_args[0]; - return true; + return false; } local_args.erase(local_args.begin()); } // max height - if (local_args.size() > 0) { + if (local_args.size() > 0 && local_args[0].find('=') == std::string::npos) { try { max_height = boost::lexical_cast<uint64_t>(local_args[0]); } catch (const boost::bad_lexical_cast &) { fail_msg_writer() << tr("bad max_height parameter:") << " " << local_args[0]; - return true; + return false; } local_args.erase(local_args.begin()); } - std::multimap<uint64_t, std::tuple<epee::console_colors, std::string, std::string>> output; - - PAUSE_READLINE(); - if (in || coinbase) { std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments; m_wallet->get_payments(payments, min_height, max_height, m_current_subaddress_account, subaddr_indices); @@ -6756,23 +6838,26 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(pd.m_tx_hash); + std::string destination = m_wallet->get_subaddress_as_str({m_current_subaddress_account, pd.m_subaddr_index.minor}); const std::string type = pd.m_coinbase ? tr("block") : tr("in"); - output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_green, type, (boost::format("%25.25s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); + const bool unlocked = m_wallet->is_tx_spendtime_unlocked(pd.m_unlock_time, pd.m_block_height); + transfers.push_back({ + pd.m_block_height, + pd.m_timestamp, + type, + true, + pd.m_amount, + pd.m_tx_hash, + payment_id, + 0, + {{destination, pd.m_amount}}, + {pd.m_subaddr_index.minor}, + note, + (unlocked) ? "unlocked" : "locked" + }); } } - auto print_subaddr_indices = [](const std::set<uint32_t>& indices) - { - stringstream ss; - bool first = true; - for (uint32_t i : indices) - { - ss << (first ? "" : ",") << i; - first = false; - } - return ss.str(); - }; - if (out) { std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>> payments; m_wallet->get_payments_out(payments, min_height, max_height, m_current_subaddress_account, subaddr_indices); @@ -6780,27 +6865,31 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) const tools::wallet2::confirmed_transfer_details &pd = i->second; uint64_t change = pd.m_change == (uint64_t)-1 ? 0 : pd.m_change; // change may not be known uint64_t fee = pd.m_amount_in - pd.m_amount_out; - std::string dests; + std::vector<std::pair<std::string, uint64_t>> destinations; for (const auto &d: pd.m_dests) { - if (!dests.empty()) - dests += ", "; - dests += get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) + ": " + print_money(d.amount); + destinations.push_back({get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr), d.amount}); } std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); - output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_magenta, tr("out"), (boost::format("%25.25s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); + transfers.push_back({ + pd.m_block_height, + pd.m_timestamp, + "out", + true, + pd.m_amount_in - change - fee, + i->first, + payment_id, + fee, + destinations, + pd.m_subaddr_indices, + note, + "-" + }); } } - // print in and out sorted by height - for (std::multimap<uint64_t, std::tuple<epee::console_colors, std::string, std::string>>::const_iterator i = output.begin(); i != output.end(); ++i) { - message_writer(std::get<0>(i->second), false) << - boost::format("%8.8llu %6.6s %s") % - ((unsigned long long)i->first) % std::get<1>(i->second) % std::get<2>(i->second); - } - if (pool) { try { @@ -6816,10 +6905,24 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(pd.m_tx_hash); + std::string destination = m_wallet->get_subaddress_as_str({m_current_subaddress_account, pd.m_subaddr_index.minor}); std::string double_spend_note; if (i->second.m_double_spend_seen) double_spend_note = tr("[Double spend seen on the network: this transaction may or may not end up being mined] "); - message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %d %s %s%s") % "pool" % "in" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note % double_spend_note).str(); + transfers.push_back({ + "pool", + pd.m_timestamp, + "in", + false, + pd.m_amount, + pd.m_tx_hash, + payment_id, + 0, + {{destination, pd.m_amount}}, + {pd.m_subaddr_index.minor}, + note + double_spend_note, + "locked" + }); } } catch (const std::exception& e) @@ -6836,20 +6939,187 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) const tools::wallet2::unconfirmed_transfer_details &pd = i->second; uint64_t amount = pd.m_amount_in; uint64_t fee = amount - pd.m_amount_out; + std::vector<std::pair<std::string, uint64_t>> destinations; + for (const auto &d: pd.m_dests) { + destinations.push_back({get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr), d.amount}); + } std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed; if ((failed && is_failed) || (!is_failed && pending)) { - message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %14.14s %s - %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % print_subaddr_indices(pd.m_subaddr_indices) % note).str(); + transfers.push_back({ + (is_failed ? "failed" : "pending"), + pd.m_timestamp, + "out", + false, + amount - pd.m_change - fee, + i->first, + payment_id, + fee, + destinations, + pd.m_subaddr_indices, + note, + "-" + }); + } + } + } + // sort by block, then by timestamp (unconfirmed last) + std::sort(transfers.begin(), transfers.end(), [](const transfer_view& a, const transfer_view& b) -> bool { + if (a.confirmed && !b.confirmed) + return true; + if (a.block == b.block) + return a.timestamp < b.timestamp; + return a.block < b.block; + }); + + return true; +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::show_transfers(const std::vector<std::string> &args_) +{ + std::vector<std::string> local_args = args_; + + if(local_args.size() > 4) { + fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); + return true; + } + + LOCK_IDLE_SCOPE(); + + std::vector<transfer_view> all_transfers; + + if (!get_transfers(local_args, all_transfers)) + return true; + + PAUSE_READLINE(); + + for (const auto& transfer : all_transfers) + { + const auto color = transfer.confirmed ? ((transfer.direction == "in" || transfer.direction == "block") ? console_color_green : console_color_magenta) : console_color_white; + + std::string destinations = "-"; + if (!transfer.outputs.empty()) + { + destinations = ""; + for (const auto& output : transfer.outputs) + { + if (!destinations.empty()) + destinations += ", "; + destinations += (transfer.direction == "in" ? output.first.substr(0, 6) : output.first) + ":" + print_money(output.second); } } + + auto formatter = boost::format("%8.8llu %6.6s %8.8s %25.25s %20.20s %s %s %14.14s %s %s - %s"); + + message_writer(color, false) << formatter + % transfer.block + % transfer.direction + % transfer.unlocked + % get_human_readable_timestamp(transfer.timestamp) + % print_money(transfer.amount) + % string_tools::pod_to_hex(transfer.hash) + % transfer.payment_id + % print_money(transfer.fee) + % destinations + % boost::algorithm::join(transfer.index | boost::adaptors::transformed([](uint32_t i) { return std::to_string(i); }), ", ") + % transfer.note; } return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::export_transfers(const std::vector<std::string>& args_) +{ + std::vector<std::string> local_args = args_; + + if(local_args.size() > 5) { + fail_msg_writer() << tr("usage: export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<path>]"); + return true; + } + + LOCK_IDLE_SCOPE(); + + std::vector<transfer_view> all_transfers; + + // might consumes arguments in local_args + if (!get_transfers(local_args, all_transfers)) + return true; + + // output filename + std::string filename = (boost::format("output%u.csv") % m_current_subaddress_account).str(); + if (local_args.size() > 0 && local_args[0].substr(0, 7) == "output=") + { + filename = local_args[0].substr(7, -1); + local_args.erase(local_args.begin()); + } + + std::ofstream file(filename); + + // header + file << + boost::format("%8.8s,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%100.100s,%20.20s,%s,%s") % + tr("block") % tr("direction") % tr("unlocked") % tr("timestamp") % tr("amount") % tr("running balance") % tr("hash") % tr("payment ID") % tr("fee") % tr("destination") % tr("amount") % tr("index") % tr("note") + << std::endl; + + uint64_t running_balance = 0; + auto formatter = boost::format("%8.8llu,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%100.100s,%20.20s,\"%s\",%s"); + + for (const auto& transfer : all_transfers) + { + // ignore unconfirmed transfers in running balance + if (transfer.confirmed) + { + if (transfer.direction == "in" || transfer.direction == "block") + running_balance += transfer.amount; + else + running_balance -= transfer.amount + transfer.fee; + } + + file << formatter + % transfer.block + % transfer.direction + % transfer.unlocked + % get_human_readable_timestamp(transfer.timestamp) + % print_money(transfer.amount) + % print_money(running_balance) + % string_tools::pod_to_hex(transfer.hash) + % transfer.payment_id + % print_money(transfer.fee) + % (transfer.outputs.size() ? transfer.outputs[0].first : "-") + % (transfer.outputs.size() ? print_money(transfer.outputs[0].second) : "") + % boost::algorithm::join(transfer.index | boost::adaptors::transformed([](uint32_t i) { return std::to_string(i); }), ", ") + % transfer.note + << std::endl; + + for (size_t i = 1; i < transfer.outputs.size(); ++i) + { + file << formatter + % "" + % "" + % "" + % "" + % "" + % "" + % "" + % "" + % "" + % transfer.outputs[i].first + % print_money(transfer.outputs[i].second) + % "" + % "" + << std::endl; + } + } + file.close(); + + success_msg_writer() << tr("CSV exported to ") << filename; + + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) { if(args_.size() > 3) @@ -6990,15 +7260,29 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { - message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); - message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); - std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); - if(!std::cin.eof()) + bool hard = false; + if (!args_.empty()) { - if (!command_line::is_yes(confirm)) + if (args_[0] != "hard") + { + fail_msg_writer() << tr("usage: rescan_bc [hard]"); return true; + } + hard = true; + } + + if (hard) + { + message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); + message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); + std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); + if(!std::cin.eof()) + { + if (!command_line::is_yes(confirm)) + return true; + } } - return refresh_main(0, true); + return refresh_main(0, hard ? ResetHard : ResetSoft, true); } //---------------------------------------------------------------------------------------------------- void simple_wallet::wallet_idle_thread() @@ -7046,7 +7330,7 @@ bool simple_wallet::run() // check and display warning, but go on anyway try_connect_to_daemon(); - refresh_main(0, false, true); + refresh_main(0, ResetNone, true); m_auto_refresh_enabled = m_wallet->auto_refresh(); m_idle_thread = boost::thread([&]{wallet_idle_thread();}); @@ -7794,6 +8078,48 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::hw_key_images_sync(const std::vector<std::string> &args) +{ + if (!m_wallet->key_on_device()) + { + fail_msg_writer() << tr("command only supported by HW wallet"); + return true; + } + if (!m_wallet->get_account().get_device().has_ki_cold_sync()) + { + fail_msg_writer() << tr("hw wallet does not support cold KI sync"); + return true; + } + if (!m_wallet->is_trusted_daemon()) + { + fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); + return true; + } + + LOCK_IDLE_SCOPE(); + try + { + message_writer(console_color_white, false) << tr("Please confirm the key image sync on the device"); + + uint64_t spent = 0, unspent = 0; + uint64_t height = m_wallet->cold_key_image_sync(spent, unspent); + if (height > 0) + { + success_msg_writer() << tr("Signed key images imported to height ") << height << ", " + << print_money(spent) << tr(" spent, ") << print_money(unspent) << tr(" unspent"); + } else { + fail_msg_writer() << tr("Failed to import key images"); + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to import key images: ") << e.what(); + return true; + } + + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::hw_reconnect(const std::vector<std::string> &args) { if (!m_wallet->key_on_device()) diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 39b715b73..421afbeda 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -83,6 +83,9 @@ namespace cryptonote std::string get_commands_str(); std::string get_command_usage(const std::vector<std::string> &args); private: + + enum ResetType { ResetNone, ResetSoft, ResetHard }; + bool handle_command_line(const boost::program_options::variables_map& vm); bool run_console_handler(); @@ -139,6 +142,7 @@ namespace cryptonote bool set_subaddress_lookahead(const std::vector<std::string> &args = std::vector<std::string>()); bool set_segregation_height(const std::vector<std::string> &args = std::vector<std::string>()); bool set_ignore_fractional_outputs(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_device_name(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -186,9 +190,10 @@ namespace cryptonote bool get_reserve_proof(const std::vector<std::string> &args); bool check_reserve_proof(const std::vector<std::string> &args); bool show_transfers(const std::vector<std::string> &args); + bool export_transfers(const std::vector<std::string> &args); bool unspent_outputs(const std::vector<std::string> &args); bool rescan_blockchain(const std::vector<std::string> &args); - bool refresh_main(uint64_t start_height, bool reset = false, bool is_init = false); + bool refresh_main(uint64_t start_height, ResetType reset, bool is_init = false); bool set_tx_note(const std::vector<std::string> &args); bool get_tx_note(const std::vector<std::string> &args); bool set_description(const std::vector<std::string> &args); @@ -200,6 +205,7 @@ namespace cryptonote bool verify(const std::vector<std::string> &args); bool export_key_images(const std::vector<std::string> &args); bool import_key_images(const std::vector<std::string> &args); + bool hw_key_images_sync(const std::vector<std::string> &args); bool hw_reconnect(const std::vector<std::string> &args); bool export_outputs(const std::vector<std::string> &args); bool import_outputs(const std::vector<std::string> &args); @@ -224,6 +230,7 @@ namespace cryptonote bool unblackball(const std::vector<std::string>& args); bool blackballed(const std::vector<std::string>& args); bool version(const std::vector<std::string>& args); + bool cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& ptx_vector, tools::wallet2::signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::function<bool(const tools::wallet2::signed_tx_set &)> accept_func); uint64_t get_daemon_blockchain_height(std::string& err); bool try_connect_to_daemon(bool silent = false, uint32_t* version = nullptr); @@ -235,6 +242,23 @@ namespace cryptonote std::string get_prompt() const; bool print_seed(bool encrypted); + struct transfer_view + { + boost::variant<uint64_t, std::string> block; + uint64_t timestamp; + std::string direction; + bool confirmed; + uint64_t amount; + crypto::hash hash; + std::string payment_id; + uint64_t fee; + std::vector<std::pair<std::string, uint64_t>> outputs; + std::set<uint32_t> index; + std::string note; + std::string unlocked; + }; + bool get_transfers(std::vector<std::string>& args_, std::vector<transfer_view>& transfers); + /*! * \brief Prints the seed with a nice message * \param seed seed to print diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index be10b9f62..4e3fb1ae5 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -57,6 +57,7 @@ target_link_libraries(wallet common cryptonote_core mnemonics + device_trezor ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} @@ -119,7 +120,8 @@ if (BUILD_GUI_DEPS) ringct ringct_basic checkpoints - version) + version + device_trezor) foreach(lib ${libs_to_merge}) list(APPEND objlibs $<TARGET_OBJECTS:obj_${lib}>) # matches naming convention in src/CMakeLists.txt diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index ddf2d74ff..7cd3b65bb 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -507,7 +507,7 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas auto key_images = m_wallet->export_key_images(); uint64_t spent = 0; uint64_t unspent = 0; - view_wallet->import_key_images(key_images,spent,unspent,false); + view_wallet->import_key_images(key_images.second, key_images.first, spent, unspent, false); clearStatus(); } catch (const std::exception &e) { LOG_ERROR("Error creating view only wallet: " << e.what()); @@ -1051,8 +1051,8 @@ UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_file // Check tx data and construct confirmation message std::string extra_message; - if (!transaction->m_unsigned_tx_set.transfers.empty()) - extra_message = (boost::format("%u outputs to import. ") % (unsigned)transaction->m_unsigned_tx_set.transfers.size()).str(); + if (!transaction->m_unsigned_tx_set.transfers.second.empty()) + extra_message = (boost::format("%u outputs to import. ") % (unsigned)transaction->m_unsigned_tx_set.transfers.second.size()).str(); transaction->checkLoadedTx([&transaction](){return transaction->m_unsigned_tx_set.txes.size();}, [&transaction](size_t n)->const tools::wallet2::tx_construction_data&{return transaction->m_unsigned_tx_set.txes[n];}, extra_message); setStatus(transaction->status(), transaction->errorString()); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f3ef67095..7d92df239 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -69,8 +69,11 @@ using namespace epee; #include "common/base58.h" #include "common/dns_utils.h" #include "common/notify.h" +#include "common/perf_timer.h" #include "ringct/rctSigs.h" #include "ringdb.h" +#include "device/device_cold.hpp" +#include "device_trezor/device_trezor.hpp" extern "C" { @@ -110,11 +113,11 @@ using namespace cryptonote; #define SUBADDRESS_LOOKAHEAD_MAJOR 50 #define SUBADDRESS_LOOKAHEAD_MINOR 200 -#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\002" +#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\003" #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" -#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" +#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\004" #define SEGREGATION_FORK_HEIGHT 99999999 #define TESTNET_SEGREGATION_FORK_HEIGHT 99999999 @@ -768,6 +771,11 @@ uint32_t get_subaddress_clamped_sum(uint32_t idx, uint32_t extra) return idx + extra; } +static void setup_shim(hw::wallet_shim * shim, tools::wallet2 * wallet) +{ + shim->get_tx_pub_key_from_received_outs = boost::bind(&tools::wallet2::get_tx_pub_key_from_received_outs, wallet, _1); +} + //----------------------------------------------------------------- } //namespace @@ -1060,8 +1068,9 @@ bool wallet2::get_multisig_seed(epee::wipeable_string& seed, const epee::wipeabl bool wallet2::reconnect_device() { bool r = true; - hw::device &hwdev = hw::get_device(m_device_name); + hw::device &hwdev = lookup_device(m_device_name); hwdev.set_name(m_device_name); + hwdev.set_network_type(m_nettype); r = hwdev.init(); if (!r){ LOG_PRINT_L2("Could not init device"); @@ -1597,6 +1606,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = amount; td.m_pk_index = pk_index - 1; @@ -2871,8 +2881,8 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> MWARNING("Failed to request output distribution: results are not for amount 0"); return false; } - start_height = res.distributions[0].start_height; - distribution = std::move(res.distributions[0].distribution); + start_height = res.distributions[0].data.start_height; + distribution = std::move(res.distributions[0].data.distribution); return true; } //---------------------------------------------------------------------------------------------------- @@ -2944,6 +2954,7 @@ bool wallet2::deinit() { m_is_initialized=false; unlock_keys_file(); + m_account.deinit(); return true; } //---------------------------------------------------------------------------------------------------- @@ -3140,13 +3151,22 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]); keys_file_data.account_data = cipher; - unlock_keys_file(); + std::string tmp_file_name = keys_file_name + ".new"; std::string buf; r = ::serialization::dump_binary(keys_file_data, buf); - r = r && epee::file_io_utils::save_string_to_file(keys_file_name, buf); //and never touch wallet_keys_file again, only read - CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << keys_file_name); + r = r && epee::file_io_utils::save_string_to_file(tmp_file_name, buf); + CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); + + unlock_keys_file(); + std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); lock_keys_file(); + if (e) { + boost::filesystem::remove(tmp_file_name); + LOG_ERROR("failed to update wallet keys file " << keys_file_name); + return false; + } + return true; } //---------------------------------------------------------------------------------------------------- @@ -3413,13 +3433,20 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ r = epee::serialization::load_t_from_binary(m_account, account_data); THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); - if (m_key_device_type == hw::device::device_type::LEDGER) { + if (m_key_device_type == hw::device::device_type::LEDGER || m_key_device_type == hw::device::device_type::TREZOR) { LOG_PRINT_L0("Account on device. Initing device..."); - hw::device &hwdev = hw::get_device(m_device_name); - hwdev.set_name(m_device_name); - hwdev.init(); - hwdev.connect(); + hw::device &hwdev = lookup_device(m_device_name); + THROW_WALLET_EXCEPTION_IF(!hwdev.set_name(m_device_name), error::wallet_internal_error, "Could not set device name " + m_device_name); + hwdev.set_network_type(m_nettype); + THROW_WALLET_EXCEPTION_IF(!hwdev.init(), error::wallet_internal_error, "Could not initialize the device " + m_device_name); + THROW_WALLET_EXCEPTION_IF(!hwdev.connect(), error::wallet_internal_error, "Could not connect to the device " + m_device_name); m_account.set_device(hwdev); + + account_public_address device_account_public_address; + THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + THROW_WALLET_EXCEPTION_IF(device_account_public_address != m_account.get_keys().m_account_address, error::wallet_internal_error, "Device wallet does not match wallet address. " + "Device address: " + cryptonote::get_account_address_as_str(m_nettype, false, device_account_public_address) + + ", wallet address: " + m_account.get_public_address_str(m_nettype)); LOG_PRINT_L0("Device inited..."); } else if (key_on_device()) { THROW_WALLET_EXCEPTION(error::wallet_internal_error, "hardware device not supported"); @@ -3450,7 +3477,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); - if(!m_watch_only && !m_multisig) + if(!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); @@ -3474,7 +3501,7 @@ bool wallet2::verify_password(const epee::wipeable_string& password) { // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). unlock_keys_file(); - bool r = verify_password(m_keys_file, password, m_watch_only || m_multisig, m_account.get_device(), m_kdf_rounds); + bool r = verify_password(m_keys_file, password, m_account.get_device().device_protocol() == hw::device::PROTOCOL_COLD || m_watch_only || m_multisig, m_account.get_device(), m_kdf_rounds); lock_keys_file(); return r; } @@ -3914,8 +3941,9 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(m_keys_file, ignored_ec), error::file_exists, m_keys_file); } - auto &hwdev = hw::get_device(device_name); + auto &hwdev = lookup_device(device_name); hwdev.set_name(device_name); + hwdev.set_network_type(m_nettype); m_account.create_from_device(hwdev); m_key_device_type = m_account.get_device().get_type(); @@ -5077,11 +5105,27 @@ void wallet2::rescan_spent() } } //---------------------------------------------------------------------------------------------------- -void wallet2::rescan_blockchain(bool refresh) +void wallet2::rescan_blockchain(bool hard, bool refresh) { - clear(); + if(hard) + { + clear(); + setup_new_blockchain(); + } + else + { + m_blockchain.clear(); + m_transfers.clear(); + m_key_images.clear(); + m_pub_keys.clear(); + m_scanned_pool_txs[0].clear(); + m_scanned_pool_txs[1].clear(); - setup_new_blockchain(); + cryptonote::block b; + generate_genesis(b); + m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); + } if (refresh) this->refresh(false); @@ -5448,7 +5492,7 @@ std::string wallet2::dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) c txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = export_outputs(); // save as binary std::ostringstream oss; boost::archive::portable_binary_oarchive ar(oss); @@ -5781,22 +5825,8 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too } // import key images - if (signed_txs.key_images.size() > m_transfers.size()) - { - LOG_PRINT_L1("More key images returned that we know outputs for"); - return false; - } - for (size_t i = 0; i < signed_txs.key_images.size(); ++i) - { - transfer_details &td = m_transfers[i]; - if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != signed_txs.key_images[i]) - LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); - td.m_key_image = signed_txs.key_images[i]; - m_key_images[m_transfers[i].m_key_image] = i; - td.m_key_image_known = true; - td.m_key_image_partial = false; - m_pub_keys[m_transfers[i].get_public_key()] = i; - } + bool r = import_key_images(signed_txs.key_images); + if (!r) return false; ptx = signed_txs.ptx; @@ -5997,10 +6027,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto rct::RangeProofType range_proof_type = rct::RangeProofBorromean; if (sd.use_bulletproofs) { - range_proof_type = rct::RangeProofBulletproof; - for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) - if (proof.V.size() > 1) - range_proof_type = rct::RangeProofPaddedBulletproof; + range_proof_type = rct::RangeProofPaddedBulletproof; } bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, range_proof_type, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); @@ -6346,6 +6373,19 @@ crypto::chacha_key wallet2::get_ringdb_key() return *m_ringdb_key; } +void wallet2::register_devices(){ + hw::trezor::register_all(); +} + +hw::device& wallet2::lookup_device(const std::string & device_descriptor){ + if (!m_devices_registered){ + m_devices_registered = true; + register_devices(); + } + + return hw::get_device(device_descriptor); +} + bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) @@ -6693,15 +6733,21 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> uint64_t rct_start_height; std::vector<uint64_t> rct_offsets; bool has_rct = false; + uint64_t max_rct_index = 0; for (size_t idx: selected_transfers) if (m_transfers[idx].is_rct()) - { has_rct = true; break; } + { + has_rct = true; + max_rct_index = std::max(max_rct_index, m_transfers[idx].m_global_output_index); + } const bool has_rct_distribution = has_rct && get_rct_distribution(rct_start_height, rct_offsets); if (has_rct_distribution) { // check we're clear enough of rct start, to avoid corner cases below THROW_WALLET_EXCEPTION_IF(rct_offsets.size() <= CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, error::get_output_distribution, "Not enough rct outputs"); + THROW_WALLET_EXCEPTION_IF(rct_offsets.back() <= max_rct_index, + error::get_output_distribution, "Daemon reports suspicious number of rct outputs"); } // get histogram for the amounts we need @@ -6757,13 +6803,13 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> { if (d.amount == amount) { - THROW_WALLET_EXCEPTION_IF(d.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small"); + THROW_WALLET_EXCEPTION_IF(d.data.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.data.start_height >= d.data.distribution.size(), error::get_output_distribution, "Distribution size too small"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.data.start_height >= d.data.distribution.size(), error::get_output_distribution, "Distribution size too small"); THROW_WALLET_EXCEPTION_IF(segregation_fork_height <= RECENT_OUTPUT_BLOCKS, error::wallet_internal_error, "Fork height too low"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.start_height, error::get_output_distribution, "Bad start height"); - uint64_t till_fork = d.distribution[segregation_fork_height - d.start_height]; - uint64_t recent = till_fork - d.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height]; + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.data.start_height, error::get_output_distribution, "Bad start height"); + uint64_t till_fork = d.data.distribution[segregation_fork_height - d.data.start_height]; + uint64_t recent = till_fork - d.data.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.data.start_height]; segregation_limit[amount] = std::make_pair(till_fork, recent); found = true; break; @@ -6792,21 +6838,23 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> //static const double shape = m_testnet ? 17.02 : 17.28; static const double scale = 1/1.61; std::gamma_distribution<double> gamma(shape, scale); + THROW_WALLET_EXCEPTION_IF(rct_offsets.size() <= CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, error::wallet_internal_error, "Bad offset calculation"); + uint64_t last_usable_block = rct_offsets.size() - 1; auto pick_gamma = [&]() { double x = gamma(engine); x = exp(x); uint64_t block_offset = x / DIFFICULTY_TARGET_V2; // this assumes constant target over the whole rct range - if (block_offset >= rct_offsets.size() - 1) + if (block_offset > last_usable_block - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE) return std::numeric_limits<uint64_t>::max(); // bad pick - block_offset = rct_offsets.size() - 2 - block_offset; - THROW_WALLET_EXCEPTION_IF(block_offset >= rct_offsets.size() - 1, error::wallet_internal_error, "Bad offset calculation"); - THROW_WALLET_EXCEPTION_IF(rct_offsets[block_offset + 1] < rct_offsets[block_offset], + block_offset = last_usable_block - block_offset; + THROW_WALLET_EXCEPTION_IF(block_offset > last_usable_block, error::wallet_internal_error, "Bad offset calculation"); + THROW_WALLET_EXCEPTION_IF(block_offset > 0 && rct_offsets[block_offset] < rct_offsets[block_offset - 1], error::get_output_distribution, "Decreasing offsets in rct distribution: " + - std::to_string(block_offset) + ": " + std::to_string(rct_offsets[block_offset]) + ", " + - std::to_string(block_offset + 1) + ": " + std::to_string(rct_offsets[block_offset + 1])); + std::to_string(block_offset - 1) + ": " + std::to_string(rct_offsets[block_offset - 1]) + ", " + + std::to_string(block_offset) + ": " + std::to_string(rct_offsets[block_offset])); uint64_t first_block_offset = block_offset, last_block_offset = block_offset; - for (size_t half_window = 0; half_window < GAMMA_PICK_HALF_WINDOW; ++half_window) + for (size_t half_window = 0; half_window <= GAMMA_PICK_HALF_WINDOW; ++half_window) { // end when we have a non empty block uint64_t cum0 = first_block_offset > 0 ? rct_offsets[first_block_offset] - rct_offsets[first_block_offset - 1] : rct_offsets[0]; @@ -6815,19 +6863,24 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> uint64_t cum1 = last_block_offset > 0 ? rct_offsets[last_block_offset] - rct_offsets[last_block_offset - 1] : rct_offsets[0]; if (cum1 > 1) break; - if (first_block_offset == 0 && last_block_offset >= rct_offsets.size() - 2) + if (first_block_offset == 0 && last_block_offset >= last_usable_block) break; // expand up to bounds if (first_block_offset > 0) --first_block_offset; - if (last_block_offset < rct_offsets.size() - 1) + else + return std::numeric_limits<uint64_t>::max(); // bad pick + if (last_block_offset < last_usable_block - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE) ++last_block_offset; + else + return std::numeric_limits<uint64_t>::max(); // bad pick } - const uint64_t n_rct = rct_offsets[last_block_offset] - (first_block_offset == 0 ? 0 : rct_offsets[first_block_offset - 1]); + const uint64_t first_rct = first_block_offset == 0 ? 0 : rct_offsets[first_block_offset - 1]; + const uint64_t n_rct = rct_offsets[last_block_offset] - first_rct; if (n_rct == 0) return rct_offsets[block_offset] ? rct_offsets[block_offset] - 1 : 0; - MDEBUG("Picking 1/" << n_rct << " in " << (last_block_offset - first_block_offset + 1) << " blocks centered around " << block_offset); - return rct_offsets[first_block_offset] + crypto::rand<uint64_t>() % n_rct; + MDEBUG("Picking 1/" << n_rct << " in " << (last_block_offset - first_block_offset + 1) << " blocks centered around " << block_offset + rct_start_height); + return first_rct + crypto::rand<uint64_t>() % n_rct; }; size_t num_selected_transfers = 0; @@ -7166,6 +7219,9 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> break; } } + bool use_histogram = amount != 0 || !has_rct_distribution; + if (!use_histogram) + num_outs = rct_offsets[rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE]; // make sure the real outputs we asked for are really included, along // with the correct key and mask: this guards against an active attack @@ -7258,6 +7314,20 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> outs.push_back(v); } } + + // save those outs in the ringdb for reuse + for (size_t i = 0; i < selected_transfers.size(); ++i) + { + const size_t idx = selected_transfers[i]; + THROW_WALLET_EXCEPTION_IF(idx >= m_transfers.size(), error::wallet_internal_error, "selected_transfers entry out of range"); + const transfer_details &td = m_transfers[idx]; + std::vector<uint64_t> ring; + ring.reserve(outs[i].size()); + for (const auto &e: outs[i]) + ring.push_back(std::get<0>(e)); + if (!set_ring(td.m_key_image, ring, false)) + MERROR("Failed to set ring for " << td.m_key_image); + } } template<typename T> @@ -7929,6 +7999,7 @@ void wallet2::light_wallet_get_unspent_outs() td.m_key_image = unspent_key_image; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = o.amount; td.m_pk_index = 0; @@ -8365,7 +8436,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // early out if we know we can't make it anyway // we could also check for being within FEE_PER_KB, but if the fee calculation // ever changes, this might be missed, so let this go through - const uint64_t min_fee = (fee_multiplier * base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)) / 1024; + const uint64_t min_fee = (fee_multiplier * base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)); uint64_t balance_subtotal = 0; uint64_t unlocked_balance_subtotal = 0; for (uint32_t index_minor : subaddr_indices) @@ -8433,12 +8504,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } } - // shuffle & sort output indices + // sort output indices { - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(unused_transfers_indices_per_subaddr.begin(), unused_transfers_indices_per_subaddr.end(), g); - std::shuffle(unused_dust_indices_per_subaddr.begin(), unused_dust_indices_per_subaddr.end(), g); auto sort_predicate = [&unlocked_balance_per_subaddr] (const std::pair<uint32_t, std::vector<size_t>>& x, const std::pair<uint32_t, std::vector<size_t>>& y) { return unlocked_balance_per_subaddr[x.first] > unlocked_balance_per_subaddr[y.first]; @@ -8630,7 +8697,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp cryptonote::transaction test_tx; pending_tx test_ptx; - needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); @@ -9084,6 +9151,62 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton return ptx_vector; } //---------------------------------------------------------------------------------------------------- +void wallet2::cold_tx_aux_import(const std::vector<pending_tx> & ptx, const std::vector<std::string> & tx_device_aux) +{ + CHECK_AND_ASSERT_THROW_MES(ptx.size() == tx_device_aux.size(), "TX aux has invalid size"); + for (size_t i = 0; i < ptx.size(); ++i){ + crypto::hash txid; + txid = get_transaction_hash(ptx[i].tx); + set_tx_device_aux(txid, tx_device_aux[i]); + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux) +{ + auto & hwdev = get_account().get_device(); + if (!hwdev.has_tx_cold_sign()){ + throw std::invalid_argument("Device does not support cold sign protocol"); + } + + unsigned_tx_set txs; + for (auto &tx: ptx_vector) + { + txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); + } + txs.transfers = std::make_pair(0, m_transfers); + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + hw::tx_aux_data aux_data; + hw::wallet_shim wallet_shim; + setup_shim(&wallet_shim, this); + aux_data.tx_recipients = dsts_info; + dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data); + tx_device_aux = aux_data.tx_device_aux; + + MDEBUG("Signed tx data from hw: " << exported_txs.ptx.size() << " transactions"); + for (auto &c_ptx: exported_txs.ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(c_ptx.tx)); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { + auto & hwdev = get_account().get_device(); + if (!hwdev.has_ki_cold_sync()){ + throw std::invalid_argument("Device does not support cold ki sync protocol"); + } + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + std::vector<std::pair<crypto::key_image, crypto::signature>> ski; + hw::wallet_shim wallet_shim; + setup_shim(&wallet_shim, this); + + dev_cold->ki_sync(&wallet_shim, m_transfers, ski); + + return import_key_images(ski, 0, spent, unspent); +} +//---------------------------------------------------------------------------------------------------- void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const { boost::optional<std::string> result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); @@ -10240,6 +10363,19 @@ std::string wallet2::get_tx_note(const crypto::hash &txid) const return i->second; } +void wallet2::set_tx_device_aux(const crypto::hash &txid, const std::string &aux) +{ + m_tx_device[txid] = aux; +} + +std::string wallet2::get_tx_device_aux(const crypto::hash &txid) const +{ + std::unordered_map<crypto::hash, std::string>::const_iterator i = m_tx_device.find(txid); + if (i == m_tx_device.end()) + return std::string(); + return i->second; +} + void wallet2::set_attribute(const std::string &key, const std::string &value) { m_attributes[key] = value; @@ -10430,31 +10566,45 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename) const { - std::vector<std::pair<crypto::key_image, crypto::signature>> ski = export_key_images(); + PERF_TIMER(export_key_images); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + const uint32_t offset = ski.first; std::string data; + data.reserve(4 + ski.second.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.resize(4); + data[0] = offset & 0xff; + data[1] = (offset >> 8) & 0xff; + data[2] = (offset >> 16) & 0xff; + data[3] = (offset >> 24) & 0xff; data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); - for (const auto &i: ski) + for (const auto &i: ski.second) { data += std::string((const char *)&i.first, sizeof(crypto::key_image)); data += std::string((const char *)&i.second, sizeof(crypto::signature)); } // encrypt data, keep magic plaintext + PERF_TIMER(export_key_images_encrypt); std::string ciphertext = encrypt_with_view_secret_key(data); return epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); } //---------------------------------------------------------------------------------------------------- -std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key_images() const +std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images() const { + PERF_TIMER(export_key_images_raw); std::vector<std::pair<crypto::key_image, crypto::signature>> ski; - ski.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_requested) + ++offset; + + ski.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; @@ -10498,11 +10648,12 @@ std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key ski.push_back(std::make_pair(td.m_key_image, signature)); } - return ski; + return std::make_pair(offset, ski); } uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) { + PERF_TIMER(import_key_images_fsu); std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -10516,6 +10667,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent try { + PERF_TIMER(import_key_images_decrypt); data = decrypt_with_view_secret_key(std::string(data, magiclen)); } catch (const std::exception &e) @@ -10523,15 +10675,17 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to decrypt ") + filename + ": " + e.what()); } - const size_t headerlen = 2 * sizeof(crypto::public_key); + const size_t headerlen = 4 + 2 * sizeof(crypto::public_key); THROW_WALLET_EXCEPTION_IF(data.size() < headerlen, error::wallet_internal_error, std::string("Bad data size from file ") + filename); - const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; - const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; + const uint32_t offset = (uint8_t)data[0] | (((uint8_t)data[1]) << 8) | (((uint8_t)data[2]) << 16) | (((uint8_t)data[3]) << 24); + const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[4]; + const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[4 + sizeof(crypto::public_key)]; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) { THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string( "Key images from ") + filename + " are for a different account"); } + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); const size_t record_size = sizeof(crypto::key_image) + sizeof(crypto::signature); THROW_WALLET_EXCEPTION_IF((data.size() - headerlen) % record_size, @@ -10548,28 +10702,33 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent ski.push_back(std::make_pair(key_image, signature)); } - return import_key_images(ski, spent, unspent); + return import_key_images(ski, offset, spent, unspent); } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) +uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent) { + PERF_TIMER(import_key_images_lots); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); + THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size() - offset, error::wallet_internal_error, "The blockchain is out of date compared to the signed key images"); - if (signed_key_images.empty()) + if (signed_key_images.empty() && offset == 0) { spent = 0; unspent = 0; return 0; } + req.key_images.reserve(signed_key_images.size()); + + PERF_TIMER_START(import_key_images_A); for (size_t n = 0; n < signed_key_images.size(); ++n) { - const transfer_details &td = m_transfers[n]; + const transfer_details &td = m_transfers[n + offset]; const crypto::key_image &key_image = signed_key_images[n].first; const crypto::signature &signature = signed_key_images[n].second; @@ -10580,30 +10739,37 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const cryptonote::txout_to_key &o = boost::get<cryptonote::txout_to_key>(out.target); const crypto::public_key pkey = o.key; - std::vector<const crypto::public_key*> pkeys; - pkeys.push_back(&pkey); - THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), - error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast<std::string>(n) + "/" - + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); - - THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::signature_check_failed, boost::lexical_cast<std::string>(n) + "/" - + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) - + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + if (!td.m_key_image_known || !(key_image == td.m_key_image)) + { + std::vector<const crypto::public_key*> pkeys; + pkeys.push_back(&pkey); + THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), + error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast<std::string>(n + offset) + "/" + + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); + THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), + error::signature_check_failed, boost::lexical_cast<std::string>(n + offset) + "/" + + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } req.key_images.push_back(epee::string_tools::pod_to_hex(key_image)); } + PERF_TIMER_STOP(import_key_images_A); + PERF_TIMER_START(import_key_images_B); for (size_t n = 0; n < signed_key_images.size(); ++n) { - m_transfers[n].m_key_image = signed_key_images[n].first; - m_key_images[m_transfers[n].m_key_image] = n; - m_transfers[n].m_key_image_known = true; - m_transfers[n].m_key_image_partial = false; + m_transfers[n + offset].m_key_image = signed_key_images[n].first; + m_key_images[m_transfers[n + offset].m_key_image] = n + offset; + m_transfers[n + offset].m_key_image_known = true; + m_transfers[n + offset].m_key_image_requested = false; + m_transfers[n + offset].m_key_image_partial = false; } + PERF_TIMER_STOP(import_key_images_B); if(check_spent) { + PERF_TIMER(import_key_images_RPC); m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -10615,7 +10781,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag std::to_string(daemon_resp.spent_status.size()) + ", expected " + std::to_string(signed_key_images.size())); for (size_t n = 0; n < daemon_resp.spent_status.size(); ++n) { - transfer_details &td = m_transfers[n]; + transfer_details &td = m_transfers[n + offset]; td.m_spent = daemon_resp.spent_status[n] != COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT; } } @@ -10626,6 +10792,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag // was created by sweep_all, so we can't know the spent height and other detailed info. std::unordered_map<crypto::key_image, crypto::hash> spent_key_images; + PERF_TIMER_START(import_key_images_C); for (const transfer_details &td: m_transfers) { for (const cryptonote::txin_v& in : td.m_tx.vin) @@ -10634,10 +10801,12 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag spent_key_images.insert(std::make_pair(boost::get<cryptonote::txin_to_key>(in).k_image, td.m_txid)); } } + PERF_TIMER_STOP(import_key_images_C); + PERF_TIMER_START(import_key_images_D); for(size_t i = 0; i < signed_key_images.size(); ++i) { - transfer_details &td = m_transfers[i]; + const transfer_details &td = m_transfers[i + offset]; uint64_t amount = td.amount(); if (td.m_spent) spent += amount; @@ -10655,6 +10824,8 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag spent_txids.insert(skii->second); } } + PERF_TIMER_STOP(import_key_images_D); + MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); if (check_spent) @@ -10664,8 +10835,12 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag COMMAND_RPC_GET_TRANSACTIONS::response gettxs_res; gettxs_req.decode_as_json = false; gettxs_req.prune = false; + gettxs_req.txs_hashes.reserve(spent_txids.size()); for (const crypto::hash& spent_txid : spent_txids) gettxs_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(spent_txid)); + + + PERF_TIMER_START(import_key_images_E); m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -10673,8 +10848,10 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag THROW_WALLET_EXCEPTION_IF(gettxs_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); THROW_WALLET_EXCEPTION_IF(gettxs_res.txs.size() != spent_txids.size(), error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong count = " + std::to_string(gettxs_res.txs.size()) + ", expected " + std::to_string(spent_txids.size())); + PERF_TIMER_STOP(import_key_images_E); // process each outgoing tx + PERF_TIMER_START(import_key_images_F); auto spent_txid = spent_txids.begin(); hw::device &hwdev = m_account.get_device(); for (const COMMAND_RPC_GET_TRANSACTIONS::entry& e : gettxs_res.txs) @@ -10770,7 +10947,9 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag ++spent_txid; } + PERF_TIMER_STOP(import_key_images_F); + PERF_TIMER_START(import_key_images_G); for (size_t n : swept_transfers) { const transfer_details& td = m_transfers[n]; @@ -10781,10 +10960,35 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const crypto::hash &spent_txid = crypto::null_hash; // spent txid is unknown m_confirmed_txs.insert(std::make_pair(spent_txid, pd)); } + PERF_TIMER_STOP(import_key_images_G); } return m_transfers[signed_key_images.size() - 1].m_block_height; } + +bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) +{ + if (key_images.size() > m_transfers.size()) + { + LOG_PRINT_L1("More key images returned that we know outputs for"); + return false; + } + for (size_t i = 0; i < key_images.size(); ++i) + { + transfer_details &td = m_transfers[i]; + if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[i]) + LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); + td.m_key_image = key_images[i]; + m_key_images[m_transfers[i].m_key_image] = i; + td.m_key_image_known = true; + td.m_key_image_requested = false; + td.m_key_image_partial = false; + m_pub_keys[m_transfers[i].get_public_key()] = i; + } + + return true; +} + wallet2::payment_container wallet2::export_payments() const { payment_container payments; @@ -10843,50 +11047,87 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } //---------------------------------------------------------------------------------------------------- -std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const +std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs() const { + PERF_TIMER(export_outputs); std::vector<tools::wallet2::transfer_details> outs; - outs.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && m_transfers[offset].m_key_image_known) + ++offset; + + outs.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; outs.push_back(td); } - return outs; + return std::make_pair(offset, outs); } //---------------------------------------------------------------------------------------------------- std::string wallet2::export_outputs_to_str() const { - std::vector<tools::wallet2::transfer_details> outs = export_outputs(); + PERF_TIMER(export_outputs_to_str); std::stringstream oss; boost::archive::portable_binary_oarchive ar(oss); - ar << outs; + const auto& outputs = export_outputs(); + ar << outputs; std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string header; header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); + PERF_TIMER(export_outputs_encryption); std::string ciphertext = encrypt_with_view_secret_key(header + oss.str()); return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs) +size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs) { - m_transfers.clear(); - m_transfers.reserve(outputs.size()); - for (size_t i = 0; i < outputs.size(); ++i) + PERF_TIMER(import_outputs); + + THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error, + "Imported outputs omit more outputs that we know of"); + + const size_t offset = outputs.first; + const size_t original_size = m_transfers.size(); + m_transfers.resize(offset + outputs.second.size()); + for (size_t i = 0; i < offset; ++i) + m_transfers[i].m_key_image_requested = false; + for (size_t i = 0; i < outputs.second.size(); ++i) { - transfer_details td = outputs[i]; + transfer_details td = outputs.second[i]; + + // skip those we've already imported, or which have different data + if (i + offset < original_size) + { + // compare the data used to create the key image below + const transfer_details &org_td = m_transfers[i + offset]; + if (!org_td.m_key_image_known) + goto process; +#define CMPF(f) if (!(td.f == org_td.f)) goto process + CMPF(m_txid); + CMPF(m_key_image); + CMPF(m_internal_output_index); +#undef CMPF + if (!(get_transaction_prefix_hash(td.m_tx) == get_transaction_prefix_hash(org_td.m_tx))) + goto process; + + // copy anyway, since the comparison does not include ancillary fields which may have changed + m_transfers[i + offset] = std::move(td); + continue; + } + +process: // the hot wallet wouldn't have known about key images (except if we already exported them) cryptonote::keypair in_ephemeral; - THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast<std::string>(i)); + THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast<std::string>(i + offset)); crypto::public_key tx_pub_key = get_tx_pub_key_from_received_outs(td); const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); @@ -10897,13 +11138,14 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; + td.m_key_image_requested = true; td.m_key_image_partial = false; THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, - error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i)); + error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i + offset)); - m_key_images[td.m_key_image] = m_transfers.size(); - m_pub_keys[td.get_public_key()] = m_transfers.size(); - m_transfers.push_back(std::move(td)); + m_key_images[td.m_key_image] = i + offset; + m_pub_keys[td.get_public_key()] = i + offset; + m_transfers[i + offset] = std::move(td); } return m_transfers.size(); @@ -10911,6 +11153,7 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail //---------------------------------------------------------------------------------------------------- size_t wallet2::import_outputs_from_str(const std::string &outputs_st) { + PERF_TIMER(import_outputs_from_str); std::string data = outputs_st; const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC); if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen)) @@ -10920,6 +11163,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) try { + PERF_TIMER(import_outputs_decrypt); data = decrypt_with_view_secret_key(std::string(data, magiclen)); } catch (const std::exception &e) @@ -10946,7 +11190,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) std::string body(data, headerlen); std::stringstream iss; iss << body; - std::vector<tools::wallet2::transfer_details> outputs; + std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; try { boost::archive::portable_binary_iarchive ar(iss); @@ -11138,6 +11382,7 @@ void wallet2::update_multisig_rescan_info(const std::vector<std::vector<rct::key m_key_images.erase(td.m_key_image); td.m_key_image = get_multisig_composite_key_image(n); td.m_key_image_known = true; + td.m_key_image_requested = false; td.m_key_image_partial = false; td.m_multisig_k = multisig_k[n]; m_key_images[td.m_key_image] = n; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 680196f01..eb0763131 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -54,6 +54,7 @@ #include "ringct/rctTypes.h" #include "ringct/rctOps.h" #include "checkpoints/checkpoints.h" +#include "serialization/pair.h" #include "wallet_errors.h" #include "common/password.h" @@ -229,7 +230,7 @@ namespace tools bool error; boost::optional<cryptonote::subaddress_receive_info> received; - tx_scan_info_t(): money_transfered(0), error(true) {} + tx_scan_info_t(): amount(0), money_transfered(0), error(true) {} }; struct transfer_details @@ -246,6 +247,7 @@ namespace tools uint64_t m_amount; bool m_rct; bool m_key_image_known; + bool m_key_image_requested; size_t m_pk_index; cryptonote::subaddress_index m_subaddr_index; bool m_key_image_partial; @@ -269,6 +271,7 @@ namespace tools FIELD(m_amount) FIELD(m_rct) FIELD(m_key_image_known) + FIELD(m_key_image_requested) FIELD(m_pk_index) FIELD(m_subaddr_index) FIELD(m_key_image_partial) @@ -416,7 +419,7 @@ namespace tools struct unsigned_tx_set { std::vector<tx_construction_data> txes; - wallet2::transfer_container transfers; + std::pair<size_t, wallet2::transfer_container> transfers; }; struct signed_tx_set @@ -764,6 +767,9 @@ namespace tools std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux); + void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux); + uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent); bool load_multisig_tx(cryptonote::blobdata blob, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool load_multisig_tx_from_file(const std::string &filename, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool sign_multisig_tx_from_file(const std::string &filename, std::vector<crypto::hash> &txids, std::function<bool(const multisig_tx_set&)> accept_func); @@ -782,7 +788,7 @@ namespace tools uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); - void rescan_blockchain(bool refresh = true); + void rescan_blockchain(bool hard, bool refresh = true); bool is_transfer_unlocked(const transfer_details& td) const; bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const; @@ -892,6 +898,9 @@ namespace tools if(ver < 25) return; a & m_last_block_reward; + if(ver < 26) + return; + a & m_tx_device; } /*! @@ -1020,6 +1029,9 @@ namespace tools void set_tx_note(const crypto::hash &txid, const std::string ¬e); std::string get_tx_note(const crypto::hash &txid) const; + void set_tx_device_aux(const crypto::hash &txid, const std::string &aux); + std::string get_tx_device_aux(const crypto::hash &txid) const; + void set_description(const std::string &description); std::string get_description() const; @@ -1061,9 +1073,9 @@ namespace tools bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; // Import/Export wallet data - std::vector<tools::wallet2::transfer_details> export_outputs() const; + std::pair<size_t, std::vector<tools::wallet2::transfer_details>> export_outputs() const; std::string export_outputs_to_str() const; - size_t import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs); + size_t import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs); size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); @@ -1071,9 +1083,11 @@ namespace tools std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); bool export_key_images(const std::string &filename) const; - std::vector<std::pair<crypto::key_image, crypto::signature>> export_key_images() const; - uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent = true); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images() const; + uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); + bool import_key_images(std::vector<crypto::key_image> key_images); + crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; void update_pool_state(bool refreshed = false); void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes); @@ -1187,6 +1201,8 @@ namespace tools void set_tx_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_tx_notify = notify; } + bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; + private: /*! * \brief Stores wallet information to wallet file. @@ -1206,7 +1222,6 @@ namespace tools void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset); void detach_blockchain(uint64_t height); void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; - bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; bool clear(); void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); @@ -1236,7 +1251,6 @@ namespace tools void set_unspent(size_t idx); void get_outs(std::vector<std::vector<get_outs_entry>> &outs, const std::vector<size_t> &selected_transfers, size_t fake_outputs_count); bool tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const; - crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; std::vector<size_t> get_only_rct(const std::vector<size_t> &unused_dust_indices, const std::vector<size_t> &unused_transfers_indices) const; void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs); @@ -1253,6 +1267,9 @@ namespace tools crypto::chacha_key get_ringdb_key(); void setup_keys(const epee::wipeable_string &password); + void register_devices(); + hw::device& lookup_device(const std::string & device_descriptor); + bool get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); uint64_t get_segregation_fork_height() const; @@ -1347,6 +1364,9 @@ namespace tools size_t m_subaddress_lookahead_major, m_subaddress_lookahead_minor; std::string m_device_name; + // Aux transaction data from device + std::unordered_map<crypto::hash, std::string> m_tx_device; + // Light wallet bool m_light_wallet; /* sends view key to daemon for scanning */ uint64_t m_light_wallet_scanned_block_height; @@ -1373,12 +1393,13 @@ namespace tools boost::optional<epee::wipeable_string> m_encrypt_keys_after_refresh; bool m_unattended; + bool m_devices_registered; std::shared_ptr<tools::Notify> m_tx_notify; }; } -BOOST_CLASS_VERSION(tools::wallet2, 25) -BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) +BOOST_CLASS_VERSION(tools::wallet2, 26) +BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 10) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) @@ -1436,6 +1457,10 @@ namespace boost x.m_multisig_k.clear(); x.m_multisig_info.clear(); } + if (ver < 10) + { + x.m_key_image_requested = false; + } } template <class Archive> @@ -1517,6 +1542,12 @@ namespace boost a & x.m_multisig_info; a & x.m_multisig_k; a & x.m_key_image_partial; + if (ver < 10) + { + initialize_transfer_details(a, x, ver); + return; + } + a & x.m_key_image_requested; } template <class Archive> diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 5e6100dfd..533784acf 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -982,7 +982,11 @@ namespace tools { res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); if (req.get_tx_keys) + { res.tx_key_list.push_back(epee::string_tools::pod_to_hex(ptx.tx_key)); + for (const crypto::secret_key& additional_tx_key : ptx.additional_tx_keys) + res.tx_key_list.back() += epee::string_tools::pod_to_hex(additional_tx_key); + } } if (req.export_raw) @@ -1781,7 +1785,7 @@ namespace tools try { - m_wallet->rescan_blockchain(); + m_wallet->rescan_blockchain(req.hard); } catch (const std::exception& e) { @@ -2460,12 +2464,13 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::vector<std::pair<crypto::key_image, crypto::signature>> ski = m_wallet->export_key_images(); - res.signed_key_images.resize(ski.size()); - for (size_t n = 0; n < ski.size(); ++n) + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(); + res.offset = ski.first; + res.signed_key_images.resize(ski.second.size()); + for (size_t n = 0; n < ski.second.size(); ++n) { - res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski[n].first); - res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski[n].second); + res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski.second[n].first); + res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski.second[n].second); } } @@ -2518,7 +2523,7 @@ namespace tools ski[n].second = *reinterpret_cast<const crypto::signature*>(bd.data()); } uint64_t spent = 0, unspent = 0; - uint64_t height = m_wallet->import_key_images(ski, spent, unspent); + uint64_t height = m_wallet->import_key_images(ski, req.offset, spent, unspent); res.spent = spent; res.unspent = unspent; res.height = height; @@ -3090,6 +3095,200 @@ namespace tools } } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request &req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response &res, epee::json_rpc::error &er) + { + if (m_wallet_dir.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_NO_WALLET_DIR; + er.message = "No wallet dir configured"; + return false; + } + + // early check for mandatory fields + if (req.filename.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'filename' is mandatory. Please provide a filename to save the restored wallet to."; + return false; + } + if (req.seed.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'seed' is mandatory. Please provide a seed you want to restore from."; + return false; + } + + namespace po = boost::program_options; + po::variables_map vm2; + const char *ptr = strchr(req.filename.c_str(), '/'); + #ifdef _WIN32 + if (!ptr) + ptr = strchr(req.filename.c_str(), '\\'); + if (!ptr) + ptr = strchr(req.filename.c_str(), ':'); + #endif + if (ptr) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Invalid filename"; + return false; + } + std::string wallet_file = m_wallet_dir + "/" + req.filename; + // check if wallet file already exists + if (!wallet_file.empty()) + { + try + { + boost::system::error_code ignored_ec; + THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(wallet_file, ignored_ec), error::file_exists, wallet_file); + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet already exists."; + return false; + } + } + crypto::secret_key recovery_key; + std::string old_language; + + // check the given seed + { + if (!crypto::ElectrumWords::words_to_bytes(req.seed, recovery_key, old_language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Electrum-style word list failed verification"; + return false; + } + } + + // process seed_offset if given + { + if (!req.seed_offset.empty()) + { + recovery_key = cryptonote::decrypt_key(recovery_key, req.seed_offset); + } + } + { + po::options_description desc("dummy"); + const command_line::arg_descriptor<std::string, true> arg_password = {"password", "password"}; + const char *argv[4]; + int argc = 3; + argv[0] = "wallet-rpc"; + argv[1] = "--password"; + argv[2] = req.password.c_str(); + argv[3] = NULL; + vm2 = *m_vm; + command_line::add_arg(desc, arg_password); + po::store(po::parse_command_line(argc, argv, desc), vm2); + } + + auto rc = tools::wallet2::make_new(vm2, true, nullptr); + std::unique_ptr<wallet2> wal; + wal = std::move(rc.first); + if (!wal) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to create wallet"; + return false; + } + + epee::wipeable_string password = rc.second.password(); + + bool was_deprecated_wallet = ((old_language == crypto::ElectrumWords::old_language_name) || + crypto::ElectrumWords::get_is_old_style_seed(req.seed)); + + std::string mnemonic_language = old_language; + if (was_deprecated_wallet) + { + // The user had used an older version of the wallet with old style mnemonics. + res.was_deprecated = true; + } + + if (old_language == crypto::ElectrumWords::old_language_name) + { + if (req.language.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet was using the old seed language. You need to specify a new seed language."; + return false; + } + std::vector<std::string> language_list; + std::vector<std::string> language_list_en; + crypto::ElectrumWords::get_language_list(language_list); + crypto::ElectrumWords::get_language_list(language_list_en, true); + if (std::find(language_list.begin(), language_list.end(), req.language) == language_list.end() && + std::find(language_list_en.begin(), language_list_en.end(), req.language) == language_list_en.end()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet was using the old seed language, and the specified new seed language is invalid."; + return false; + } + mnemonic_language = req.language; + } + + wal->set_seed_language(mnemonic_language); + + crypto::secret_key recovery_val; + try + { + recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false); + MINFO("Wallet has been restored.\n"); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + // // Convert the secret key back to seed + epee::wipeable_string electrum_words; + if (!crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to encode seed"; + return false; + } + res.seed = electrum_words.data(); + + if (!wal) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to generate wallet"; + return false; + } + + // set blockheight if given + try + { + wal->set_refresh_from_block_height(req.restore_height); + wal->rewrite(wallet_file, password); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + delete m_wallet; + } + m_wallet = wal.release(); + res.address = m_wallet->get_account().get_public_address_str(m_wallet->nettype()); + res.info = "Wallet has been restored successfully."; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 887723ed5..abbbe82c5 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -137,6 +137,7 @@ namespace tools MAP_JON_RPC_WE("open_wallet", on_open_wallet, wallet_rpc::COMMAND_RPC_OPEN_WALLET) MAP_JON_RPC_WE("close_wallet", on_close_wallet, wallet_rpc::COMMAND_RPC_CLOSE_WALLET) MAP_JON_RPC_WE("change_wallet_password", on_change_wallet_password, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD) + MAP_JON_RPC_WE("restore_deterministic_wallet", on_restore_deterministic_wallet, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET) MAP_JON_RPC_WE("is_multisig", on_is_multisig, wallet_rpc::COMMAND_RPC_IS_MULTISIG) MAP_JON_RPC_WE("prepare_multisig", on_prepare_multisig, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG) MAP_JON_RPC_WE("make_multisig", on_make_multisig, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG) @@ -216,6 +217,7 @@ namespace tools bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er); bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er); bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er); + bool on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request& req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response& res, epee::json_rpc::error& er); bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er); bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er); bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 924f3a0f1..afb8c6e91 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 5 +#define WALLET_RPC_VERSION_MINOR 6 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -1057,7 +1057,10 @@ namespace wallet_rpc { struct request { + bool hard; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(hard, false); END_KV_SERIALIZE_MAP() }; @@ -1579,9 +1582,11 @@ namespace wallet_rpc struct response { + uint32_t offset; std::vector<signed_key_image> signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(offset); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; @@ -1602,9 +1607,11 @@ namespace wallet_rpc struct request { + uint32_t offset; std::vector<signed_key_image> signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(offset, (uint32_t)0); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; @@ -1924,6 +1931,43 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET + { + struct request + { + uint64_t restore_height; + std::string filename; + std::string seed; + std::string seed_offset; + std::string password; + std::string language; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(restore_height, (uint64_t)0) + KV_SERIALIZE(filename) + KV_SERIALIZE(seed) + KV_SERIALIZE(seed_offset) + KV_SERIALIZE(password) + KV_SERIALIZE(language) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string address; + std::string seed; + std::string info; + bool was_deprecated; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(seed) + KV_SERIALIZE(info) + KV_SERIALIZE(was_deprecated) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_IS_MULTISIG { struct request diff --git a/tests/data/fuzz/cold-outputs/OUTPUTS2 b/tests/data/fuzz/cold-outputs/OUTPUTS2 Binary files differindex 907bcdb91..33cf39024 100644 --- a/tests/data/fuzz/cold-outputs/OUTPUTS2 +++ b/tests/data/fuzz/cold-outputs/OUTPUTS2 diff --git a/tests/fuzz/cold-outputs.cpp b/tests/fuzz/cold-outputs.cpp index 488a3b931..29b3ed267 100644 --- a/tests/fuzz/cold-outputs.cpp +++ b/tests/fuzz/cold-outputs.cpp @@ -77,7 +77,7 @@ int ColdOutputsFuzzer::run(const std::string &filename) s = std::string("\x01\x16serialization::archive") + s; try { - std::vector<tools::wallet2::transfer_details> outputs; + std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; std::stringstream iss; iss << s; boost::archive::portable_binary_iarchive ar(iss); diff --git a/tests/libwallet_api_tests/CMakeLists.txt b/tests/libwallet_api_tests/CMakeLists.txt index ef1b666ed..1a9cbc5a6 100644 --- a/tests/libwallet_api_tests/CMakeLists.txt +++ b/tests/libwallet_api_tests/CMakeLists.txt @@ -50,6 +50,8 @@ target_link_libraries(libwallet_api_tests ${Boost_FILESYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${Boost_SYSTEM_LIBRARY} + ${Boost_LOCALE_LIBRARY} + ${ICU_LIBRARIES} ${GTEST_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBRARIES}) diff --git a/tests/performance_tests/CMakeLists.txt b/tests/performance_tests/CMakeLists.txt index 837d39bd3..5cd054d86 100644 --- a/tests/performance_tests/CMakeLists.txt +++ b/tests/performance_tests/CMakeLists.txt @@ -46,6 +46,8 @@ set(performance_tests_headers range_proof.h bulletproof.h crypto_ops.h + sc_reduce32.h + sc_check.h multiexp.h multi_tx_test_base.h performance_tests.h diff --git a/tests/performance_tests/crypto_ops.h b/tests/performance_tests/crypto_ops.h index 3c68583c5..4766a1205 100644 --- a/tests/performance_tests/crypto_ops.h +++ b/tests/performance_tests/crypto_ops.h @@ -47,6 +47,7 @@ enum test_op op_scalarmultKey, op_scalarmultH, op_scalarmult8, + op_ge_dsm_precomp, op_ge_double_scalarmult_base_vartime, op_ge_double_scalarmult_precomp_vartime, op_ge_double_scalarmult_precomp_vartime2, @@ -84,6 +85,7 @@ public: ge_cached tmp_cached; ge_p1p1 tmp_p1p1; ge_p2 tmp_p2; + ge_dsmp dsmp; switch (op) { case op_sc_add: sc_add(key.bytes, scalar0.bytes, scalar1.bytes); break; @@ -101,6 +103,7 @@ public: case op_scalarmultKey: rct::scalarmultKey(point0, scalar0); break; case op_scalarmultH: rct::scalarmultH(scalar0); break; case op_scalarmult8: rct::scalarmult8(point0); break; + case op_ge_dsm_precomp: ge_dsm_precomp(dsmp, &p3_0); break; case op_ge_double_scalarmult_base_vartime: ge_double_scalarmult_base_vartime(&tmp_p2, scalar0.bytes, &p3_0, scalar1.bytes); break; case op_ge_double_scalarmult_precomp_vartime: ge_double_scalarmult_precomp_vartime(&tmp_p2, scalar0.bytes, &p3_0, scalar1.bytes, precomp0); break; case op_ge_double_scalarmult_precomp_vartime2: ge_double_scalarmult_precomp_vartime2(&tmp_p2, scalar0.bytes, precomp0, scalar1.bytes, precomp1); break; diff --git a/tests/performance_tests/ge_frombytes_vartime.h b/tests/performance_tests/ge_frombytes_vartime.h index ef9625d6b..3f7d55182 100644 --- a/tests/performance_tests/ge_frombytes_vartime.h +++ b/tests/performance_tests/ge_frombytes_vartime.h @@ -49,22 +49,29 @@ public: if (!base_class::init()) return false; + cryptonote::account_base m_alice; + cryptonote::transaction m_tx; + m_alice.generate(); std::vector<tx_destination_entry> destinations; destinations.push_back(tx_destination_entry(1, m_alice.get_keys().m_account_address, false)); - return construct_tx(this->m_miners[this->real_source_idx].get_keys(), this->m_sources, destinations, boost::none, std::vector<uint8_t>(), m_tx, 0); + if (!construct_tx(this->m_miners[this->real_source_idx].get_keys(), this->m_sources, destinations, boost::none, std::vector<uint8_t>(), m_tx, 0)) + return false; + + const cryptonote::txin_to_key& txin = boost::get<cryptonote::txin_to_key>(m_tx.vin[0]); + m_key = rct::ki2rct(txin.k_image); + + return true; } bool test() { ge_p3 unp; - const cryptonote::txin_to_key& txin = boost::get<cryptonote::txin_to_key>(m_tx.vin[0]); - return ge_frombytes_vartime(&unp, (const unsigned char*) &txin.k_image) == 0; + return ge_frombytes_vartime(&unp, (const unsigned char*) &m_key) == 0; } private: - cryptonote::account_base m_alice; - cryptonote::transaction m_tx; + rct::key m_key; }; diff --git a/tests/performance_tests/ge_tobytes.h b/tests/performance_tests/ge_tobytes.h new file mode 100644 index 000000000..3d46f4838 --- /dev/null +++ b/tests/performance_tests/ge_tobytes.h @@ -0,0 +1,79 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "crypto/crypto.h" +#include "cryptonote_basic/cryptonote_basic.h" + +#include "single_tx_test_base.h" + +class test_ge_tobytes : public multi_tx_test_base<1> +{ +public: + static const size_t loop_count = 10000; + + typedef multi_tx_test_base<1> base_class; + + bool init() + { + using namespace cryptonote; + + if (!base_class::init()) + return false; + + cryptonote::account_base m_alice; + cryptonote::transaction m_tx; + + m_alice.generate(); + + std::vector<tx_destination_entry> destinations; + destinations.push_back(tx_destination_entry(1, m_alice.get_keys().m_account_address, false)); + + if (!construct_tx(this->m_miners[this->real_source_idx].get_keys(), this->m_sources, destinations, boost::none, std::vector<uint8_t>(), m_tx, 0)) + return false; + + const cryptonote::txin_to_key& txin = boost::get<cryptonote::txin_to_key>(m_tx.vin[0]); + if (ge_frombytes_vartime(&m_p3, (const unsigned char*) &txin.k_image) != 0) + return false; + + return true; + } + + bool test() + { + rct::key key; + ge_p3_tobytes(key.bytes, &m_p3); + return true; + } + +private: + ge_p3 m_p3; +}; diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index 87a1573c2..58daf6220 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -42,6 +42,7 @@ #include "derive_public_key.h" #include "derive_secret_key.h" #include "ge_frombytes_vartime.h" +#include "ge_tobytes.h" #include "generate_key_derivation.h" #include "generate_key_image.h" #include "generate_key_image_helper.h" @@ -50,6 +51,7 @@ #include "is_out_to_acc.h" #include "subaddress_expand.h" #include "sc_reduce32.h" +#include "sc_check.h" #include "cn_fast_hash.h" #include "rct_mlsag.h" #include "equality.h" @@ -182,8 +184,10 @@ int main(int argc, char** argv) TEST_PERFORMANCE0(filter, p, test_derive_public_key); TEST_PERFORMANCE0(filter, p, test_derive_secret_key); TEST_PERFORMANCE0(filter, p, test_ge_frombytes_vartime); + TEST_PERFORMANCE0(filter, p, test_ge_tobytes); TEST_PERFORMANCE0(filter, p, test_generate_keypair); TEST_PERFORMANCE0(filter, p, test_sc_reduce32); + TEST_PERFORMANCE0(filter, p, test_sc_check); TEST_PERFORMANCE1(filter, p, test_signature, false); TEST_PERFORMANCE1(filter, p, test_signature, true); @@ -249,6 +253,7 @@ int main(int argc, char** argv) TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmultKey); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmultH); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmult8); + TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_dsm_precomp); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_double_scalarmult_base_vartime); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_double_scalarmult_precomp_vartime); TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_double_scalarmult_precomp_vartime2); diff --git a/tests/performance_tests/multiexp.h b/tests/performance_tests/multiexp.h index b8b87b3a6..b6e86ebd4 100644 --- a/tests/performance_tests/multiexp.h +++ b/tests/performance_tests/multiexp.h @@ -78,9 +78,9 @@ public: case multiexp_straus_cached: return res == straus(data, straus_cache); case multiexp_pippenger: - return res == pippenger(data, NULL, c); + return res == pippenger(data, NULL, 0, c); case multiexp_pippenger_cached: - return res == pippenger(data, pippenger_cache, c); + return res == pippenger(data, pippenger_cache, 0, c); default: return false; } diff --git a/tests/performance_tests/sc_check.h b/tests/performance_tests/sc_check.h new file mode 100644 index 000000000..036abf12d --- /dev/null +++ b/tests/performance_tests/sc_check.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "crypto/crypto.h" + +class test_sc_check +{ +public: + static const size_t loop_count = 10000000; + + bool init() + { + m_scalar = crypto::rand<crypto::ec_scalar>(); + return true; + } + + bool test() + { + sc_check((unsigned char*)m_scalar.data); + return true; + } + +private: + crypto::ec_scalar m_scalar; +}; diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index 47177db1c..fc488bb14 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -118,7 +118,7 @@ public: virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); } virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const { return false; } - virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) {} + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) {} virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& details) {} virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const { return 0; } virtual bool txpool_has_tx(const crypto::hash &txid) const { return false; } diff --git a/tests/unit_tests/keccak.cpp b/tests/unit_tests/keccak.cpp index 4276b0e1d..37da65d76 100644 --- a/tests/unit_tests/keccak.cpp +++ b/tests/unit_tests/keccak.cpp @@ -37,7 +37,7 @@ extern "C" { #define TEST_KECCAK(sz, chunks) \ std::string data; \ data.resize(sz); \ - for (size_t i = 0; i < sz; ++i) \ + for (size_t i = 0; i < data.size(); ++i) \ data[i] = i * 17; \ uint8_t md0[32], md1[32]; \ keccak((const uint8_t*)data.data(), data.size(), md0, 32); \ diff --git a/tests/unit_tests/notify.cpp b/tests/unit_tests/notify.cpp index 84861cb45..4daeeddee 100644 --- a/tests/unit_tests/notify.cpp +++ b/tests/unit_tests/notify.cpp @@ -44,7 +44,14 @@ TEST(notify, works) #ifdef __GLIBC__ mode_t prevmode = umask(077); #endif - char name_template[] = "/tmp/monero-notify-unit-test-XXXXXX"; + const char *tmp = getenv("TEMP"); + if (!tmp) + tmp = "/tmp"; + static const char *filename = "monero-notify-unit-test-XXXXXX"; + const size_t len = strlen(tmp) + 1 + strlen(filename); + char *name_template = (char*)malloc(len + 1); + ASSERT_TRUE(name_template != NULL); + snprintf(name_template, len + 1, "%s/%s", tmp, filename); int fd = mkstemp(name_template); #ifdef __GLIBC__ umask(prevmode); @@ -68,4 +75,5 @@ TEST(notify, works) ASSERT_TRUE(s == "1111111111111111111111111111111111111111111111111111111111111111"); boost::filesystem::remove(name_template); + free(name_template); } diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 2f7b5aac7..b4517af2f 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -908,6 +908,17 @@ TEST(Serialization, portability_outputs) ASSERT_TRUE(td2.m_pk_index == 0); } +struct unsigned_tx_set +{ + std::vector<tools::wallet2::tx_construction_data> txes; + tools::wallet2::transfer_container transfers; +}; +template <class Archive> +inline void serialize(Archive &a, unsigned_tx_set &x, const boost::serialization::version_type ver) +{ + a & x.txes; + a & x.transfers; +} #define UNSIGNED_TX_PREFIX "Monero unsigned tx set\003" TEST(Serialization, portability_unsigned_tx) { @@ -918,7 +929,7 @@ TEST(Serialization, portability_unsigned_tx) ASSERT_TRUE(r); const size_t magiclen = strlen(UNSIGNED_TX_PREFIX); ASSERT_FALSE(strncmp(s.c_str(), UNSIGNED_TX_PREFIX, magiclen)); - tools::wallet2::unsigned_tx_set exported_txs; + unsigned_tx_set exported_txs; s = s.substr(magiclen); r = false; try |