diff options
Diffstat (limited to 'tests/performance_tests')
27 files changed, 96 insertions, 99 deletions
diff --git a/tests/performance_tests/CMakeLists.txt b/tests/performance_tests/CMakeLists.txt index 5cd054d86..aa424da80 100644 --- a/tests/performance_tests/CMakeLists.txt +++ b/tests/performance_tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014-2018, The Monero Project +# Copyright (c) 2014-2019, The Monero Project # # All rights reserved. # diff --git a/tests/performance_tests/bulletproof.h b/tests/performance_tests/bulletproof.h index 7bb702c3d..7136fbbfe 100644 --- a/tests/performance_tests/bulletproof.h +++ b/tests/performance_tests/bulletproof.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/check_tx_signature.h b/tests/performance_tests/check_tx_signature.h index 755d8f941..329714c56 100644 --- a/tests/performance_tests/check_tx_signature.h +++ b/tests/performance_tests/check_tx_signature.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/cn_fast_hash.h b/tests/performance_tests/cn_fast_hash.h index 2ddaef269..63b2c5bee 100644 --- a/tests/performance_tests/cn_fast_hash.h +++ b/tests/performance_tests/cn_fast_hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/cn_slow_hash.h b/tests/performance_tests/cn_slow_hash.h index b484afa41..4f410d62d 100644 --- a/tests/performance_tests/cn_slow_hash.h +++ b/tests/performance_tests/cn_slow_hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -34,6 +34,7 @@ #include "crypto/crypto.h" #include "cryptonote_basic/cryptonote_basic.h" +template<unsigned int variant> class test_cn_slow_hash { public: @@ -42,18 +43,15 @@ public: #pragma pack(push, 1) struct data_t { - char data[13]; + char data[43]; }; #pragma pack(pop) - static_assert(13 == sizeof(data_t), "Invalid structure size"); + static_assert(43 == sizeof(data_t), "Invalid structure size"); bool init() { - if (!epee::string_tools::hex_to_pod("63617665617420656d70746f72", m_data)) - return false; - - if (!epee::string_tools::hex_to_pod("bbec2cacf69866a8e740380fe7b818fc78f8571221742d729d9d02d7f8989b87", m_expected_hash)) + if (!epee::string_tools::hex_to_pod("63617665617420656d70746f763617665617420656d70746f72263617665617420656d70746f7201020304", m_data)) return false; return true; @@ -62,11 +60,10 @@ public: bool test() { crypto::hash hash; - crypto::cn_slow_hash(&m_data, sizeof(m_data), hash); - return hash == m_expected_hash; + crypto::cn_slow_hash(&m_data, sizeof(m_data), hash, variant); + return true; } private: data_t m_data; - crypto::hash m_expected_hash; }; diff --git a/tests/performance_tests/construct_tx.h b/tests/performance_tests/construct_tx.h index 71d42073d..ebfe46edf 100644 --- a/tests/performance_tests/construct_tx.h +++ b/tests/performance_tests/construct_tx.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/crypto_ops.h b/tests/performance_tests/crypto_ops.h index 3ebb6f470..5b215cef4 100644 --- a/tests/performance_tests/crypto_ops.h +++ b/tests/performance_tests/crypto_ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/derive_public_key.h b/tests/performance_tests/derive_public_key.h index 35c4ff062..ad0b3b55f 100644 --- a/tests/performance_tests/derive_public_key.h +++ b/tests/performance_tests/derive_public_key.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/derive_secret_key.h b/tests/performance_tests/derive_secret_key.h index c478a2a99..ba01b57c0 100644 --- a/tests/performance_tests/derive_secret_key.h +++ b/tests/performance_tests/derive_secret_key.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/equality.h b/tests/performance_tests/equality.h index 8d24d7da7..5788ebec8 100644 --- a/tests/performance_tests/equality.h +++ b/tests/performance_tests/equality.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/ge_frombytes_vartime.h b/tests/performance_tests/ge_frombytes_vartime.h index 3f7d55182..1a0601c74 100644 --- a/tests/performance_tests/ge_frombytes_vartime.h +++ b/tests/performance_tests/ge_frombytes_vartime.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/ge_tobytes.h b/tests/performance_tests/ge_tobytes.h index 3d46f4838..66e6fce38 100644 --- a/tests/performance_tests/ge_tobytes.h +++ b/tests/performance_tests/ge_tobytes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/generate_key_derivation.h b/tests/performance_tests/generate_key_derivation.h index 3f6238265..1f0066589 100644 --- a/tests/performance_tests/generate_key_derivation.h +++ b/tests/performance_tests/generate_key_derivation.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/generate_key_image.h b/tests/performance_tests/generate_key_image.h index 5155e64a5..df39d7088 100644 --- a/tests/performance_tests/generate_key_image.h +++ b/tests/performance_tests/generate_key_image.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/generate_key_image_helper.h b/tests/performance_tests/generate_key_image_helper.h index ede48b6ca..d4d83c07d 100644 --- a/tests/performance_tests/generate_key_image_helper.h +++ b/tests/performance_tests/generate_key_image_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/generate_keypair.h b/tests/performance_tests/generate_keypair.h index 91c830166..cf28f7f58 100644 --- a/tests/performance_tests/generate_keypair.h +++ b/tests/performance_tests/generate_keypair.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/is_out_to_acc.h b/tests/performance_tests/is_out_to_acc.h index 9f81995ac..8b4516843 100644 --- a/tests/performance_tests/is_out_to_acc.h +++ b/tests/performance_tests/is_out_to_acc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index bfe6de895..e6558a364 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -77,10 +77,12 @@ int main(int argc, char** argv) const command_line::arg_descriptor<bool> arg_verbose = { "verbose", "Verbose output", false }; const command_line::arg_descriptor<bool> arg_stats = { "stats", "Including statistics (min/median)", false }; const command_line::arg_descriptor<unsigned> arg_loop_multiplier = { "loop-multiplier", "Run for that many times more loops", 1 }; + const command_line::arg_descriptor<std::string> arg_timings_database = { "timings-database", "Keep timings history in a file" }; command_line::add_arg(desc_options, arg_filter); command_line::add_arg(desc_options, arg_verbose); command_line::add_arg(desc_options, arg_stats); command_line::add_arg(desc_options, arg_loop_multiplier); + command_line::add_arg(desc_options, arg_timings_database); po::variables_map vm; bool r = command_line::handle_error_helper(desc_options, [&]() @@ -93,7 +95,10 @@ int main(int argc, char** argv) return 1; const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter)); + const std::string timings_database = command_line::get_arg(vm, arg_timings_database); Params p; + if (!timings_database.empty()) + p.td = TimingsDatabase(timings_database); p.verbose = command_line::get_arg(vm, arg_verbose); p.stats = command_line::get_arg(vm, arg_stats); p.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier); @@ -193,7 +198,10 @@ int main(int argc, char** argv) TEST_PERFORMANCE2(filter, p, test_wallet2_expand_subaddresses, 50, 200); - TEST_PERFORMANCE0(filter, p, test_cn_slow_hash); + TEST_PERFORMANCE1(filter, p, test_cn_slow_hash, 0); + TEST_PERFORMANCE1(filter, p, test_cn_slow_hash, 1); + TEST_PERFORMANCE1(filter, p, test_cn_slow_hash, 2); + TEST_PERFORMANCE1(filter, p, test_cn_slow_hash, 4); TEST_PERFORMANCE1(filter, p, test_cn_fast_hash, 32); TEST_PERFORMANCE1(filter, p, test_cn_fast_hash, 16384); diff --git a/tests/performance_tests/multi_tx_test_base.h b/tests/performance_tests/multi_tx_test_base.h index ae9d6ea7a..bdbbee37d 100644 --- a/tests/performance_tests/multi_tx_test_base.h +++ b/tests/performance_tests/multi_tx_test_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/performance_tests.h b/tests/performance_tests/performance_tests.h index d37dda729..606c5980f 100644 --- a/tests/performance_tests/performance_tests.h +++ b/tests/performance_tests/performance_tests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // @@ -37,7 +37,9 @@ #include <boost/regex.hpp> #include "misc_language.h" +#include "stats.h" #include "common/perf_timer.h" +#include "common/timings.h" class performance_timer { @@ -67,6 +69,7 @@ private: struct Params { + TimingsDatabase td; bool verbose; bool stats; unsigned loop_multiplier; @@ -85,6 +88,8 @@ public: bool run() { + static_assert(0 < T::loop_count, "T::loop_count must be greater than 0"); + T test; if (!test.init()) return false; @@ -106,11 +111,13 @@ public: m_per_call_timers[i].pause(); } m_elapsed = timer.elapsed_ms(); + m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers)); return true; } int elapsed_time() const { return m_elapsed; } + size_t get_size() const { return m_stats->get_size(); } int time_per_call(int scale = 1) const { @@ -118,59 +125,19 @@ public: return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier); } - uint64_t per_call_min() const - { - uint64_t v = std::numeric_limits<uint64_t>::max(); - for (const auto &pt: m_per_call_timers) - v = std::min(v, pt.value()); - return v; - } - - uint64_t per_call_max() const - { - uint64_t v = std::numeric_limits<uint64_t>::min(); - for (const auto &pt: m_per_call_timers) - v = std::max(v, pt.value()); - return v; - } - - uint64_t per_call_mean() const - { - uint64_t v = 0; - for (const auto &pt: m_per_call_timers) - v += pt.value(); - return v / m_per_call_timers.size(); - } - - uint64_t per_call_median() const - { - std::vector<uint64_t> values; - values.reserve(m_per_call_timers.size()); - for (const auto &pt: m_per_call_timers) - values.push_back(pt.value()); - return epee::misc_utils::median(values); - } + uint64_t get_min() const { return m_stats->get_min(); } + uint64_t get_max() const { return m_stats->get_max(); } + double get_mean() const { return m_stats->get_mean(); } + uint64_t get_median() const { return m_stats->get_median(); } + double get_stddev() const { return m_stats->get_standard_deviation(); } + double get_non_parametric_skew() const { return m_stats->get_non_parametric_skew(); } + std::vector<uint64_t> get_quantiles(size_t n) const { return m_stats->get_quantiles(n); } - uint64_t per_call_stddev() const + bool is_same_distribution(size_t npoints, double mean, double stddev) const { - if (m_per_call_timers.size() <= 1) - return 0; - const uint64_t mean = per_call_mean(); - uint64_t acc = 0; - for (const auto &pt: m_per_call_timers) - { - int64_t dv = pt.value() - mean; - acc += dv * dv; - } - acc /= m_per_call_timers.size () - 1; - return sqrt(acc); + return m_stats->is_same_distribution_99(npoints, mean, stddev); } - uint64_t min_time_ns() const { return tools::ticks_to_ns(per_call_min()); } - uint64_t max_time_ns() const { return tools::ticks_to_ns(per_call_max()); } - uint64_t median_time_ns() const { return tools::ticks_to_ns(per_call_median()); } - uint64_t standard_deviation_time_ns() const { return tools::ticks_to_ns(per_call_stddev()); } - private: /** * Warm up processor core, enabling turbo boost, etc. @@ -191,10 +158,11 @@ private: int m_elapsed; Params m_params; std::vector<tools::PerformanceTimer> m_per_call_timers; + std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats; }; template <typename T> -void run_test(const std::string &filter, const Params ¶ms, const char* test_name) +void run_test(const std::string &filter, Params ¶ms, const char* test_name) { boost::smatch match; if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter))) @@ -210,10 +178,10 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_ std::cout << " elapsed: " << runner.elapsed_time() << " ms\n"; if (params.stats) { - std::cout << " min: " << runner.min_time_ns() << " ns\n"; - std::cout << " max: " << runner.max_time_ns() << " ns\n"; - std::cout << " median: " << runner.median_time_ns() << " ns\n"; - std::cout << " std dev: " << runner.standard_deviation_time_ns() << " ns\n"; + std::cout << " min: " << runner.get_min() << " ns\n"; + std::cout << " max: " << runner.get_max() << " ns\n"; + std::cout << " median: " << runner.get_median() << " ns\n"; + std::cout << " std dev: " << runner.get_stddev() << " ns\n"; } } else @@ -221,24 +189,48 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_ std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:"; } const char *unit = "ms"; - uint64_t scale = 1000000; - int time_per_call = runner.time_per_call(); - if (time_per_call < 30000) { + double scale = 1000000; + uint64_t time_per_call = runner.time_per_call(); + if (time_per_call < 100) { + scale = 1000; time_per_call = runner.time_per_call(1000); #ifdef _WIN32 unit = "\xb5s"; #else unit = "µs"; #endif - scale = 1000; } + const auto quantiles = runner.get_quantiles(10); + double min = runner.get_min(); + double max = runner.get_max(); + double med = runner.get_median(); + double mean = runner.get_mean(); + double stddev = runner.get_stddev(); + double npskew = runner.get_non_parametric_skew(); + + std::vector<TimingsDatabase::instance> prev_instances = params.td.get(test_name); + params.td.add(test_name, {time(NULL), runner.get_size(), min, max, mean, med, stddev, npskew, quantiles}); + std::cout << (params.verbose ? " time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : ""); if (params.stats) { - uint64_t min_ns = runner.min_time_ns() / scale; - uint64_t med_ns = runner.median_time_ns() / scale; - uint64_t stddev_ns = runner.standard_deviation_time_ns() / scale; - std::cout << " (min " << min_ns << " " << unit << ", median " << med_ns << " " << unit << ", std dev " << stddev_ns << " " << unit << ")"; + uint64_t mins = min / scale; + uint64_t maxs = max / scale; + uint64_t meds = med / scale; + uint64_t p95s = quantiles[9] / scale; + uint64_t stddevs = stddev / scale; + std::string cmp; + if (!prev_instances.empty()) + { + const TimingsDatabase::instance &prev_instance = prev_instances.back(); + if (!runner.is_same_distribution(prev_instance.npoints, prev_instance.mean, prev_instance.stddev)) + { + double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean); + cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster"); + } +cmp += " -- " + std::to_string(prev_instance.mean); + } + std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp; } std::cout << std::endl; } diff --git a/tests/performance_tests/performance_utils.h b/tests/performance_tests/performance_utils.h index 7d6f65977..8a45bea90 100644 --- a/tests/performance_tests/performance_utils.h +++ b/tests/performance_tests/performance_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/range_proof.h b/tests/performance_tests/range_proof.h index 0ce962cd9..548237814 100644 --- a/tests/performance_tests/range_proof.h +++ b/tests/performance_tests/range_proof.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/rct_mlsag.h b/tests/performance_tests/rct_mlsag.h index 888f4b260..0141710f7 100644 --- a/tests/performance_tests/rct_mlsag.h +++ b/tests/performance_tests/rct_mlsag.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2017, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/sc_reduce32.h b/tests/performance_tests/sc_reduce32.h index fc5606414..8db1ca70a 100644 --- a/tests/performance_tests/sc_reduce32.h +++ b/tests/performance_tests/sc_reduce32.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/signature.h b/tests/performance_tests/signature.h index 21110b525..63c63ea46 100644 --- a/tests/performance_tests/signature.h +++ b/tests/performance_tests/signature.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/single_tx_test_base.h b/tests/performance_tests/single_tx_test_base.h index 365d25444..52fbf5f80 100644 --- a/tests/performance_tests/single_tx_test_base.h +++ b/tests/performance_tests/single_tx_test_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018, The Monero Project +// Copyright (c) 2014-2019, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/subaddress_expand.h b/tests/performance_tests/subaddress_expand.h index 2a13ff5c2..61fcb41f3 100644 --- a/tests/performance_tests/subaddress_expand.h +++ b/tests/performance_tests/subaddress_expand.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018, The Monero Project +// Copyright (c) 2017-2019, The Monero Project // // All rights reserved. // |