diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/performance_tests/main.cpp | 5 | ||||
| -rw-r--r-- | tests/performance_tests/performance_tests.h | 116 |
2 files changed, 59 insertions, 62 deletions
diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index 6749b71e4..bffbf699b 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -77,10 +77,12 @@ int main(int argc, char** argv) const command_line::arg_descriptor<bool> arg_verbose = { "verbose", "Verbose output", false }; const command_line::arg_descriptor<bool> arg_stats = { "stats", "Including statistics (min/median)", false }; const command_line::arg_descriptor<unsigned> arg_loop_multiplier = { "loop-multiplier", "Run for that many times more loops", 1 }; + const command_line::arg_descriptor<std::string> arg_timings_database = { "timings-database", "Keep timings history in a file" }; command_line::add_arg(desc_options, arg_filter); command_line::add_arg(desc_options, arg_verbose); command_line::add_arg(desc_options, arg_stats); command_line::add_arg(desc_options, arg_loop_multiplier); + command_line::add_arg(desc_options, arg_timings_database); po::variables_map vm; bool r = command_line::handle_error_helper(desc_options, [&]() @@ -93,7 +95,10 @@ int main(int argc, char** argv) return 1; const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter)); + const std::string timings_database = command_line::get_arg(vm, arg_timings_database); Params p; + if (!timings_database.empty()) + p.td = TimingsDatabase(timings_database); p.verbose = command_line::get_arg(vm, arg_verbose); p.stats = command_line::get_arg(vm, arg_stats); p.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier); diff --git a/tests/performance_tests/performance_tests.h b/tests/performance_tests/performance_tests.h index d37dda729..17d16b0f6 100644 --- a/tests/performance_tests/performance_tests.h +++ b/tests/performance_tests/performance_tests.h @@ -37,7 +37,9 @@ #include <boost/regex.hpp> #include "misc_language.h" +#include "stats.h" #include "common/perf_timer.h" +#include "common/timings.h" class performance_timer { @@ -67,6 +69,7 @@ private: struct Params { + TimingsDatabase td; bool verbose; bool stats; unsigned loop_multiplier; @@ -85,6 +88,8 @@ public: bool run() { + static_assert(0 < T::loop_count, "T::loop_count must be greater than 0"); + T test; if (!test.init()) return false; @@ -106,11 +111,13 @@ public: m_per_call_timers[i].pause(); } m_elapsed = timer.elapsed_ms(); + m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers)); return true; } int elapsed_time() const { return m_elapsed; } + size_t get_size() const { return m_stats->get_size(); } int time_per_call(int scale = 1) const { @@ -118,59 +125,19 @@ public: return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier); } - uint64_t per_call_min() const - { - uint64_t v = std::numeric_limits<uint64_t>::max(); - for (const auto &pt: m_per_call_timers) - v = std::min(v, pt.value()); - return v; - } - - uint64_t per_call_max() const - { - uint64_t v = std::numeric_limits<uint64_t>::min(); - for (const auto &pt: m_per_call_timers) - v = std::max(v, pt.value()); - return v; - } - - uint64_t per_call_mean() const - { - uint64_t v = 0; - for (const auto &pt: m_per_call_timers) - v += pt.value(); - return v / m_per_call_timers.size(); - } - - uint64_t per_call_median() const - { - std::vector<uint64_t> values; - values.reserve(m_per_call_timers.size()); - for (const auto &pt: m_per_call_timers) - values.push_back(pt.value()); - return epee::misc_utils::median(values); - } + uint64_t get_min() const { return m_stats->get_min(); } + uint64_t get_max() const { return m_stats->get_max(); } + double get_mean() const { return m_stats->get_mean(); } + uint64_t get_median() const { return m_stats->get_median(); } + double get_stddev() const { return m_stats->get_standard_deviation(); } + double get_non_parametric_skew() const { return m_stats->get_non_parametric_skew(); } + std::vector<uint64_t> get_quantiles(size_t n) const { return m_stats->get_quantiles(n); } - uint64_t per_call_stddev() const + bool is_same_distribution(size_t npoints, double mean, double stddev) const { - if (m_per_call_timers.size() <= 1) - return 0; - const uint64_t mean = per_call_mean(); - uint64_t acc = 0; - for (const auto &pt: m_per_call_timers) - { - int64_t dv = pt.value() - mean; - acc += dv * dv; - } - acc /= m_per_call_timers.size () - 1; - return sqrt(acc); + return m_stats->is_same_distribution_99(npoints, mean, stddev); } - uint64_t min_time_ns() const { return tools::ticks_to_ns(per_call_min()); } - uint64_t max_time_ns() const { return tools::ticks_to_ns(per_call_max()); } - uint64_t median_time_ns() const { return tools::ticks_to_ns(per_call_median()); } - uint64_t standard_deviation_time_ns() const { return tools::ticks_to_ns(per_call_stddev()); } - private: /** * Warm up processor core, enabling turbo boost, etc. @@ -191,10 +158,11 @@ private: int m_elapsed; Params m_params; std::vector<tools::PerformanceTimer> m_per_call_timers; + std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats; }; template <typename T> -void run_test(const std::string &filter, const Params ¶ms, const char* test_name) +void run_test(const std::string &filter, Params ¶ms, const char* test_name) { boost::smatch match; if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter))) @@ -210,10 +178,10 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_ std::cout << " elapsed: " << runner.elapsed_time() << " ms\n"; if (params.stats) { - std::cout << " min: " << runner.min_time_ns() << " ns\n"; - std::cout << " max: " << runner.max_time_ns() << " ns\n"; - std::cout << " median: " << runner.median_time_ns() << " ns\n"; - std::cout << " std dev: " << runner.standard_deviation_time_ns() << " ns\n"; + std::cout << " min: " << runner.get_min() << " ns\n"; + std::cout << " max: " << runner.get_max() << " ns\n"; + std::cout << " median: " << runner.get_median() << " ns\n"; + std::cout << " std dev: " << runner.get_stddev() << " ns\n"; } } else @@ -221,24 +189,48 @@ void run_test(const std::string &filter, const Params ¶ms, const char* test_ std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:"; } const char *unit = "ms"; - uint64_t scale = 1000000; - int time_per_call = runner.time_per_call(); - if (time_per_call < 30000) { + double scale = 1000000; + uint64_t time_per_call = runner.time_per_call(); + if (time_per_call < 100) { + scale = 1000; time_per_call = runner.time_per_call(1000); #ifdef _WIN32 unit = "\xb5s"; #else unit = "µs"; #endif - scale = 1000; } + const auto quantiles = runner.get_quantiles(10); + double min = runner.get_min(); + double max = runner.get_max(); + double med = runner.get_median(); + double mean = runner.get_mean(); + double stddev = runner.get_stddev(); + double npskew = runner.get_non_parametric_skew(); + + std::vector<TimingsDatabase::instance> prev_instances = params.td.get(test_name); + params.td.add(test_name, {time(NULL), runner.get_size(), min, max, mean, med, stddev, npskew, quantiles}); + std::cout << (params.verbose ? " time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : ""); if (params.stats) { - uint64_t min_ns = runner.min_time_ns() / scale; - uint64_t med_ns = runner.median_time_ns() / scale; - uint64_t stddev_ns = runner.standard_deviation_time_ns() / scale; - std::cout << " (min " << min_ns << " " << unit << ", median " << med_ns << " " << unit << ", std dev " << stddev_ns << " " << unit << ")"; + uint64_t mins = min / scale; + uint64_t maxs = max / scale; + uint64_t meds = med / scale; + uint64_t p95s = quantiles[9] / scale; + uint64_t stddevs = stddev / scale; + std::string cmp; + if (!prev_instances.empty()) + { + const TimingsDatabase::instance &prev_instance = prev_instances.back(); + if (!runner.is_same_distribution(prev_instance.npoints, prev_instance.mean, prev_instance.stddev)) + { + double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean); + cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster"); + } +cmp += " -- " + std::to_string(prev_instance.mean); + } + std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp; } std::cout << std::endl; } |