performance_tests: better stats, and keep track of timing history

2 files changed, 59 insertions, 62 deletions

diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp
index 6749b71e4..bffbf699b 100644
--- a/tests/performance_tests/main.cpp
+++ b/tests/performance_tests/main.cpp
@@ -77,10 +77,12 @@ int main(int argc, char** argv)
   const command_line::arg_descriptor<bool> arg_verbose = { "verbose", "Verbose output", false };
   const command_line::arg_descriptor<bool> arg_stats = { "stats", "Including statistics (min/median)", false };
   const command_line::arg_descriptor<unsigned> arg_loop_multiplier = { "loop-multiplier", "Run for that many times more loops", 1 };
+  const command_line::arg_descriptor<std::string> arg_timings_database = { "timings-database", "Keep timings history in a file" };
   command_line::add_arg(desc_options, arg_filter);
   command_line::add_arg(desc_options, arg_verbose);
   command_line::add_arg(desc_options, arg_stats);
   command_line::add_arg(desc_options, arg_loop_multiplier);
+  command_line::add_arg(desc_options, arg_timings_database);
 
   po::variables_map vm;
   bool r = command_line::handle_error_helper(desc_options, [&]()
@@ -93,7 +95,10 @@ int main(int argc, char** argv)
     return 1;
 
   const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter));
+  const std::string timings_database = command_line::get_arg(vm, arg_timings_database);
   Params p;
+  if (!timings_database.empty())
+    p.td = TimingsDatabase(timings_database);
   p.verbose = command_line::get_arg(vm, arg_verbose);
   p.stats = command_line::get_arg(vm, arg_stats);
   p.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier);
diff --git a/tests/performance_tests/performance_tests.h b/tests/performance_tests/performance_tests.h
index d37dda729..17d16b0f6 100644
--- a/tests/performance_tests/performance_tests.h
+++ b/tests/performance_tests/performance_tests.h
@@ -37,7 +37,9 @@
 #include <boost/regex.hpp>
 
 #include "misc_language.h"
+#include "stats.h"
 #include "common/perf_timer.h"
+#include "common/timings.h"
 
 class performance_timer
 {
@@ -67,6 +69,7 @@ private:
 
 struct Params
 {
+  TimingsDatabase td;
   bool verbose;
   bool stats;
   unsigned loop_multiplier;
@@ -85,6 +88,8 @@ public:
 
   bool run()
   {
+    static_assert(0 < T::loop_count, "T::loop_count must be greater than 0");
+
     T test;
     if (!test.init())
       return false;
@@ -106,11 +111,13 @@ public:
         m_per_call_timers[i].pause();
     }
     m_elapsed = timer.elapsed_ms();
+    m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers));
 
     return true;
   }
 
   int elapsed_time() const { return m_elapsed; }
+  size_t get_size() const { return m_stats->get_size(); }
 
   int time_per_call(int scale = 1) const
   {
@@ -118,59 +125,19 @@ public:
     return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier);
   }
 
-  uint64_t per_call_min() const
-  {
-    uint64_t v = std::numeric_limits<uint64_t>::max();
-    for (const auto &pt: m_per_call_timers)
-      v = std::min(v, pt.value());
-    return v;
-  }
-
-  uint64_t per_call_max() const
-  {
-    uint64_t v = std::numeric_limits<uint64_t>::min();
-    for (const auto &pt: m_per_call_timers)
-      v = std::max(v, pt.value());
-    return v;
-  }
-
-  uint64_t per_call_mean() const
-  {
-    uint64_t v = 0;
-    for (const auto &pt: m_per_call_timers)
-      v += pt.value();
-    return v / m_per_call_timers.size();
-  }
-
-  uint64_t per_call_median() const
-  {
-    std::vector<uint64_t> values;
-    values.reserve(m_per_call_timers.size());
-    for (const auto &pt: m_per_call_timers)
-      values.push_back(pt.value());
-    return epee::misc_utils::median(values);
-  }
+  uint64_t get_min() const { return m_stats->get_min(); }
+  uint64_t get_max() const { return m_stats->get_max(); }
+  double get_mean() const { return m_stats->get_mean(); }
+  uint64_t get_median() const { return m_stats->get_median(); }
+  double get_stddev() const { return m_stats->get_standard_deviation(); }
+  double get_non_parametric_skew() const { return m_stats->get_non_parametric_skew(); }
+  std::vector<uint64_t> get_quantiles(size_t n) const { return m_stats->get_quantiles(n); }
 
-  uint64_t per_call_stddev() const
+  bool is_same_distribution(size_t npoints, double mean, double stddev) const
   {
-    if (m_per_call_timers.size() <= 1)
-      return 0;
-    const uint64_t mean = per_call_mean();
-    uint64_t acc = 0;
-    for (const auto &pt: m_per_call_timers)
-    {
-      int64_t dv = pt.value() - mean;
-      acc += dv * dv;
-    }
-    acc /= m_per_call_timers.size () - 1;
-    return sqrt(acc);
+    return m_stats->is_same_distribution_99(npoints, mean, stddev);
   }
 
-  uint64_t min_time_ns() const { return tools::ticks_to_ns(per_call_min()); }
-  uint64_t max_time_ns() const { return tools::ticks_to_ns(per_call_max()); }
-  uint64_t median_time_ns() const { return tools::ticks_to_ns(per_call_median()); }
-  uint64_t standard_deviation_time_ns() const { return tools::ticks_to_ns(per_call_stddev()); }
-
 private:
   /**
    * Warm up processor core, enabling turbo boost, etc.
@@ -191,10 +158,11 @@ private:
   int m_elapsed;
   Params m_params;
   std::vector<tools::PerformanceTimer> m_per_call_timers;
+  std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats;
 };
 
 template <typename T>
-void run_test(const std::string &filter, const Params &params, const char* test_name)
+void run_test(const std::string &filter, Params &params, const char* test_name)
 {
   boost::smatch match;
   if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter)))
@@ -210,10 +178,10 @@ void run_test(const std::string &filter, const Params &params, const char* test_
       std::cout << "  elapsed:       " << runner.elapsed_time() << " ms\n";
       if (params.stats)
       {
-        std::cout << "  min:       " << runner.min_time_ns() << " ns\n";
-        std::cout << "  max:       " << runner.max_time_ns() << " ns\n";
-        std::cout << "  median:    " << runner.median_time_ns() << " ns\n";
-        std::cout << "  std dev:   " << runner.standard_deviation_time_ns() << " ns\n";
+        std::cout << "  min:       " << runner.get_min() << " ns\n";
+        std::cout << "  max:       " << runner.get_max() << " ns\n";
+        std::cout << "  median:    " << runner.get_median() << " ns\n";
+        std::cout << "  std dev:   " << runner.get_stddev() << " ns\n";
       }
     }
     else
@@ -221,24 +189,48 @@ void run_test(const std::string &filter, const Params &params, const char* test_
       std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:";
     }
     const char *unit = "ms";
-    uint64_t scale = 1000000;
-    int time_per_call = runner.time_per_call();
-    if (time_per_call < 30000) {
+    double scale = 1000000;
+    uint64_t time_per_call = runner.time_per_call();
+    if (time_per_call < 100) {
+     scale = 1000;
      time_per_call = runner.time_per_call(1000);
 #ifdef _WIN32
      unit = "\xb5s";
 #else
      unit = "µs";
 #endif
-     scale = 1000;
     }
+    const auto quantiles = runner.get_quantiles(10);
+    double min = runner.get_min();
+    double max = runner.get_max();
+    double med = runner.get_median();
+    double mean = runner.get_mean();
+    double stddev = runner.get_stddev();
+    double npskew = runner.get_non_parametric_skew();
+
+    std::vector<TimingsDatabase::instance> prev_instances = params.td.get(test_name);
+    params.td.add(test_name, {time(NULL), runner.get_size(), min, max, mean, med, stddev, npskew, quantiles});
+
     std::cout << (params.verbose ? "  time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : "");
     if (params.stats)
     {
-      uint64_t min_ns = runner.min_time_ns() / scale;
-      uint64_t med_ns = runner.median_time_ns() / scale;
-      uint64_t stddev_ns = runner.standard_deviation_time_ns() / scale;
-      std::cout << " (min " << min_ns << " " << unit << ", median " << med_ns << " " << unit << ", std dev " << stddev_ns << " " << unit << ")";
+      uint64_t mins = min / scale;
+      uint64_t maxs = max / scale;
+      uint64_t meds = med / scale;
+      uint64_t p95s = quantiles[9] / scale;
+      uint64_t stddevs = stddev / scale;
+      std::string cmp;
+      if (!prev_instances.empty())
+      {
+        const TimingsDatabase::instance &prev_instance = prev_instances.back();
+        if (!runner.is_same_distribution(prev_instance.npoints, prev_instance.mean, prev_instance.stddev))
+        {
+          double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean);
+          cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster");
+        }
+cmp += "  -- " + std::to_string(prev_instance.mean);
+      }
+      std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp;
     }
     std::cout << std::endl;
   }