aboutsummaryrefslogtreecommitdiff
path: root/tests/performance_tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests/performance_tests')
-rw-r--r--tests/performance_tests/main.cpp5
-rw-r--r--tests/performance_tests/performance_tests.h116
2 files changed, 59 insertions, 62 deletions
diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp
index 6749b71e4..bffbf699b 100644
--- a/tests/performance_tests/main.cpp
+++ b/tests/performance_tests/main.cpp
@@ -77,10 +77,12 @@ int main(int argc, char** argv)
const command_line::arg_descriptor<bool> arg_verbose = { "verbose", "Verbose output", false };
const command_line::arg_descriptor<bool> arg_stats = { "stats", "Including statistics (min/median)", false };
const command_line::arg_descriptor<unsigned> arg_loop_multiplier = { "loop-multiplier", "Run for that many times more loops", 1 };
+ const command_line::arg_descriptor<std::string> arg_timings_database = { "timings-database", "Keep timings history in a file" };
command_line::add_arg(desc_options, arg_filter);
command_line::add_arg(desc_options, arg_verbose);
command_line::add_arg(desc_options, arg_stats);
command_line::add_arg(desc_options, arg_loop_multiplier);
+ command_line::add_arg(desc_options, arg_timings_database);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
@@ -93,7 +95,10 @@ int main(int argc, char** argv)
return 1;
const std::string filter = tools::glob_to_regex(command_line::get_arg(vm, arg_filter));
+ const std::string timings_database = command_line::get_arg(vm, arg_timings_database);
Params p;
+ if (!timings_database.empty())
+ p.td = TimingsDatabase(timings_database);
p.verbose = command_line::get_arg(vm, arg_verbose);
p.stats = command_line::get_arg(vm, arg_stats);
p.loop_multiplier = command_line::get_arg(vm, arg_loop_multiplier);
diff --git a/tests/performance_tests/performance_tests.h b/tests/performance_tests/performance_tests.h
index d37dda729..17d16b0f6 100644
--- a/tests/performance_tests/performance_tests.h
+++ b/tests/performance_tests/performance_tests.h
@@ -37,7 +37,9 @@
#include <boost/regex.hpp>
#include "misc_language.h"
+#include "stats.h"
#include "common/perf_timer.h"
+#include "common/timings.h"
class performance_timer
{
@@ -67,6 +69,7 @@ private:
struct Params
{
+ TimingsDatabase td;
bool verbose;
bool stats;
unsigned loop_multiplier;
@@ -85,6 +88,8 @@ public:
bool run()
{
+ static_assert(0 < T::loop_count, "T::loop_count must be greater than 0");
+
T test;
if (!test.init())
return false;
@@ -106,11 +111,13 @@ public:
m_per_call_timers[i].pause();
}
m_elapsed = timer.elapsed_ms();
+ m_stats.reset(new Stats<tools::PerformanceTimer, uint64_t>(m_per_call_timers));
return true;
}
int elapsed_time() const { return m_elapsed; }
+ size_t get_size() const { return m_stats->get_size(); }
int time_per_call(int scale = 1) const
{
@@ -118,59 +125,19 @@ public:
return m_elapsed * scale / (T::loop_count * m_params.loop_multiplier);
}
- uint64_t per_call_min() const
- {
- uint64_t v = std::numeric_limits<uint64_t>::max();
- for (const auto &pt: m_per_call_timers)
- v = std::min(v, pt.value());
- return v;
- }
-
- uint64_t per_call_max() const
- {
- uint64_t v = std::numeric_limits<uint64_t>::min();
- for (const auto &pt: m_per_call_timers)
- v = std::max(v, pt.value());
- return v;
- }
-
- uint64_t per_call_mean() const
- {
- uint64_t v = 0;
- for (const auto &pt: m_per_call_timers)
- v += pt.value();
- return v / m_per_call_timers.size();
- }
-
- uint64_t per_call_median() const
- {
- std::vector<uint64_t> values;
- values.reserve(m_per_call_timers.size());
- for (const auto &pt: m_per_call_timers)
- values.push_back(pt.value());
- return epee::misc_utils::median(values);
- }
+ uint64_t get_min() const { return m_stats->get_min(); }
+ uint64_t get_max() const { return m_stats->get_max(); }
+ double get_mean() const { return m_stats->get_mean(); }
+ uint64_t get_median() const { return m_stats->get_median(); }
+ double get_stddev() const { return m_stats->get_standard_deviation(); }
+ double get_non_parametric_skew() const { return m_stats->get_non_parametric_skew(); }
+ std::vector<uint64_t> get_quantiles(size_t n) const { return m_stats->get_quantiles(n); }
- uint64_t per_call_stddev() const
+ bool is_same_distribution(size_t npoints, double mean, double stddev) const
{
- if (m_per_call_timers.size() <= 1)
- return 0;
- const uint64_t mean = per_call_mean();
- uint64_t acc = 0;
- for (const auto &pt: m_per_call_timers)
- {
- int64_t dv = pt.value() - mean;
- acc += dv * dv;
- }
- acc /= m_per_call_timers.size () - 1;
- return sqrt(acc);
+ return m_stats->is_same_distribution_99(npoints, mean, stddev);
}
- uint64_t min_time_ns() const { return tools::ticks_to_ns(per_call_min()); }
- uint64_t max_time_ns() const { return tools::ticks_to_ns(per_call_max()); }
- uint64_t median_time_ns() const { return tools::ticks_to_ns(per_call_median()); }
- uint64_t standard_deviation_time_ns() const { return tools::ticks_to_ns(per_call_stddev()); }
-
private:
/**
* Warm up processor core, enabling turbo boost, etc.
@@ -191,10 +158,11 @@ private:
int m_elapsed;
Params m_params;
std::vector<tools::PerformanceTimer> m_per_call_timers;
+ std::unique_ptr<Stats<tools::PerformanceTimer, uint64_t>> m_stats;
};
template <typename T>
-void run_test(const std::string &filter, const Params &params, const char* test_name)
+void run_test(const std::string &filter, Params &params, const char* test_name)
{
boost::smatch match;
if (!filter.empty() && !boost::regex_match(std::string(test_name), match, boost::regex(filter)))
@@ -210,10 +178,10 @@ void run_test(const std::string &filter, const Params &params, const char* test_
std::cout << " elapsed: " << runner.elapsed_time() << " ms\n";
if (params.stats)
{
- std::cout << " min: " << runner.min_time_ns() << " ns\n";
- std::cout << " max: " << runner.max_time_ns() << " ns\n";
- std::cout << " median: " << runner.median_time_ns() << " ns\n";
- std::cout << " std dev: " << runner.standard_deviation_time_ns() << " ns\n";
+ std::cout << " min: " << runner.get_min() << " ns\n";
+ std::cout << " max: " << runner.get_max() << " ns\n";
+ std::cout << " median: " << runner.get_median() << " ns\n";
+ std::cout << " std dev: " << runner.get_stddev() << " ns\n";
}
}
else
@@ -221,24 +189,48 @@ void run_test(const std::string &filter, const Params &params, const char* test_
std::cout << test_name << " (" << T::loop_count * params.loop_multiplier << " calls) - OK:";
}
const char *unit = "ms";
- uint64_t scale = 1000000;
- int time_per_call = runner.time_per_call();
- if (time_per_call < 30000) {
+ double scale = 1000000;
+ uint64_t time_per_call = runner.time_per_call();
+ if (time_per_call < 100) {
+ scale = 1000;
time_per_call = runner.time_per_call(1000);
#ifdef _WIN32
unit = "\xb5s";
#else
unit = "µs";
#endif
- scale = 1000;
}
+ const auto quantiles = runner.get_quantiles(10);
+ double min = runner.get_min();
+ double max = runner.get_max();
+ double med = runner.get_median();
+ double mean = runner.get_mean();
+ double stddev = runner.get_stddev();
+ double npskew = runner.get_non_parametric_skew();
+
+ std::vector<TimingsDatabase::instance> prev_instances = params.td.get(test_name);
+ params.td.add(test_name, {time(NULL), runner.get_size(), min, max, mean, med, stddev, npskew, quantiles});
+
std::cout << (params.verbose ? " time per call: " : " ") << time_per_call << " " << unit << "/call" << (params.verbose ? "\n" : "");
if (params.stats)
{
- uint64_t min_ns = runner.min_time_ns() / scale;
- uint64_t med_ns = runner.median_time_ns() / scale;
- uint64_t stddev_ns = runner.standard_deviation_time_ns() / scale;
- std::cout << " (min " << min_ns << " " << unit << ", median " << med_ns << " " << unit << ", std dev " << stddev_ns << " " << unit << ")";
+ uint64_t mins = min / scale;
+ uint64_t maxs = max / scale;
+ uint64_t meds = med / scale;
+ uint64_t p95s = quantiles[9] / scale;
+ uint64_t stddevs = stddev / scale;
+ std::string cmp;
+ if (!prev_instances.empty())
+ {
+ const TimingsDatabase::instance &prev_instance = prev_instances.back();
+ if (!runner.is_same_distribution(prev_instance.npoints, prev_instance.mean, prev_instance.stddev))
+ {
+ double pc = fabs(100. * (prev_instance.mean - runner.get_mean()) / prev_instance.mean);
+ cmp = ", " + std::to_string(pc) + "% " + (mean > prev_instance.mean ? "slower" : "faster");
+ }
+cmp += " -- " + std::to_string(prev_instance.mean);
+ }
+ std::cout << " (min " << mins << " " << unit << ", 90th " << p95s << " " << unit << ", median " << meds << " " << unit << ", std dev " << stddevs << " " << unit << ")" << cmp;
}
std::cout << std::endl;
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-17 05:47:34 +0000