Benchmark.hpp

Go to the documentation of this file.

#pragma once
 
#include "cli/CLIOptions.hpp"
#include "io/EvaluateResult.hpp"
 
#include <nlohmann/json.hpp>
#include <string>
#include <vector>
 
namespace CLI {
  class App;
}
 
namespace ppforest2::cli {
  void setup_benchmark(CLI::App& app, Params& params);

  struct ScenarioResult : io::EvaluateResult {
    ScenarioResult() = default;
 
    ScenarioResult(std::string name, double scenario_time_ms, nlohmann::json const& j)
        : io::EvaluateResult(j)
        , name(std::move(name))
        , scenario_time_ms(scenario_time_ms) {}
 
    std::string name;
 
    // Scenario wall-clock time (including warmup + process overhead)
    double scenario_time_ms = 0;
  };

  struct BenchmarkSuite {
    explicit BenchmarkSuite(std::string name = "ppforest2 benchmark")
        : name(std::move(name)) {}
 
    std::string name;
    std::vector<nlohmann::json> scenarios;
  };

  struct SuiteResult {
    SuiteResult() = default;
    explicit SuiteResult(nlohmann::json const& j);
 
    std::string suite_name;
    std::string timestamp;
    std::vector<ScenarioResult> results;
    double total_time_ms = 0;
 
    nlohmann::json to_json() const;
    std::string to_csv() const;
  };

  BenchmarkSuite parse_suite(std::string const& path);

  BenchmarkSuite parse_suite(nlohmann::json const& j);

  SuiteResult run_suite(BenchmarkSuite const& suite, std::string const& binary_path, io::Output& out);

  ScenarioResult run_scenario(nlohmann::json const& scenario, std::string const& binary_path);

  int run_benchmark(Params const& params, std::string const& binary_path);
}