Benchmark.hpp

Go to the documentation of this file.

#pragma once
 
#include "cli/CLIOptions.hpp"
 
#include <nlohmann/json.hpp>
#include <string>
#include <vector>
#include <optional>
#include <functional>
 
namespace CLI {
  class App;
}
 
namespace ppforest2::cli {
  CLI::App* setup_benchmark(CLI::App& app, CLIOptions& params);

  struct ConvergenceCriteria {
    float cv = 0.05f;
    int window = 3;
    int min = 10;
    int max = 200;
  };

  struct Scenario {
    std::string name;
 
    // Data source: either a CSV path or simulated NxPxG
    std::string data;
 
    // Data parameters (used for simulation; ignored when data is set)
    int n = 1000;
    int p = 10;
    int g = 3;
 
    // Model parameters
    int size     = 100;
    float vars   = 0.5f;
    float lambda = 0.5f;
    int threads  = -1;
 
    // Evaluation parameters
    float train_ratio = 0.7f;
    int seed          = 0;
    int warmup        = 0;
 
    // Iteration mode: if iterations > 0, fixed mode; otherwise convergence
    int iterations = -1;
    ConvergenceCriteria convergence;
  };

  struct ScenarioResult {
    std::string name;
    std::string data;
 
    // Data shape (copied from scenario for reporting; 0 for real-data scenarios)
    int n = 0, p = 0, g = 0;
    int size          = 0;
    float vars        = 0;
    float train_ratio = 0.7f;
 
    // Aggregated metrics
    int runs             = 0;
    double mean_time_ms  = 0;
    double std_time_ms   = 0;
    double mean_tr_error = 0;
    double mean_te_error = 0;
    long peak_rss_bytes  = -1;
    double peak_rss_mb   = -1;
 
    // Scenario wall-clock time (including warmup + process overhead)
    double scenario_time_ms = 0;
  };

  struct BenchmarkSuite {
    std::string name = "ppforest2 benchmark";
    std::vector<Scenario> scenarios;
  };

  struct SuiteResult {
    std::string suite_name;
    std::string timestamp;
    std::vector<ScenarioResult> results;
    double total_time_ms = 0;
 
    nlohmann::json to_json() const;
  };

  BenchmarkSuite parse_suite(std::string const& path);

  BenchmarkSuite parse_suite(nlohmann::json const& j);

  SuiteResult parse_results(std::string const& path);

  using ProgressCallback = std::function<void(int scenario_index, int total, std::string const& name)>;

  SuiteResult run_suite(BenchmarkSuite const& suite,
                        std::string const& binary_path,
                        bool quiet                = false,
                        ProgressCallback progress = nullptr);

  ScenarioResult run_scenario(Scenario const& scenario, std::string const& binary_path, bool quiet = false);

  int run_benchmark(CLIOptions& params, std::string const& binary_path);
}