BenchmarkReport.hpp

Go to the documentation of this file.

#pragma once
 
#include "cli/Benchmark.hpp"
#include "io/Output.hpp"
#include "io/Table.hpp"
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
 
namespace ppforest2::cli {
  struct Baseline : SuiteResult {
    explicit Baseline(SuiteResult suite)
        : SuiteResult(std::move(suite)) {
      for (auto const& r : results) {
        index_[r.name] = &r;
      }
    }

    std::optional<std::reference_wrapper<ScenarioResult const>> find(std::string const& name) const {
      auto it = index_.find(name);
 
      if (it == index_.end()) {
        return std::nullopt;
      }
 
      return std::cref(*it->second);
    }
 
  private:
    std::unordered_map<std::string, ScenarioResult const*> index_;
  };

  struct BenchmarkReport {
    double delta_threshold = 1.0;
 
    explicit BenchmarkReport(SuiteResult const& r, std::optional<Baseline> const& b = std::nullopt)
        : result(r)
        , baseline(b) {}

    struct Style {
      virtual ~Style() = default;
 
    protected:
      friend struct BenchmarkReport;
 
      virtual std::string good(std::string const& s) const    = 0;
      virtual std::string bad(std::string const& s) const     = 0;
      virtual std::string neutral(std::string const& s) const = 0;
 
      using Columns = std::vector<ppforest2::io::layout::Column>;
 
      virtual std::string format_title(std::string const& name, std::string const& timestamp) const         = 0;
      virtual std::string format_baseline_label(std::string const& timestamp) const                         = 0;
      virtual std::string format_header(Columns const& columns) const                                       = 0;
      virtual std::string format_row(Columns const& columns, ppforest2::io::layout::Row const& cells) const = 0;
      virtual std::string format_footer(std::size_t count, double total_ms) const                           = 0;
    };

    struct Text;

    struct Markdown;

    void print(io::Output& out, Style const& style) const;

    nlohmann::json to_json() const;

    std::string to_csv() const;
 
  private:
    SuiteResult const& result;
    std::optional<Baseline> const& baseline;
 
    static double delta_pct(double current, double baseline);
    static std::vector<ppforest2::io::layout::Column> make_columns(bool has_baseline);
 
    struct Deltas {
      std::optional<double> time;
      std::optional<double> rss;
      std::optional<double> tr_err;
      std::optional<double> te_err;
    };
 
    static Deltas compute_deltas(ScenarioResult const& r, Baseline const& baseline);
  };

  struct BenchmarkReport::Text : BenchmarkReport::Style {
    std::string good(std::string const& s) const override;
    std::string bad(std::string const& s) const override;
    std::string neutral(std::string const& s) const override;
 
    std::string format_title(std::string const& name, std::string const& ts) const override;
    std::string format_baseline_label(std::string const& ts) const override;
    std::string format_header(Columns const& columns) const override;
    std::string format_row(Columns const& columns, ppforest2::io::layout::Row const& cells) const override;
    std::string format_footer(std::size_t count, double total_ms) const override;
  };

  struct BenchmarkReport::Markdown : BenchmarkReport::Style {
    std::string good(std::string const& s) const override;
    std::string bad(std::string const& s) const override;
    std::string neutral(std::string const& s) const override;
 
    std::string format_title(std::string const& name, std::string const& ts) const override;
    std::string format_baseline_label(std::string const& ts) const override;
    std::string format_header(Columns const& columns) const override;
    std::string format_row(Columns const& columns, ppforest2::io::layout::Row const& cells) const override;
    std::string format_footer(std::size_t count, double total_ms) const override;
  };
}