krabka_parse_benches/

lib.rs

//! Parses Criterion benchmark output in bencher format into structured JSON
//! summaries, and judges repeated reference/candidate measurements against
//! variance-calibrated budgets.
//!
//! Two lanes share one set of statistics. The criterion lane compares
//! [`BenchmarkSummary`] files with [`compare_summaries`]. The broker
//! performance lane judges a [`record::PerformanceRecord`] against a
//! [`contract::Contract`] with [`budget::evaluate`]. Both take medians over
//! repeated runs and allow the larger of a fixed floor and a multiple of the
//! relative median absolute deviation: [`calibrated_tolerance`].
//!
//! Each criterion summary also names the bench executables its pass ran, as
//! [`executables`] verified them, so a verdict says which binaries produced
//! its numbers.
//!
//! Note: If a shared `krabka-tools` repository is established across the
//! `krabka-io` organization, this crate can be migrated there as a common
//! benchmark utility for `krabka-broker`, `krabka-protocol`, and `krabka-client-rs`.

pub mod budget;
pub mod contract;
pub mod executables;
mod num;
pub mod record;
pub mod render;

use {
    budget::evaluate,
    clap::{Parser, Subcommand},
    contract::{BudgetError, Contract, contract_digest},
    executables::{BenchExecutable, ExecutableManifest, manifest_from_cargo_messages, verify_pass},
    record::{PerformanceRecord, assemble},
    regex::Regex,
    render::render_markdown,
    serde::{Deserialize, Serialize},
    std::{
        collections::BTreeMap,
        env,
        fs::{self, File},
        io::{self, BufRead, BufReader, Write},
        path::{Path, PathBuf},
        sync::LazyLock,
        time::SystemTime,
    },
    thiserror::Error,
    time::format_description::well_known::Rfc3339,
};

static BENCHER_REGEX: LazyLock<Result<Regex, regex::Error>> = LazyLock::new(|| {
    Regex::new(
        r"test\s+([\w\/\-\.]+)\s+\.\.\.\s+bench:\s+([\d,]+)\s+ns\/iter\s+\(\+\/-\s+([\d,]+)\)",
    )
});

/// Errors encountered while parsing benchmark output.
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum ParseBenchesError {
    /// Benchmark results directory does not exist.
    #[error("directory '{0}' does not exist")]
    DirectoryNotFound(PathBuf),

    /// No `.txt` output files were found in the results directory.
    #[error("no benchmark output files (*.txt) found in '{0}'")]
    NoTxtFiles(PathBuf),

    /// Output files were found, but no benchmark metrics could be parsed.
    #[error("parsed 0 benchmark metrics from '{0}'")]
    NoMetricsParsed(PathBuf),

    /// A duplicate benchmark identifier was found.
    #[error("duplicate benchmark metric '{name}' found in '{file}'")]
    DuplicateBenchmark {
        /// Name of the duplicated benchmark identifier.
        name: String,
        /// Path of the file containing the collision.
        file: PathBuf,
    },

    /// Failed to parse a numeric value from benchmark output.
    #[error("failed to parse numeric benchmark value: {0}")]
    InvalidNumber(String),

    /// The bencher line pattern did not compile.
    #[error("bencher line pattern: {0}")]
    Pattern(#[source] regex::Error),

    /// Underlying I/O error.
    #[error("I/O error: {0}")]
    Io(#[from] io::Error),

    /// Underlying JSON serialization error.
    #[error("JSON error: {0}")]
    Json(#[from] serde_json::Error),

    /// Fewer than three repeated summaries were supplied for one side.
    #[error("{side} needs at least 3 repeated benchmark summaries, got {count}")]
    TooFewSamples {
        /// The comparison side.
        side: &'static str,
        /// Number of supplied summaries.
        count: usize,
    },

    /// Reference and candidate summaries did not contain the same benchmarks.
    #[error("reference and candidate benchmark sets differ")]
    BenchmarkSetMismatch,

    /// A sample was non-finite or non-positive without every sample being zero.
    #[error("invalid sample for benchmark '{0}'")]
    InvalidSample(String),

    /// A supplied summary had no benchmark metrics.
    #[error("benchmark summaries contain no metrics")]
    EmptyBenchmarkSet,

    /// The configured tolerance was negative, NaN, or infinite.
    #[error("minimum tolerance must be a finite non-negative ratio")]
    InvalidTolerance,

    /// Cargo's JSON messages named no bench executable.
    #[error("cargo built no bench executable")]
    NoBenchExecutables,

    /// A pass's Cargo log named no executable that it ran.
    #[error("the cargo log names no bench executable that ran")]
    NoExecutableRan,

    /// An executable manifest was built from a different commit than the pass.
    #[error("executables were built from {manifest} but the pass is labelled {pass}")]
    CommitMismatch {
        /// The commit the manifest records.
        manifest: String,
        /// The commit the pass was labelled with.
        pass: String,
    },

    /// A pass ran an executable that its side's manifest does not list.
    #[error("pass ran '{0}', which its side did not prebuild")]
    UnrecordedExecutable(PathBuf),

    /// A prebuilt executable's bytes changed after the manifest recorded them.
    #[error("executable '{0}' changed after it was prebuilt")]
    ExecutableChanged(PathBuf),

    /// A side's summaries name no executable.
    #[error("{0} summaries do not record which executables ran")]
    MissingExecutableIdentity(&'static str),

    /// A side's passes ran different executables.
    #[error("{0} passes ran different executables")]
    ExecutableDrift(&'static str),

    /// Reference and candidate ran the same executable file.
    #[error(
        "reference and candidate both ran '{0}'; they must build into separate target directories"
    )]
    SharedExecutable(PathBuf),

    /// A broker performance lane command failed.
    #[error(transparent)]
    Budget(#[from] BudgetError),
}

/// A single benchmark measurement in nanoseconds per iteration.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkMetric {
    /// Mean nanoseconds per iteration.
    pub ns_per_iter: f64,
    /// Variance in nanoseconds per iteration (+/- bound).
    pub variance_ns: f64,
}

/// A benchmark measurement together with the name of the benchmark it
/// belongs to, as read from one line of bencher output.
#[derive(Debug, Clone, PartialEq)]
pub struct NamedMetric {
    /// Canonical benchmark name.
    pub name: String,
    /// The measurement of that benchmark.
    pub metric: BenchmarkMetric,
}

/// Aggregated benchmark summary document.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkSummary {
    /// Benchmark suite name (e.g. `krabka-broker`).
    pub suite: String,
    /// Git commit SHA of the run.
    pub commit: String,
    /// ISO8601/RFC3339 UTC timestamp when the summary was generated.
    pub timestamp: String,
    /// Map of canonical benchmark names to metrics, sorted alphabetically.
    pub benchmarks: BTreeMap<String, BenchmarkMetric>,
    /// The bench executables this pass ran; empty when none were recorded.
    #[serde(default)]
    pub executables: Vec<BenchExecutable>,
}

/// One benchmark's variance-calibrated reference/candidate decision.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkComparison {
    /// Median reference time in nanoseconds.
    pub reference_median_ns: f64,
    /// Median candidate time in nanoseconds.
    pub candidate_median_ns: f64,
    /// Candidate/reference ratio.
    pub ratio: f64,
    /// Allowed relative slowdown, derived from repeated-run MAD with a floor.
    pub tolerance: f64,
    /// Whether this benchmark stayed within its tolerance, or its candidate
    /// passes overlapped the reference passes.
    pub passed: bool,
}

/// Machine-readable verdict over repeated same-host benchmark runs.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BenchmarkVerdict {
    /// Exact commits represented by the reference samples.
    pub reference_commits: Vec<String>,
    /// Exact commits represented by the candidate samples.
    pub candidate_commits: Vec<String>,
    /// The executables every reference pass ran.
    pub reference_executables: Vec<BenchExecutable>,
    /// The executables every candidate pass ran.
    pub candidate_executables: Vec<BenchExecutable>,
    /// Per-benchmark decisions.
    pub benchmarks: BTreeMap<String, BenchmarkComparison>,
    /// True only when every benchmark passed.
    pub passed: bool,
}

/// The median of `values`; an even count averages the middle two. An empty
/// `values` has no median, and gives NaN.
#[must_use]
pub fn median(mut values: Vec<f64>) -> f64 {
    values.sort_by(f64::total_cmp);
    let middle = values.len() / 2;
    let lower = middle.checked_sub(1).and_then(|lower| values.get(lower));
    match (lower, values.get(middle)) {
        (Some(&lower), Some(&upper)) if values.len().is_multiple_of(2) => {
            f64::midpoint(lower, upper)
        }
        (_, Some(&upper)) => upper,
        (_, None) => f64::NAN,
    }
}

/// The median absolute deviation of `values` around `center`, as a ratio of
/// `center`.
#[must_use]
pub fn relative_mad(values: &[f64], center: f64) -> f64 {
    median(values.iter().map(|value| (value - center).abs()).collect()) / center
}

/// The relative change a comparison allows: `multiplier` times the larger of
/// the two sides' spreads, but never less than `floor`.
#[must_use]
pub fn calibrated_tolerance(
    floor: f64,
    multiplier: f64,
    reference_spread: f64,
    candidate_spread: f64,
) -> f64 {
    floor.max(multiplier * reference_spread.max(candidate_spread))
}

/// The executables every summary of one side ran.
fn side_executables(
    side: &'static str,
    summaries: &[BenchmarkSummary],
) -> Result<Vec<BenchExecutable>, ParseBenchesError> {
    let Some(first) = summaries.first().map(|summary| &summary.executables) else {
        return Err(ParseBenchesError::TooFewSamples { side, count: 0 });
    };
    if first.is_empty() {
        return Err(ParseBenchesError::MissingExecutableIdentity(side));
    }
    if summaries
        .iter()
        .any(|summary| summary.executables != *first)
    {
        return Err(ParseBenchesError::ExecutableDrift(side));
    }
    Ok(first.clone())
}

/// Compare at least three repeated reference and candidate summaries.
///
/// Every summary must name the executables its pass ran. Each side's passes
/// must all have run the same executables, and no executable path may appear
/// on both sides: a shared path means the two revisions were measured with
/// the same binary.
///
/// The tolerance is three times the larger side's median absolute deviation,
/// with `minimum_tolerance` as a floor. A benchmark fails only when its median
/// ratio exceeds that tolerance and every candidate pass is also slower than
/// every reference pass: with a handful of passes per side, a median shift
/// inside the two sides' overlapping ranges is run-to-run noise, not a
/// regression. Bilateral all-zero samples are tied
/// below bencher's integer-nanosecond resolution. Other missing, non-finite, or
/// non-positive samples are errors and therefore cannot silently pass.
///
/// # Errors
///
/// Returns [`ParseBenchesError`] for too few samples, mismatched benchmark
/// sets, missing, drifting or shared executables, or invalid measurements.
pub fn compare_summaries(
    reference: &[BenchmarkSummary],
    candidate: &[BenchmarkSummary],
    minimum_tolerance: f64,
) -> Result<BenchmarkVerdict, ParseBenchesError> {
    if !minimum_tolerance.is_finite() || minimum_tolerance < 0.0 {
        return Err(ParseBenchesError::InvalidTolerance);
    }
    for (side, summaries) in [("reference", reference), ("candidate", candidate)] {
        if summaries.len() < 3 {
            return Err(ParseBenchesError::TooFewSamples {
                side,
                count: summaries.len(),
            });
        }
    }
    let reference_names: Vec<_> = reference
        .first()
        .ok_or(ParseBenchesError::TooFewSamples {
            side: "reference",
            count: 0,
        })?
        .benchmarks
        .keys()
        .collect();
    if reference_names.is_empty() {
        return Err(ParseBenchesError::EmptyBenchmarkSet);
    }
    if reference.iter().any(|summary| {
        summary
            .benchmarks
            .keys()
            .ne(reference_names.iter().copied())
    }) || candidate.iter().any(|summary| {
        summary
            .benchmarks
            .keys()
            .ne(reference_names.iter().copied())
    }) {
        return Err(ParseBenchesError::BenchmarkSetMismatch);
    }
    let reference_executables = side_executables("reference", reference)?;
    let candidate_executables = side_executables("candidate", candidate)?;
    if let Some(shared) = reference_executables.iter().find(|reference| {
        candidate_executables
            .iter()
            .any(|candidate| candidate.path == reference.path)
    }) {
        return Err(ParseBenchesError::SharedExecutable(shared.path.clone()));
    }

    let mut benchmarks = BTreeMap::new();
    for name in reference_names {
        // Every summary holds exactly the reference names, checked above.
        let side_values = |summaries: &[BenchmarkSummary]| {
            summaries
                .iter()
                .map(|summary| {
                    summary
                        .benchmarks
                        .get(name)
                        .map(|metric| metric.ns_per_iter)
                })
                .collect::<Option<Vec<_>>>()
                .ok_or(ParseBenchesError::BenchmarkSetMismatch)
        };
        let reference_values = side_values(reference)?;
        let candidate_values = side_values(candidate)?;
        let mut values = reference_values.iter().chain(&candidate_values);
        if values.clone().all(|value| *value == 0.0) {
            benchmarks.insert(name.clone(), BenchmarkComparison {
                reference_median_ns: 0.0,
                candidate_median_ns: 0.0,
                ratio: 1.0,
                tolerance: minimum_tolerance,
                passed: true,
            });
            continue;
        }
        if values.any(|value| !value.is_finite() || *value <= 0.0) {
            return Err(ParseBenchesError::InvalidSample(name.clone()));
        }
        let reference_median_ns = median(reference_values.clone());
        let candidate_median_ns = median(candidate_values.clone());
        let tolerance = calibrated_tolerance(
            minimum_tolerance,
            3.0,
            relative_mad(&reference_values, reference_median_ns),
            relative_mad(&candidate_values, candidate_median_ns),
        );
        let ratio = candidate_median_ns / reference_median_ns;
        let slowest_reference = reference_values.iter().copied().fold(f64::MIN, f64::max);
        let fastest_candidate = candidate_values.iter().copied().fold(f64::MAX, f64::min);
        benchmarks.insert(name.clone(), BenchmarkComparison {
            reference_median_ns,
            candidate_median_ns,
            ratio,
            tolerance,
            passed: ratio <= 1.0 + tolerance || fastest_candidate <= slowest_reference,
        });
    }
    let passed = benchmarks.values().all(|benchmark| benchmark.passed);
    Ok(BenchmarkVerdict {
        reference_commits: reference
            .iter()
            .map(|summary| summary.commit.clone())
            .collect(),
        candidate_commits: candidate
            .iter()
            .map(|summary| summary.commit.clone())
            .collect(),
        reference_executables,
        candidate_executables,
        benchmarks,
        passed,
    })
}

/// Read every JSON summary in `directory`, sorted by filename.
///
/// # Errors
///
/// Returns [`ParseBenchesError`] when the directory is missing, contains fewer
/// than three summaries, or a summary is unreadable or invalid JSON.
pub fn read_summaries(
    directory: &Path,
    side: &'static str,
) -> Result<Vec<BenchmarkSummary>, ParseBenchesError> {
    if !directory.is_dir() {
        return Err(ParseBenchesError::DirectoryNotFound(
            directory.to_path_buf(),
        ));
    }
    let mut paths = Vec::new();
    for entry in fs::read_dir(directory)? {
        let path = entry?.path();
        if path.extension().and_then(|extension| extension.to_str()) == Some("json") {
            paths.push(path);
        }
    }
    paths.sort();
    if paths.len() < 3 {
        return Err(ParseBenchesError::TooFewSamples {
            side,
            count: paths.len(),
        });
    }
    paths
        .into_iter()
        .map(|path| Ok(serde_json::from_reader(File::open(path)?)?))
        .collect()
}

/// CLI configuration arguments for parsing benchmarks.
#[derive(Parser, Debug, Clone)]
#[command(
    name = "krabka-parse-benches",
    about = "Parses Criterion benchmark output in bencher format into a structured JSON summary"
)]
pub struct Args {
    /// Directory containing benchmark `.txt` output files.
    #[arg(long, default_value = "bench-results")]
    pub results_dir: PathBuf,

    /// Output JSON summary file path (defaults to `<results_dir>/broker-benchmarks.json`).
    #[arg(long)]
    pub output: Option<PathBuf>,

    /// Suite identifier for the benchmark summary.
    #[arg(long, default_value = "krabka-broker")]
    pub suite: String,

    /// Git commit SHA (defaults to `GITHUB_SHA` env var or `"unknown"`).
    #[arg(long)]
    pub commit: Option<String>,

    /// Directory of repeated reference summary JSON files to compare.
    #[arg(long, requires = "candidate_dir")]
    pub reference_dir: Option<PathBuf>,

    /// Directory of repeated candidate summary JSON files to compare.
    #[arg(long, requires = "reference_dir")]
    pub candidate_dir: Option<PathBuf>,

    /// The executable manifest this pass's side was prebuilt into. With it,
    /// the summary records which executables the pass ran, and parsing fails
    /// unless they are the manifest's, unchanged.
    #[arg(long, requires = "cargo_log")]
    pub executables: Option<PathBuf>,

    /// Cargo's standard error from this pass, whose `Running` lines name the
    /// executables it ran.
    #[arg(long, requires = "executables")]
    pub cargo_log: Option<PathBuf>,

    /// The directory Cargo ran in, against which relative `Running` paths
    /// resolve (defaults to the current directory).
    #[arg(long, requires = "executables")]
    pub source_dir: Option<PathBuf>,

    /// Minimum allowed slowdown percentage for comparison mode.
    #[arg(long, default_value_t = 3.0)]
    pub minimum_tolerance_percent: f64,

    /// The broker performance lane; without one, the criterion lane runs.
    #[command(subcommand)]
    pub command: Option<Command>,
}

/// The commands besides parsing and comparing criterion output.
#[derive(Subcommand, Debug, Clone)]
pub enum Command {
    /// Record the bench executables `cargo bench --no-run
    /// --message-format=json` built, with their digests.
    Executables {
        /// Cargo's JSON messages.
        cargo_messages: PathBuf,
        /// The commit the executables were built from.
        #[arg(long)]
        commit: String,
        /// Where to write the manifest JSON.
        #[arg(long)]
        output: PathBuf,
    },
    /// Validate a regression contract and print its digest.
    CheckContract {
        /// The contract JSON.
        contract: PathBuf,
    },
    /// Assemble a `qualify.sh budget` artifact directory into a record.
    Assemble {
        /// The artifact directory.
        artifact_dir: PathBuf,
        /// Where to write the record JSON.
        #[arg(long)]
        output: PathBuf,
    },
    /// Judge a record against its contract; a rejection exits 1.
    Evaluate {
        /// The record JSON.
        record: PathBuf,
        /// The contract the record was measured under.
        #[arg(long)]
        contract: PathBuf,
        /// Where to write the verdict JSON.
        #[arg(long)]
        output: PathBuf,
        /// Where to write the published Markdown.
        #[arg(long)]
        markdown: Option<PathBuf>,
    },
}

/// Parses a single line of Criterion bencher output.
///
/// Matches patterns of the shape:
/// `test log/append/100rec_1024B ... bench: 95,979 ns/iter (+/- 1,377,729)`
///
/// # Errors
///
/// Returns [`ParseBenchesError::InvalidNumber`] if matched numeric fields cannot
/// be parsed into floats.
pub fn parse_bencher_line(line: &str) -> Result<Option<NamedMetric>, ParseBenchesError> {
    let regex = BENCHER_REGEX
        .as_ref()
        .map_err(|error| ParseBenchesError::Pattern(error.clone()))?;
    let Some(caps) = regex.captures(line) else {
        return Ok(None);
    };

    let name = caps[1].to_string();
    let ns_raw = caps[2].replace(',', "");
    let variance_raw = caps[3].replace(',', "");

    let ns_per_iter: f64 = ns_raw
        .parse()
        .map_err(|_| ParseBenchesError::InvalidNumber(caps[2].to_string()))?;
    let variance_ns: f64 = variance_raw
        .parse()
        .map_err(|_| ParseBenchesError::InvalidNumber(caps[3].to_string()))?;

    Ok(Some(NamedMetric {
        name,
        metric: BenchmarkMetric {
            ns_per_iter,
            variance_ns,
        },
    }))
}

/// Parses all benchmark metrics from a `.txt` file into the target map.
///
/// # Errors
///
/// Returns an error if the file cannot be read, numeric parsing fails, or a
/// duplicate benchmark name is encountered.
pub fn parse_bencher_file(
    path: &Path,
    benchmarks: &mut BTreeMap<String, BenchmarkMetric>,
) -> Result<(), ParseBenchesError> {
    let file = File::open(path)?;
    let reader = BufReader::new(file);

    for line in reader.lines() {
        let line = line?;
        if let Some(NamedMetric { name, metric }) = parse_bencher_line(&line)?
            && benchmarks.insert(name.clone(), metric).is_some()
        {
            return Err(ParseBenchesError::DuplicateBenchmark {
                name,
                file: path.to_path_buf(),
            });
        }
    }

    Ok(())
}

/// Scans the given directory for all `*.txt` files and parses their benchmark metrics.
///
/// # Errors
///
/// Returns an error if:
/// - The directory does not exist ([`ParseBenchesError::DirectoryNotFound`]).
/// - No `*.txt` files are found ([`ParseBenchesError::NoTxtFiles`]).
/// - No metrics were parsed from any found files ([`ParseBenchesError::NoMetricsParsed`]).
/// - Duplicate benchmark metrics are detected across files.
pub fn parse_benchmark_dir(
    results_dir: &Path,
) -> Result<BTreeMap<String, BenchmarkMetric>, ParseBenchesError> {
    if !results_dir.exists() || !results_dir.is_dir() {
        return Err(ParseBenchesError::DirectoryNotFound(
            results_dir.to_path_buf(),
        ));
    }

    let mut txt_files = Vec::new();
    for entry in fs::read_dir(results_dir)? {
        let entry = entry?;
        let path = entry.path();
        if path.is_file() && path.extension().and_then(|ext| ext.to_str()) == Some("txt") {
            txt_files.push(path);
        }
    }

    if txt_files.is_empty() {
        return Err(ParseBenchesError::NoTxtFiles(results_dir.to_path_buf()));
    }

    txt_files.sort();

    let mut benchmarks = BTreeMap::new();
    for file_path in &txt_files {
        parse_bencher_file(file_path, &mut benchmarks)?;
    }

    if benchmarks.is_empty() {
        return Err(ParseBenchesError::NoMetricsParsed(
            results_dir.to_path_buf(),
        ));
    }

    Ok(benchmarks)
}

/// Formats a [`SystemTime`] as an ISO8601 / RFC3339 UTC string (`YYYY-MM-DDTHH:MM:SSZ`).
#[must_use]
pub fn format_rfc3339_utc(time: SystemTime) -> String {
    let offset_time = time::OffsetDateTime::from(time)
        .replace_nanosecond(0)
        .unwrap_or_else(|_| time::OffsetDateTime::from(time));
    offset_time
        .format(&Rfc3339)
        .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_owned())
}

/// Resolves commit SHA from an optional CLI argument, falling back to an optional environment SHA.
#[must_use]
pub fn resolve_commit_sha(commit_arg: Option<&str>, env_sha: Option<&str>) -> String {
    let raw = commit_arg
        .and_then(|s| {
            let t = s.trim();
            (!t.is_empty()).then_some(t)
        })
        .or_else(|| {
            let s = env_sha?;
            let t = s.trim();
            (!t.is_empty()).then_some(t)
        });

    let Some(trimmed) = raw else {
        return "unknown".to_owned();
    };

    trimmed.to_owned()
}

/// Generates a [`BenchmarkSummary`] from the provided command-line arguments.
///
/// # Errors
///
/// Returns [`ParseBenchesError`] if directory scanning or metric parsing fails.
pub fn generate_summary(args: &Args) -> Result<BenchmarkSummary, ParseBenchesError> {
    let benchmarks = parse_benchmark_dir(&args.results_dir)?;
    let env_sha = env::var("GITHUB_SHA").ok();
    let commit = resolve_commit_sha(args.commit.as_deref(), env_sha.as_deref());
    let timestamp = format_rfc3339_utc(SystemTime::now());
    let executables = match (&args.executables, &args.cargo_log) {
        (Some(manifest), Some(cargo_log)) => {
            let manifest: ExecutableManifest = serde_json::from_reader(File::open(manifest)?)?;
            let source_dir = args.source_dir.clone().unwrap_or_default();
            verify_pass(
                &manifest,
                &fs::read_to_string(cargo_log)?,
                &source_dir,
                &commit,
            )?
        }
        _ => Vec::new(),
    };

    Ok(BenchmarkSummary {
        suite: args.suite.clone(),
        commit,
        timestamp,
        benchmarks,
        executables,
    })
}

/// Executes benchmark parsing and writes the resulting JSON summary to disk.
///
/// # Errors
///
/// Returns [`ParseBenchesError`] if parsing, directory creation, or file write fails.
pub fn run_from_args(args: &Args) -> Result<BenchmarkSummary, ParseBenchesError> {
    let summary = generate_summary(args)?;

    let out_path = args
        .output
        .clone()
        .unwrap_or_else(|| args.results_dir.join("broker-benchmarks.json"));

    if let Some(parent) = out_path.parent() {
        fs::create_dir_all(parent)?;
    }

    let json_bytes = serde_json::to_vec_pretty(&summary)?;
    let mut file = File::create(&out_path)?;
    file.write_all(&json_bytes)?;
    file.write_all(b"\n")?;

    Ok(summary)
}

fn write_json<T: Serialize>(path: &Path, value: &T) -> Result<(), BudgetError> {
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    let mut bytes = serde_json::to_vec_pretty(value)?;
    bytes.push(b'\n');
    fs::write(path, bytes)?;
    Ok(())
}

/// Run one subcommand and say what it found.
///
/// The returned message is printed on success. `Ok(false)` means the command
/// ran and the comparison was rejected; the verdict and Markdown are written
/// either way, so a rejection is published with its evidence.
///
/// # Errors
///
/// Returns [`ParseBenchesError`] when an input cannot be read, parsed or
/// judged.
pub fn run_command(command: &Command) -> Result<(bool, String), ParseBenchesError> {
    match command {
        Command::Executables {
            cargo_messages,
            commit,
            output,
        } => {
            let manifest =
                manifest_from_cargo_messages(&fs::read_to_string(cargo_messages)?, commit)?;
            write_json(output, &manifest)?;
            Ok((
                true,
                format!(
                    "{} bench executables of {commit} recorded in {}",
                    manifest.executables.len(),
                    output.display()
                ),
            ))
        }
        Command::CheckContract { contract } => {
            let bytes = fs::read(contract)?;
            let parsed = Contract::from_json(&bytes)?;
            Ok((
                true,
                format!(
                    "{}: {} cells, {} rounds, sha256 {}",
                    parsed.name,
                    parsed.cells.len(),
                    parsed.rounds,
                    contract_digest(&bytes)
                ),
            ))
        }
        Command::Assemble {
            artifact_dir,
            output,
        } => {
            let record = assemble(artifact_dir)?;
            write_json(output, &record)?;
            Ok((
                true,
                format!(
                    "{} trials written to {}",
                    record.trials.len(),
                    output.display()
                ),
            ))
        }
        Command::Evaluate {
            record,
            contract,
            output,
            markdown,
        } => {
            let contract_bytes = fs::read(contract)?;
            let parsed = Contract::from_json(&contract_bytes)?;
            let record = PerformanceRecord::from_json(&fs::read(record)?)?;
            let verdict = evaluate(&parsed, &contract_bytes, &record)?;
            write_json(output, &verdict)?;
            if let Some(markdown) = markdown {
                if let Some(parent) = markdown.parent() {
                    fs::create_dir_all(parent)?;
                }
                fs::write(markdown, render_markdown(&parsed, &record, &verdict))?;
            }
            let summary = if verdict.passed {
                format!("performance budget passed: {}", output.display())
            } else {
                format!(
                    "performance budget rejected ({} reasons): {}",
                    verdict.rejections.len(),
                    output.display()
                )
            };
            Ok((verdict.passed, summary))
        }
    }
}