krabka_parse_benches/
executables.rs

1//! Which compiled benchmark executables a measurement pass ran.
2//!
3//! The nightly criterion lane measures a reference and a candidate revision on
4//! the same host. Cargo decides freshness per target directory, so when both
5//! revisions share one, a candidate that differs only in source can be judged
6//! fresh and the reference's binary runs again under the candidate's name. The
7//! lane therefore builds each revision into its own target directory before it
8//! measures either, and this module keeps the evidence:
9//!
10//! - [`manifest_from_cargo_messages`] records each bench executable that
11//!   `cargo bench --no-run --message-format=json` built, by canonical path and
12//!   SHA-256, in an [`ExecutableManifest`] for one commit.
13//! - [`verify_pass`] reads the `Running` lines Cargo printed during one pass,
14//!   and accepts the pass only when every executable it ran is in the manifest
15//!   and still has the recorded digest.
16//! - [`compare_summaries`](crate::compare_summaries) refuses a comparison whose
17//!   reference and candidate share an executable path, or whose passes on one
18//!   side ran different executables.
19
20use {
21    crate::ParseBenchesError,
22    serde::{Deserialize, Serialize},
23    sha2::{Digest, Sha256},
24    std::{
25        fs,
26        path::{Path, PathBuf},
27    },
28};
29
30/// One compiled benchmark executable.
31#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
32pub struct BenchExecutable {
33    /// The Cargo bench target name, such as `log`.
34    pub target: String,
35    /// The canonical path of the executable.
36    pub path: PathBuf,
37    /// The SHA-256 of its bytes, in lowercase hex.
38    pub sha256: String,
39}
40
41/// The bench executables that one revision was built into.
42#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
43pub struct ExecutableManifest {
44    /// The commit the executables were built from.
45    pub commit: String,
46    /// The executables, sorted by target name and path.
47    pub executables: Vec<BenchExecutable>,
48}
49
50/// The fields of one Cargo JSON message that identify a bench executable.
51#[derive(Deserialize)]
52struct CargoMessage {
53    reason: String,
54    #[serde(default)]
55    target: Option<CargoTarget>,
56    #[serde(default)]
57    executable: Option<PathBuf>,
58}
59
60#[derive(Deserialize)]
61struct CargoTarget {
62    name: String,
63    kind: Vec<String>,
64}
65
66/// The SHA-256 of the file at `path`, in lowercase hex.
67///
68/// # Errors
69///
70/// Returns [`ParseBenchesError::Io`] when the file cannot be read.
71pub fn file_digest(path: &Path) -> Result<String, ParseBenchesError> {
72    Ok(hex::encode(Sha256::digest(fs::read(path)?)))
73}
74
75/// Record the bench executables in Cargo's `--message-format=json` output.
76///
77/// Every `compiler-artifact` message for a `bench` target that names an
78/// executable contributes one entry, with its path canonicalized and its
79/// current bytes digested.
80///
81/// # Errors
82///
83/// Returns [`ParseBenchesError::Json`] for a line that is not a Cargo JSON
84/// message, [`ParseBenchesError::Io`] for an executable that cannot be read,
85/// and [`ParseBenchesError::NoBenchExecutables`] when no bench executable was
86/// built.
87pub fn manifest_from_cargo_messages(
88    messages: &str,
89    commit: &str,
90) -> Result<ExecutableManifest, ParseBenchesError> {
91    let mut executables = Vec::new();
92    for line in messages.lines().filter(|line| !line.trim().is_empty()) {
93        let message: CargoMessage = serde_json::from_str(line)?;
94        let (Some(target), Some(executable)) = (message.target, message.executable) else {
95            continue;
96        };
97        if message.reason != "compiler-artifact" || !target.kind.iter().any(|kind| kind == "bench")
98        {
99            continue;
100        }
101        let path = fs::canonicalize(&executable)?;
102        executables.push(BenchExecutable {
103            target: target.name,
104            sha256: file_digest(&path)?,
105            path,
106        });
107    }
108    if executables.is_empty() {
109        return Err(ParseBenchesError::NoBenchExecutables);
110    }
111    executables.sort();
112    executables.dedup();
113    Ok(ExecutableManifest {
114        commit: commit.to_owned(),
115        executables,
116    })
117}
118
119/// The executables Cargo reports running in `cargo_log`, resolved against
120/// `source_dir`.
121///
122/// Cargo prints `Running benches/log.rs (target/release/deps/log-0123)` to
123/// standard error before it starts each bench executable. The path is relative
124/// to the directory Cargo ran in when the target directory is below it, and
125/// absolute otherwise. With `CARGO_TERM_COLOR=always` Cargo wraps the
126/// `Running` label in ANSI colour sequences, which are removed before the
127/// line is read.
128#[must_use]
129pub fn ran_executables(cargo_log: &str, source_dir: &Path) -> Vec<PathBuf> {
130    cargo_log
131        .lines()
132        .filter_map(|line| {
133            let line = strip_ansi_csi(line);
134            let rest = line.trim().strip_prefix("Running ")?;
135            let inner = rest.strip_suffix(')')?;
136            let (_, path) = inner.rsplit_once(" (")?;
137            Some(source_dir.join(path))
138        })
139        .collect()
140}
141
142/// `line` without its ANSI control sequences: `ESC [`, any parameter and
143/// intermediate bytes, and the final byte in `@` to `~`.
144fn strip_ansi_csi(line: &str) -> String {
145    let mut plain = String::with_capacity(line.len());
146    let mut chars = line.chars().peekable();
147    while let Some(c) = chars.next() {
148        if c == '\x1b' && chars.peek() == Some(&'[') {
149            chars.next();
150            for c in chars.by_ref() {
151                if ('@'..='~').contains(&c) {
152                    break;
153                }
154            }
155        } else {
156            plain.push(c);
157        }
158    }
159    plain
160}
161
162/// Check that one measurement pass ran only the manifest's executables,
163/// unchanged, and return the executables it ran.
164///
165/// # Errors
166///
167/// Returns [`ParseBenchesError::CommitMismatch`] when the manifest was built
168/// from another commit, [`ParseBenchesError::NoExecutableRan`] when the log
169/// names no executable, [`ParseBenchesError::UnrecordedExecutable`] when the
170/// pass ran an executable the manifest does not list, and
171/// [`ParseBenchesError::ExecutableChanged`] when a listed executable's bytes
172/// no longer match its digest.
173pub fn verify_pass(
174    manifest: &ExecutableManifest,
175    cargo_log: &str,
176    source_dir: &Path,
177    commit: &str,
178) -> Result<Vec<BenchExecutable>, ParseBenchesError> {
179    if manifest.commit != commit {
180        return Err(ParseBenchesError::CommitMismatch {
181            manifest: manifest.commit.clone(),
182            pass: commit.to_owned(),
183        });
184    }
185    let ran = ran_executables(cargo_log, source_dir);
186    if ran.is_empty() {
187        return Err(ParseBenchesError::NoExecutableRan);
188    }
189    let mut verified = Vec::new();
190    for path in ran {
191        let canonical = fs::canonicalize(&path)
192            .map_err(|_| ParseBenchesError::UnrecordedExecutable(path.clone()))?;
193        let recorded = manifest
194            .executables
195            .iter()
196            .find(|executable| executable.path == canonical)
197            .ok_or_else(|| ParseBenchesError::UnrecordedExecutable(canonical.clone()))?;
198        if file_digest(&canonical)? != recorded.sha256 {
199            return Err(ParseBenchesError::ExecutableChanged(canonical));
200        }
201        verified.push(recorded.clone());
202    }
203    verified.sort();
204    verified.dedup();
205    Ok(verified)
206}
krabka_parse_benches/executables.rs

krabka_parse_benches/
executables.rs
