krabka_parse_benches/
render.rs

1//! Render a record and its verdict as the Markdown that is published.
2//!
3//! The page states the contract it applies to before any number, and it
4//! names the two exact builds rather than "krabka" and "Kafka": a result here
5//! is a measured limit of one contract on one host, not a broker ranking.
6
7use {
8    crate::{
9        budget::{BudgetVerdict, Rejection, Status, metric_value},
10        contract::{Contract, Metric, Mode},
11        record::{PerformanceRecord, Side, TrialOutcome},
12    },
13    std::fmt::Write as _,
14};
15
16fn value(metric: Metric, value: f64) -> String {
17    match metric {
18        Metric::RecordsPerSecond => format!("{value:.0}"),
19        Metric::PeakRssMib => format!("{value:.1}"),
20        Metric::LatencyMsP50
21        | Metric::LatencyMsP95
22        | Metric::LatencyMsP99
23        | Metric::CpuMicrosPerRecord => format!("{value:.3}"),
24    }
25}
26
27fn percent(ratio: f64) -> String { format!("{:+.1}%", ratio * 100.0) }
28
29fn status(status: Status, gate_performance: bool) -> &'static str {
30    match status {
31        Status::WithinBudget => "within budget",
32        Status::Improved => "improved",
33        Status::NoisyNotMaterial => "noisy, not material",
34        Status::MaterialRegression if gate_performance => "**material regression**",
35        Status::MaterialRegression => "material difference (not gating)",
36        Status::ExcessVariance => "**excess variance**",
37        Status::NotBudgeted => "recorded",
38    }
39}
40
41fn rejection(rejection: &Rejection) -> String {
42    match rejection {
43        Rejection::CorrectnessFailure {
44            cell,
45            round,
46            side,
47            reason,
48        } => format!(
49            "correctness failure in `{cell}` round {round} ({}): {reason}",
50            side.name()
51        ),
52        Rejection::MissingTrial { cell, round, side } => {
53            format!("missing trial: `{cell}` round {round} ({})", side.name())
54        }
55        Rejection::NotInterleaved { cell, round } => format!(
56            "`{cell}` ran the same side first in rounds {} and {round}",
57            round - 1
58        ),
59        Rejection::MaterialRegression {
60            cell,
61            metric,
62            worse_change,
63            allowed_change,
64        } => format!(
65            "`{cell}` {} is {} worse; the budget allows {}",
66            metric.label(),
67            percent(*worse_change),
68            percent(*allowed_change)
69        ),
70        Rejection::ExcessVariance {
71            cell,
72            metric,
73            spread,
74            max_spread,
75        } => format!(
76            "`{cell}` {} spread is {}; the variance budget is {}",
77            metric.label(),
78            percent(*spread),
79            percent(*max_spread)
80        ),
81    }
82}
83
84/// The published Markdown for one comparison.
85#[must_use]
86pub fn render_markdown(
87    contract: &Contract,
88    record: &PerformanceRecord,
89    verdict: &BudgetVerdict,
90) -> String {
91    let mut out = String::new();
92    let reference = &record.reference;
93    let candidate = &record.candidate;
94    let against = match record.mode {
95        Mode::BeforeAfter => "an earlier krabka revision",
96        Mode::PinnedKafka => "the pinned Apache Kafka image",
97    };
98    let _ = writeln!(out, "# Performance budget: `{}`\n", contract.name);
99    let _ = writeln!(
100        out,
101        "These results compare one candidate build with {against} under the \
102         `{}` contract on the host below. They are measured limits of that \
103         contract, not a ranking of brokers and not a supported limit.\n",
104        contract.name
105    );
106    let _ = writeln!(
107        out,
108        "**Verdict: {}.**\n",
109        if verdict.passed { "pass" } else { "reject" }
110    );
111    if !verdict.rejections.is_empty() {
112        for item in &verdict.rejections {
113            let _ = writeln!(out, "- {}", rejection(item));
114        }
115        out.push('\n');
116    }
117
118    out.push_str("## Provenance\n\n| Side | Label | Revision | Image | Digest |\n| :--- | :--- | :--- | :--- | :--- |\n");
119    for (side, subject) in [(Side::Reference, reference), (Side::Candidate, candidate)] {
120        let _ = writeln!(
121            out,
122            "| {} | {} | `{}` | `{}` | `{}` |",
123            side.name(),
124            subject.label,
125            subject.revision,
126            subject.image,
127            subject.image_digest
128        );
129    }
130    let host = &record.host;
131    let _ = writeln!(
132        out,
133        "\nClient: `{}` (`{}`). Contract digest: `{}`. Started {}.\n",
134        record.client_image, record.client_image_digest, record.contract_sha256, record.started_at
135    );
136    let _ = writeln!(
137        out,
138        "Host: {}, {} logical CPUs, {} bytes of memory, {} bytes of free disk, \
139         kernel {}.\n",
140        host.cpu_model,
141        host.logical_cpus,
142        host.memory_bytes,
143        host.disk_available_bytes,
144        host.kernel
145    );
146    let cluster = &contract.cluster;
147    let _ = writeln!(
148        out,
149        "Cluster: {} combined broker/controllers, replication factor {}, \
150         `min.insync.replicas={}`, `acks=all`, idempotence, 65,536-byte batches, \
151         5 ms linger. {} interleaved rounds; each round runs every cell once per side.\n",
152        cluster.brokers, cluster.replication_factor, cluster.min_insync_replicas, contract.rounds
153    );
154    if !record.settings.is_empty() {
155        out.push_str("Settings:");
156        for (key, setting) in &record.settings {
157            let _ = write!(out, " `{key}={setting}`");
158        }
159        out.push_str(".\n\n");
160    }
161
162    out.push_str("## Budgets\n\nDeclared before the run. A regression is material when the median gets worse by more than the materiality floor and by more than the multiplier times the larger side's relative median absolute deviation. A side whose spread exceeds the variance budget makes the cell inconclusive.\n\n| Metric | Materiality | Spread multiplier | Variance budget |\n| :--- | ---: | ---: | ---: |\n");
163    for (metric, budget) in &contract.budgets {
164        let _ = writeln!(
165            out,
166            "| {} | {:.1}% | {:.1} | {:.1}% |",
167            metric.label(),
168            budget.materiality_percent,
169            budget.spread_multiplier,
170            budget.max_spread_percent
171        );
172    }
173
174    out.push_str("\n## Results\n\nMedians over the interleaved rounds. A change is relative to the reference median; positive is worse.\n\n| Cell | Metric | Reference | Candidate | Change | Spread (ref / cand) | Allowed | Status |\n| :--- | :--- | ---: | ---: | ---: | :--- | ---: | :--- |\n");
175    for cell in &contract.cells {
176        let Some(metrics) = verdict.cells.get(&cell.id) else {
177            let _ = writeln!(
178                out,
179                "| `{}` | all | — | — | — | — | — | not evaluated |",
180                cell.id
181            );
182            continue;
183        };
184        for (metric, judged) in metrics {
185            let allowed = if judged.status == Status::NotBudgeted {
186                "—".to_owned()
187            } else {
188                percent(judged.allowed_change)
189            };
190            let _ = writeln!(
191                out,
192                "| `{}` | {} | {} | {} | {} | {:.1}% / {:.1}% | {} | {} |",
193                cell.id,
194                metric.label(),
195                value(*metric, judged.reference_median),
196                value(*metric, judged.candidate_median),
197                percent(judged.worse_change),
198                judged.reference_spread * 100.0,
199                judged.candidate_spread * 100.0,
200                allowed,
201                status(judged.status, verdict.gate_performance)
202            );
203        }
204    }
205
206    out.push_str("\n## Raw trials\n\n| Order | Cell | Round | Side | records/s | p50 ms | p95 ms | p99 ms | CPU µs/record | peak RSS MiB | Outcome |\n| ---: | :--- | ---: | :--- | ---: | ---: | ---: | ---: | ---: | ---: | :--- |\n");
207    for trial in &record.trials {
208        let _ = write!(
209            out,
210            "| {} | `{}` | {} | {} |",
211            trial.order,
212            trial.cell,
213            trial.round,
214            trial.side.name()
215        );
216        match &trial.outcome {
217            TrialOutcome::Completed(measurement) => {
218                for metric in Metric::ALL {
219                    let _ = write!(
220                        out,
221                        " {} |",
222                        value(metric, metric_value(metric, measurement))
223                    );
224                }
225                let _ = writeln!(
226                    out,
227                    " sent {} / consumed {} / duplicates {} / errors {} |",
228                    measurement.sent,
229                    measurement.consumed,
230                    measurement.duplicates,
231                    measurement.errors
232                );
233            }
234            TrialOutcome::Failed { reason } => {
235                let _ = writeln!(out, " — | — | — | — | — | — | failed: {reason} |");
236            }
237        }
238    }
239    out
240}
krabka_parse_benches/render.rs

krabka_parse_benches/
render.rs
