krabka_raft/config/
metadata_log.rs

1//! Kafka's `MetadataLogConfig`: where the metadata log lives, how its
2//! segments roll, how long its prefix stays once a snapshot covers it, and how
3//! often an idle leader writes to it.
4//!
5//! The metadata log is the single partition `__cluster_metadata-0`. Kafka keeps
6//! it in a directory of that name under `metadata.log.dir`, beside its
7//! segments, its KIP-630 snapshots and its `quorum-state` file, and krabka
8//! keeps the same layout, so the JVM tools and Kafka's system tests find the
9//! files where they look for them.
10
11use {
12    krabka_units::{
13        fmt::Human as _,
14        prelude::{ByteSize, Time, days, gibibytes, mebibytes, millis},
15    },
16    std::path::{Path, PathBuf},
17};
18
19/// The name of the metadata partition's directory under the metadata log
20/// directory: Kafka's `UnifiedLog.logDirName` of `__cluster_metadata-0`.
21pub const METADATA_PARTITION_DIR: &str = "__cluster_metadata-0";
22
23/// `metadata.log.segment.bytes` default: 1 GiB.
24pub const DEFAULT_METADATA_LOG_SEGMENT_SIZE: ByteSize = gibibytes(1);
25
26/// The least `metadata.log.segment.bytes` Kafka accepts: 8 MiB. Kafka's
27/// `ConfigDef` refuses a smaller value, and only its internal
28/// `internal.metadata.log.segment.bytes` goes below it, for tests.
29pub const MIN_METADATA_LOG_SEGMENT_SIZE: ByteSize = mebibytes(8);
30
31/// `metadata.log.segment.ms` default: seven days.
32pub const DEFAULT_METADATA_LOG_SEGMENT_ROLL_INTERVAL: Time = days(7);
33
34/// `metadata.max.retention.bytes` default: 100 MiB.
35pub const DEFAULT_METADATA_MAX_RETENTION_SIZE: ByteSize = mebibytes(100);
36
37/// `metadata.max.retention.ms` default: seven days.
38pub const DEFAULT_METADATA_MAX_RETENTION: Time = days(7);
39
40/// `metadata.max.idle.interval.ms` default: 500 ms.
41pub const DEFAULT_METADATA_MAX_IDLE_INTERVAL: Time = millis(500);
42
43/// The metadata partition directory under `metadata_log_dir`, Kafka's
44/// `metadata.log.dir`. It holds the metadata log segments, the KIP-630
45/// `<end_offset>-<epoch>.checkpoint` snapshots and the `quorum-state` file.
46#[must_use]
47pub fn metadata_partition_dir(metadata_log_dir: &Path) -> PathBuf {
48    metadata_log_dir.join(METADATA_PARTITION_DIR)
49}
50
51/// How the metadata log rolls its segments and when it gives up the prefix
52/// that a snapshot covers. The defaults are Kafka's.
53///
54/// Quantities render in the operator form (`1GiB`, `7d`), as the rest of
55/// `ControllerConfig`'s `Debug` does.
56#[derive(Clone, Copy, PartialEq, derive_more::Debug, krabka_macros::FieldDefaults)]
57pub struct MetadataLogConfig {
58    /// `metadata.log.segment.bytes`: the active segment rolls before an
59    /// append that would take it past this size.
60    #[debug("{:?}", segment_size.human().to_string())]
61    #[default(DEFAULT_METADATA_LOG_SEGMENT_SIZE)]
62    pub segment_size: ByteSize,
63    /// `metadata.log.segment.ms`: the active segment rolls before an append
64    /// that comes this long after the segment's first record.
65    #[debug("{:?}", segment_roll_interval.human().to_string())]
66    #[default(DEFAULT_METADATA_LOG_SEGMENT_ROLL_INTERVAL)]
67    pub segment_roll_interval: Time,
68    /// `metadata.max.retention.bytes`: once the log and its snapshots together
69    /// are larger than this, the oldest snapshot goes and the log start moves
70    /// up to the next one. `None` is Kafka's negative value, which keeps every
71    /// snapshot whatever the size.
72    #[debug("{:?}", max_retention_size.map(|size| size.human().to_string()))]
73    #[default(Some(DEFAULT_METADATA_MAX_RETENTION_SIZE))]
74    pub max_retention_size: Option<ByteSize>,
75    /// `metadata.max.retention.ms`: a snapshot whose last record is older than
76    /// this goes, and the log start moves up to the next one. `None` is
77    /// Kafka's negative value, which keeps every snapshot whatever its age.
78    #[debug("{:?}", max_retention.map(|time| time.human().to_string()))]
79    #[default(Some(DEFAULT_METADATA_MAX_RETENTION))]
80    pub max_retention: Option<Time>,
81    /// `metadata.max.idle.interval.ms` (KIP-835): how often the leader appends
82    /// a `NoOpRecord`. Zero appends none.
83    #[debug("{:?}", max_idle_interval.human().to_string())]
84    #[default(DEFAULT_METADATA_MAX_IDLE_INTERVAL)]
85    pub max_idle_interval: Time,
86}
87
88#[cfg(test)]
89mod tests {
90    use {
91        super::*,
92        assert2::check,
93        krabka_units::prelude::{ByteSizeExt, TimeExt},
94    };
95
96    /// The defaults are the numbers Kafka's `MetadataLogConfig` documents.
97    #[test]
98    fn defaults_are_kafkas() {
99        let config = MetadataLogConfig::default();
100        check!(
101            (
102                config.segment_size.bytes_u64(),
103                config.segment_roll_interval.millis_i64(),
104                config.max_retention_size.map(ByteSizeExt::bytes_u64),
105                config.max_retention.map(TimeExt::millis_i64),
106                config.max_idle_interval.millis_i64(),
107            ) == (
108                1024 * 1024 * 1024,
109                7 * 24 * 60 * 60 * 1000,
110                Some(100 * 1024 * 1024),
111                Some(7 * 24 * 60 * 60 * 1000),
112                500,
113            )
114        );
115        check!(MIN_METADATA_LOG_SEGMENT_SIZE.bytes_u64() == 8 * 1024 * 1024);
116    }
117
118    #[test]
119    fn the_partition_directory_is_kafkas() {
120        check!(
121            metadata_partition_dir(Path::new("/mnt/kafka/kafka-metadata-logs"))
122                == PathBuf::from("/mnt/kafka/kafka-metadata-logs/__cluster_metadata-0")
123        );
124    }
125}
krabka_raft/config/metadata_log.rs

krabka_raft/config/
metadata_log.rs
