krabka_remote_storage_topic/
lib.rs

1//! Topic-backed [`RemoteLogMetadataManager`](krabka_remote_storage::RemoteLogMetadataManager) for Krabka, part of
2//! the KIP-405 tiered-storage stack.
3//!
4//! This crate ships [`TopicBasedRemoteLogMetadataManager`], the
5//! production replacement for [`krabka_remote_storage::InmemoryRemoteLogMetadataManager`].
6//! The manager appends remote-segment lifecycle events, which are add,
7//! update, and partition-delete, to an event log. In production that log is
8//! the `__remote_log_metadata` Kafka topic. Every broker rebuilds its local
9//! cache from the same log. After a restart, a broker re-reads the topic from
10//! offset 0 and re-applies the full history to recover its cache.
11//!
12//! ## What this crate provides
13//!
14//! - [`TopicBasedRemoteLogMetadataManager`] — the
15//!   [`RemoteLogMetadataManager`](krabka_remote_storage::RemoteLogMetadataManager)
16//!   implementation.
17//! - [`MetadataEventLog`] — the publish/subscribe seam between the
18//!   manager and the underlying durable transport.
19//! - [`InProcessMetadataEventLog`] — an in-memory fixture for unit
20//!   tests and for modelling the multi-broker case without bringing
21//!   up a real cluster (multiple managers cloned from the same `Arc`
22//!   observe each other's writes).
23//! - [`MetadataEvent`] and the [`serde`] module — the on-wire binary
24//!   codec for the three event variants.
25//! - [`metadata_partition_for`] — the
26//!   `TopicIdPartition → metadata-topic-partition` hash.
27//! - [`KafkaMetadataEventLog`] — the production [`MetadataEventLog`]
28//!   adapter that wires the trait to
29//!   [`krabka_client_producer`] / [`krabka_client_core`] /
30//!   [`krabka_client_admin`], persisting events in the
31//!   `__remote_log_metadata` topic. Reads use manual per-partition
32//!   `Fetch` loops over `krabka_client_core`, with no consumer group.
33//! - [`MetadataPartitionReadiness`] — the per-metadata-partition readiness
34//!   sample that
35//!   [`TopicBasedRemoteLogMetadataManager::metadata_partition_readiness`]
36//!   reports, which the broker exports as gauges.
37//! - [`SwappableRlmm`] — the hot-swap facade the broker boots behind so
38//!   it can start on the fail-closed `NotReadyRlmm` and upgrade to the
39//!   topic-backed manager once its listener is serving.
40//!   `Broker::start` selects the topic-backed manager when the
41//!   `[remote_storage.kafka_metadata]` config section is present and
42//!   `in_memory` is not set to `true`.
43//!
44//! ## Operational boundaries
45//!
46//! The metadata topic is an append-only event log created with delete cleanup
47//! and infinite retention. The manager maintains a local snapshot cache so
48//! restarts can resume from committed per-partition offsets instead of replaying
49//! the full topic every time. It does not use a Kafka consumer group or broker
50//! offset commits. The broker drives assignments explicitly with
51//! [`TopicBasedRemoteLogMetadataManager::reconcile_assignment`]. Internal Kafka
52//! clients use plaintext loopback by default, or the TLS/SASL settings supplied
53//! through [`KafkaMetadataLogConfig::security`].
54//!
55//! ## In-process manager for tests and local tools
56//!
57//! ```no_run
58//! use {
59//!     krabka_remote_storage_topic::{
60//!         InProcessMetadataEventLog, TopicBasedRemoteLogMetadataManager,
61//!     },
62//!     std::{path::PathBuf, time::Duration},
63//! };
64//!
65//! # async fn run() -> Result<(), Box<dyn std::error::Error>> {
66//! let event_log = InProcessMetadataEventLog::new(16);
67//! let manager = TopicBasedRemoteLogMetadataManager::start(
68//!     event_log,
69//!     tokio::runtime::Handle::current(),
70//!     PathBuf::from("/var/lib/krabka/rlmm-cache"),
71//!     Duration::from_secs(30),
72//! )?;
73//!
74//! manager.reconcile_assignment(&[0, 1]).await;
75//! # Ok(())
76//! # }
77//! ```
78
79#![doc(html_root_url = "https://docs.rs/krabka-remote-storage-topic/1.1.0")]
80
81pub mod error;
82pub mod kafka_log;
83pub mod log;
84pub mod manager;
85pub mod not_ready;
86pub mod partitioning;
87pub mod serde;
88pub mod snapshot;
89pub mod swappable;
90
91pub use {
92    error::{CodecError, MetadataLogError, SnapshotError},
93    kafka_log::{
94        DEFAULT_METADATA_EVENT_QUEUE_CAPACITY, DEFAULT_METADATA_FETCH_MAX_BYTES,
95        DEFAULT_METADATA_FETCH_MAX_WAIT, DEFAULT_METADATA_FETCH_RETRY_BACKOFF,
96        DEFAULT_METADATA_TOPIC_CREATE_TIMEOUT, DEFAULT_MIN_ISR, DEFAULT_NUM_PARTITIONS,
97        DEFAULT_REPLICATION, KafkaMetadataEventLog, KafkaMetadataLogConfig, METADATA_TOPIC,
98        MetadataEventQueueCapacity,
99    },
100    log::{
101        AssignmentHandle, InProcessMetadataEventLog, KeyedEvent, MetadataEventLog,
102        MetadataEventRecord, MetadataEventStream, PartitionStart, RangeVisitor,
103    },
104    manager::{MetadataPartitionReadiness, TopicBasedRemoteLogMetadataManager},
105    not_ready::NotReadyRlmm,
106    partitioning::{metadata_partition_for, metadata_partitions_for},
107    serde::MetadataEvent,
108    snapshot::{SNAPSHOT_FILE_NAME, SNAPSHOT_FORMAT_VERSION, Snapshot},
109    swappable::SwappableRlmm,
110};
krabka_remote_storage_topic/lib.rs

krabka_remote_storage_topic/
lib.rs
