krabka_remote_storage/
metadata_manager.rs

1//! The [`RemoteLogMetadataManager`] SPI, which stores and queries
2//! remote-segment metadata.
3
4use {
5    crate::{
6        error::RemoteStorageError,
7        metadata::{
8            RemoteLogSegmentMetadata, RemoteLogSegmentMetadataUpdate,
9            RemotePartitionDeleteMetadata, TopicIdPartition,
10        },
11    },
12    krabka_ids::LeaderEpoch,
13};
14
15/// SPI for the store that holds metadata about remote log segments.
16///
17/// The reference implementation is
18/// [`InmemoryRemoteLogMetadataManager`](crate::InmemoryRemoteLogMetadataManager);
19/// a production implementation stores the metadata in an internal topic, as
20/// Kafka's `TopicBasedRemoteLogMetadataManager` does. Implementations must be
21/// `Send + Sync`.
22///
23/// The lifecycle invariants every implementation enforces:
24///
25/// - The caller introduces a segment with
26///   [`add_remote_log_segment_metadata`](RemoteLogMetadataManager::add_remote_log_segment_metadata)
27///   in state
28///   [`CopySegmentStarted`](crate::RemoteLogSegmentState::CopySegmentStarted).
29/// - Every later change is an
30///   [`update_remote_log_segment_metadata`](RemoteLogMetadataManager::update_remote_log_segment_metadata)
31///   that follows a valid state transition.
32/// - Only
33///   [`CopySegmentFinished`](crate::RemoteLogSegmentState::CopySegmentFinished)
34///   segments are visible to the offset/epoch queries.
35pub trait RemoteLogMetadataManager: Send + Sync {
36    /// Records a newly-started segment copy. The metadata's state must be
37    /// [`CopySegmentStarted`](crate::RemoteLogSegmentState::CopySegmentStarted)
38    /// and its id must not already be known.
39    ///
40    /// # Errors
41    ///
42    /// Returns [`RemoteStorageError::InvalidAdd`] if the starting state is
43    /// wrong or the segment id is a duplicate.
44    fn add_remote_log_segment_metadata(
45        &self,
46        metadata: RemoteLogSegmentMetadata,
47    ) -> Result<(), RemoteStorageError>;
48
49    /// Advances a known segment's lifecycle state.
50    ///
51    /// # Errors
52    ///
53    /// Returns [`RemoteStorageError::SegmentNotFound`] if the segment is
54    /// unknown, or [`RemoteStorageError::InvalidSegmentTransition`] if the
55    /// transition is not permitted.
56    fn update_remote_log_segment_metadata(
57        &self,
58        update: RemoteLogSegmentMetadataUpdate,
59    ) -> Result<(), RemoteStorageError>;
60
61    /// Finds the finished segment that contains `offset` for the given
62    /// `leader_epoch`, if any.
63    ///
64    /// # Errors
65    ///
66    /// Returns [`RemoteStorageError`] only on an underlying store failure;
67    /// a simple "no such segment" is `Ok(None)`.
68    fn remote_log_segment_metadata(
69        &self,
70        topic_id_partition: &TopicIdPartition,
71        leader_epoch: LeaderEpoch,
72        offset: i64,
73    ) -> Result<Option<RemoteLogSegmentMetadata>, RemoteStorageError>;
74
75    /// The highest offset that the remote tier holds for `leader_epoch`.
76    ///
77    /// This is the maximum `end_offset` across the finished segments that
78    /// carry that epoch.
79    ///
80    /// # Errors
81    ///
82    /// Returns [`RemoteStorageError`] only on an underlying store failure.
83    fn highest_offset_for_epoch(
84        &self,
85        topic_id_partition: &TopicIdPartition,
86        leader_epoch: LeaderEpoch,
87    ) -> Result<Option<i64>, RemoteStorageError>;
88
89    /// All segments known for a partition, in `start_offset` order. The
90    /// state does not matter.
91    ///
92    /// # Errors
93    ///
94    /// Returns [`RemoteStorageError`] only on an underlying store failure.
95    fn list_remote_log_segments(
96        &self,
97        topic_id_partition: &TopicIdPartition,
98    ) -> Result<Vec<RemoteLogSegmentMetadata>, RemoteStorageError>;
99
100    /// All segments that carry `leader_epoch`, in `start_offset` order. The
101    /// state does not matter.
102    ///
103    /// # Errors
104    ///
105    /// Returns [`RemoteStorageError`] only on an underlying store failure.
106    fn list_remote_log_segments_by_epoch(
107        &self,
108        topic_id_partition: &TopicIdPartition,
109        leader_epoch: LeaderEpoch,
110    ) -> Result<Vec<RemoteLogSegmentMetadata>, RemoteStorageError>;
111
112    /// Records a partition-delete lifecycle event.
113    ///
114    /// # Errors
115    ///
116    /// Returns [`RemoteStorageError::InvalidPartitionDeleteTransition`] if
117    /// the transition is not permitted from the partition's current
118    /// delete-state.
119    fn put_remote_partition_delete_metadata(
120        &self,
121        metadata: RemotePartitionDeleteMetadata,
122    ) -> Result<(), RemoteStorageError>;
123
124    /// Whether this manager has applied every metadata event for
125    /// `topic_id_partition` that the metadata store had committed when the
126    /// call began.
127    ///
128    /// This is a read-your-peers'-writes check. Another broker can publish
129    /// an event, for example a `DeleteSegmentStarted` update, and wait for
130    /// the store to acknowledge it. A read on this manager that follows a
131    /// `true` answer sees that event. A `false` answer means the local view
132    /// can be stale, so the caller must not judge from it. A manager that
133    /// does not consume the partition's metadata answers `false`, because
134    /// it cannot vouch for the view.
135    ///
136    /// # Errors
137    ///
138    /// Returns [`RemoteStorageError::NotReady`] while the manager is still
139    /// catching up after it took on the partition, and
140    /// [`RemoteStorageError::Io`] if it cannot read the store's committed
141    /// position.
142    fn caught_up(&self, topic_id_partition: &TopicIdPartition) -> Result<bool, RemoteStorageError>;
143}
krabka_remote_storage/metadata_manager.rs

krabka_remote_storage/
metadata_manager.rs
