krabka_object_store/
config.rs

1//! Object-store connection config types shared across Krabka.
2
3use std::{path::PathBuf, time::Duration};
4
5/// Default threshold above which a segment upload changes from a single PUT to
6/// a streaming multipart upload. The threshold is 100 MiB, well below AWS's
7/// 5 GiB single-PUT cap.
8pub const DEFAULT_MULTIPART_THRESHOLD: u64 = 100 * 1024 * 1024;
9
10/// Default per-part size for multipart uploads. The size is 16 MiB. AWS
11/// requires >= 5 MiB per non-final part and caps the parts at 10 000, so 16 MiB
12/// scales past any real segment.
13pub const DEFAULT_MULTIPART_CHUNK_SIZE: usize = 16 * 1024 * 1024;
14
15/// Default number of times a single object-store request is retried before the
16/// error reaches the caller. It matches `object_store`'s own default of 10.
17pub const DEFAULT_MAX_RETRIES: usize = 10;
18
19/// Default ceiling on the wall-clock time one request may spend across all of
20/// its retries.
21///
22/// It is 3 minutes, `object_store`'s default, which stays under
23/// the 5-minute mark past which a request's credentials may expire mid-retry.
24pub const DEFAULT_RETRY_TIMEOUT: Duration = Duration::from_mins(3);
25
26/// Default ceiling on one HTTP request, connect phase included. It is 30
27/// seconds, `object_store`'s default, chosen because an object-store request
28/// may transfer a whole segment.
29pub const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
30
31/// Default ceiling on the connect phase alone. It is 5 seconds, matching
32/// `object_store`.
33pub const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
34
35/// Selects and parameterises the object-store backend to construct.
36#[derive(Clone, Debug)]
37pub enum ObjectStoreConfig {
38    /// Any S3-compatible endpoint, such as AWS S3, `MinIO`, or Cloudflare R2.
39    S3(S3Config),
40    /// Native Google Cloud Storage. It supports keyless GKE Workload Identity.
41    Gcs(GcsConfig),
42    /// Local filesystem rooted at `root`, for development and test.
43    Local { root: PathBuf },
44    /// In-process store, for tests.
45    InMemory,
46}
47
48/// Connection and bucket parameters for an S3-compatible backend.
49///
50/// The credentials come either from `access_key_id` and `secret_access_key` or
51/// from the standard AWS credential chain. When both fields are `None`,
52/// `object_store` falls back to the environment-variable chain.
53///
54/// `Debug` redacts the credential fields, so a stray `{:?}` call or tracing
55/// call never leaks them.
56#[krabka_macros::s3_fields(runtime)]
57#[krabka_macros::object_store_config]
58#[derive(Clone, PartialEq, Eq, derive_more::Debug, krabka_macros::FieldDefaults)]
59pub struct S3Config {
60    /// S3 bucket name.
61    pub bucket: String,
62    /// Optional key prefix inside the bucket. No leading slash and no trailing
63    /// slash.
64    pub prefix: Option<String>,
65    /// AWS region. AWS S3 requires it. Use the placeholder `"us-east-1"` for
66    /// `MinIO` and R2.
67    pub region: String,
68    /// Allow plaintext HTTP. `MinIO` without TLS requires it.
69    pub allow_http: bool,
70    /// Files of at least this size upload with multipart. Defaults to
71    /// [`DEFAULT_MULTIPART_THRESHOLD`].
72    #[default(DEFAULT_MULTIPART_THRESHOLD)]
73    pub multipart_threshold: u64,
74    /// Per-part size for multipart. Defaults to [`DEFAULT_MULTIPART_CHUNK_SIZE`].
75    #[default(DEFAULT_MULTIPART_CHUNK_SIZE)]
76    pub multipart_chunk_size: usize,
77    /// Enable conditional puts (`If-None-Match`) so a `PutMode::Create` write
78    /// fails instead of silently overwriting. Defaults to `true`.
79    #[default(true)]
80    pub conditional_put: bool,
81    /// Send `x-amz-checksum-sha256` so the server verifies each object on
82    /// ingest. Defaults to `true`.
83    #[default(true)]
84    pub checksum_sha256: bool,
85}
86
87/// Connection and bucket parameters for native Google Cloud Storage.
88///
89/// If every credential field is `None`, the config selects Workload Identity or
90/// ADC through the metadata server. This is the keyless GKE production path.
91///
92/// `Debug` redacts the credential fields, so a stray `{:?}` call or tracing
93/// call never leaks them.
94#[krabka_macros::gcs_fields(runtime)]
95#[krabka_macros::object_store_config]
96#[derive(Clone, PartialEq, Eq, derive_more::Debug, krabka_macros::FieldDefaults)]
97pub struct GcsConfig {
98    /// Allow plaintext HTTP. Emulators without TLS require it.
99    pub allow_http: bool,
100    /// Files of at least this size upload with resumable multipart. Defaults
101    /// to [`DEFAULT_MULTIPART_THRESHOLD`].
102    #[default(DEFAULT_MULTIPART_THRESHOLD)]
103    pub multipart_threshold: u64,
104    /// Per-part size for multipart. Defaults to [`DEFAULT_MULTIPART_CHUNK_SIZE`].
105    #[default(DEFAULT_MULTIPART_CHUNK_SIZE)]
106    pub multipart_chunk_size: usize,
107}
108
109#[cfg(test)]
110mod tests {
111    use {
112        super::*,
113        assert2::{assert, check},
114    };
115
116    #[test]
117    fn multipart_size_constants() {
118        assert!(DEFAULT_MULTIPART_THRESHOLD == 100 * 1024 * 1024);
119        assert!(DEFAULT_MULTIPART_CHUNK_SIZE == 16 * 1024 * 1024);
120    }
121
122    #[test]
123    fn s3_config_default_uses_multipart_constants() {
124        let cfg = S3Config::default();
125        assert!(cfg.multipart_threshold == DEFAULT_MULTIPART_THRESHOLD);
126        assert!(cfg.multipart_chunk_size == DEFAULT_MULTIPART_CHUNK_SIZE);
127    }
128
129    #[test]
130    fn s3_config_default_enables_integrity_knobs() {
131        let cfg = S3Config::default();
132        assert!(cfg.conditional_put);
133        assert!(cfg.checksum_sha256);
134    }
135
136    #[test]
137    fn s3_config_debug_shows_integrity_knobs() {
138        let dbg = format!("{:?}", S3Config::default());
139        assert!(dbg.contains("conditional_put: true"));
140        assert!(dbg.contains("checksum_sha256: true"));
141    }
142
143    #[test]
144    fn s3_config_debug_redacts_credentials() {
145        let cfg = S3Config {
146            bucket: "b".into(),
147            region: "us-east-1".into(),
148            access_key_id: Some("AKIASECRET".into()),
149            secret_access_key: Some("supersecret".into()),
150            ..Default::default()
151        };
152        let dbg = format!("{cfg:?}");
153        assert!(!dbg.contains("AKIASECRET"));
154        assert!(!dbg.contains("supersecret"));
155        assert!(dbg.contains("***"));
156    }
157
158    #[test]
159    fn gcs_config_default_uses_multipart_constants() {
160        let cfg = GcsConfig::default();
161        assert!(cfg.multipart_threshold == DEFAULT_MULTIPART_THRESHOLD);
162        assert!(cfg.multipart_chunk_size == DEFAULT_MULTIPART_CHUNK_SIZE);
163    }
164
165    #[test]
166    fn gcs_config_debug_redacts_credentials() {
167        let cfg = GcsConfig {
168            bucket: "b".into(),
169            service_account_key: Some("{\"private_key\":\"leak\"}".into()),
170            ..Default::default()
171        };
172        let dbg = format!("{cfg:?}");
173        assert!(!dbg.contains("leak"));
174        assert!(dbg.contains("***"));
175    }
176
177    #[test]
178    fn object_store_config_debug_redacts_via_inner() {
179        let cfg = ObjectStoreConfig::S3(S3Config {
180            secret_access_key: Some("supersecret".into()),
181            ..Default::default()
182        });
183        assert!(!format!("{cfg:?}").contains("supersecret"));
184    }
185
186    /// The four durability knobs must default to the values documented in
187    /// `docs/config-reference.md`, on both backends: an operator who sets
188    /// none of them gets exactly this.
189    #[yare::parameterized(
190        s3 = { "s3", false },
191        gcs = { "gcs", true },
192    )]
193    fn backend_defaults_carry_the_documented_bounds(name: &str, gcs: bool) {
194        let (retries, retry, request, connect) = if gcs {
195            let cfg = GcsConfig::default();
196            (
197                cfg.max_retries,
198                cfg.retry_timeout,
199                cfg.request_timeout,
200                cfg.connect_timeout,
201            )
202        } else {
203            let cfg = S3Config::default();
204            (
205                cfg.max_retries,
206                cfg.retry_timeout,
207                cfg.request_timeout,
208                cfg.connect_timeout,
209            )
210        };
211        check!(retries == 10, "{name}");
212        check!(retry == Duration::from_secs(180), "{name}");
213        check!(request == Duration::from_secs(30), "{name}");
214        check!(connect == Duration::from_secs(5), "{name}");
215    }
216
217    /// The bounds are not credential material, so a redacting `Debug` must
218    /// still print them: an operator reading a startup log is looking for the
219    /// value that is timing their store out.
220    #[yare::parameterized(
221        s3_retries = { false, "max_retries: 10" },
222        s3_retry_timeout = { false, "retry_timeout: 180s" },
223        s3_request_timeout = { false, "request_timeout: 30s" },
224        s3_connect_timeout = { false, "connect_timeout: 5s" },
225        gcs_retries = { true, "max_retries: 10" },
226        gcs_retry_timeout = { true, "retry_timeout: 180s" },
227        gcs_request_timeout = { true, "request_timeout: 30s" },
228        gcs_connect_timeout = { true, "connect_timeout: 5s" },
229    )]
230    fn debug_prints_the_bounds_on_both_backends(gcs: bool, field: &str) {
231        let dbg = if gcs {
232            format!("{:?}", GcsConfig::default())
233        } else {
234            format!("{:?}", S3Config::default())
235        };
236        check!(dbg.contains(field), "{field} missing from {dbg}");
237    }
238}
krabka_object_store/config.rs

krabka_object_store/
config.rs
