1use krabka_protocol::owned::{
7 common::streams_group_heartbeat_request::{key_value::KeyValue, topic_info::TopicInfo},
8 streams_group_heartbeat_request::{CopartitionGroup, Subtopology, Topology},
9};
10use krabka_units::prelude::*;
11use serde::Serialize;
12
13use super::grouping::GroupTopics;
14
15const INTERNAL_TOPIC_DEFAULT_RF: i16 = -1;
20
21const REPARTITION_SEGMENT_SIZE: ByteSize = mebibytes(50);
24
25fn repartition_topic_configs() -> Vec<KeyValue> {
32 topic_configs([
33 ("cleanup.policy", "delete".to_string()),
34 ("message.timestamp.type", "CreateTime".to_string()),
35 (
36 "retention.ms",
37 krabka_units::convert::wire::opt_time_to_millis_i64(None).to_string(),
38 ),
39 (
40 "segment.bytes",
41 REPARTITION_SEGMENT_SIZE.bytes_i64().to_string(),
42 ),
43 ])
44}
45
46fn changelog_topic_configs() -> Vec<KeyValue> {
49 topic_configs([
50 ("cleanup.policy", "compact".to_string()),
51 ("message.timestamp.type", "CreateTime".to_string()),
52 ])
53}
54
55fn windowed_changelog_topic_configs(retention: Time) -> Vec<KeyValue> {
65 vec![
66 KeyValue {
67 key: "cleanup.policy".into(),
68 value: "compact,delete".into(),
69 ..Default::default()
70 },
71 KeyValue {
72 key: "message.timestamp.type".into(),
73 value: "CreateTime".into(),
74 ..Default::default()
75 },
76 KeyValue {
77 key: "retention.ms".into(),
78 value: retention.millis_i64().to_string(),
79 ..Default::default()
80 },
81 ]
82}
83
84fn versioned_changelog_topic_configs(min_compaction_lag: Time) -> Vec<KeyValue> {
94 vec![
95 KeyValue {
96 key: "cleanup.policy".into(),
97 value: "compact".into(),
98 ..Default::default()
99 },
100 KeyValue {
101 key: "message.timestamp.type".into(),
102 value: "CreateTime".into(),
103 ..Default::default()
104 },
105 KeyValue {
106 key: "min.compaction.lag.ms".into(),
107 value: min_compaction_lag.millis_i64().to_string(),
108 ..Default::default()
109 },
110 ]
111}
112
113fn join_window_changelog_topic_configs(retention: Time) -> Vec<KeyValue> {
122 vec![
123 KeyValue {
124 key: "cleanup.policy".into(),
125 value: "delete".into(),
126 ..Default::default()
127 },
128 KeyValue {
129 key: "message.timestamp.type".into(),
130 value: "CreateTime".into(),
131 ..Default::default()
132 },
133 KeyValue {
134 key: "retention.ms".into(),
135 value: retention.millis_i64().to_string(),
136 ..Default::default()
137 },
138 ]
139}
140
141fn topic_configs<const N: usize>(pairs: [(&str, String); N]) -> Vec<KeyValue> {
144 pairs
145 .into_iter()
146 .map(|(key, value)| KeyValue {
147 key: key.to_string(),
148 value,
149 ..Default::default()
150 })
151 .collect()
152}
153
154pub(crate) fn changelog_topic_name(
158 application_id: &str,
159 store: &str,
160 changelog_override: Option<&str>,
161) -> String {
162 changelog_override.map_or_else(
163 || format!("{application_id}-{store}-changelog"),
164 str::to_string,
165 )
166}
167
168pub(crate) fn to_wire(groups: &[GroupTopics], application_id: &str) -> Topology {
171 let mut subtopologies: Vec<Subtopology> = groups
172 .iter()
173 .map(|g| subtopology(g, application_id))
174 .collect();
175 subtopologies.sort_by(|a, b| a.subtopology_id.cmp(&b.subtopology_id));
176 Topology {
177 epoch: 0,
178 subtopologies,
179 ..Default::default()
180 }
181}
182
183fn subtopology(g: &GroupTopics, app: &str) -> Subtopology {
184 let mut source_topics = g.source_topics.clone();
185 source_topics.sort();
186 let mut repartition_sink_topics = g.repartition_sink_topics.clone();
187 repartition_sink_topics.sort();
188
189 let mut repartition_source_topics: Vec<TopicInfo> = g
190 .repartition_source_topics
191 .iter()
192 .map(|name| TopicInfo {
193 name: name.clone(),
194 partitions: 0,
195 replication_factor: INTERNAL_TOPIC_DEFAULT_RF,
196 topic_configs: repartition_topic_configs(),
197 ..Default::default()
198 })
199 .collect();
200 repartition_source_topics.sort_by(|a, b| a.name.cmp(&b.name));
201
202 let repartition_names: Vec<String> = repartition_source_topics
205 .iter()
206 .map(|t| t.name.clone())
207 .collect();
208 let copartition_groups = g
209 .copartition_groups
210 .iter()
211 .map(|members| copartition_group(&source_topics, &repartition_names, members))
212 .collect();
213
214 let mut state_changelog_topics: Vec<TopicInfo> = g
215 .changelog_stores
216 .iter()
217 .map(|(store, changelog_override, changelog_kind)| TopicInfo {
218 name: changelog_topic_name(app, store, changelog_override.as_deref()),
219 partitions: 0,
220 replication_factor: INTERNAL_TOPIC_DEFAULT_RF,
221 topic_configs: match changelog_kind {
222 crate::topology::node::ChangelogKind::Kv => changelog_topic_configs(),
223 crate::topology::node::ChangelogKind::AggWindow { retention } => {
224 windowed_changelog_topic_configs(*retention)
225 }
226 crate::topology::node::ChangelogKind::JoinWindow { retention } => {
227 join_window_changelog_topic_configs(*retention)
228 }
229 crate::topology::node::ChangelogKind::Versioned { min_compaction_lag } => {
230 versioned_changelog_topic_configs(*min_compaction_lag)
231 }
232 },
233 ..Default::default()
234 })
235 .collect();
236 state_changelog_topics.sort_by(|a, b| a.name.cmp(&b.name));
237
238 Subtopology {
239 subtopology_id: g.id.clone(),
240 source_topics,
241 source_topic_regex: Vec::new(),
242 state_changelog_topics,
243 repartition_sink_topics,
244 repartition_source_topics,
245 copartition_groups,
246 ..Default::default()
247 }
248}
249
250#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
267pub struct WireTopology {
268 pub epoch: i32,
269 pub subtopologies: Vec<WireSubtopology>,
270}
271
272#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
274pub struct WireSubtopology {
275 pub subtopology_id: String,
276 pub source_topics: Vec<String>,
277 pub source_topic_regex: Vec<String>,
278 pub repartition_sink_topics: Vec<String>,
279 pub repartition_source_topics: Vec<WireTopicInfo>,
280 pub state_changelog_topics: Vec<WireTopicInfo>,
281 pub copartition_groups: Vec<WireCopartitionGroup>,
282}
283
284#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
286pub struct WireTopicInfo {
287 pub name: String,
288 pub partitions: i32,
289 pub replication_factor: i16,
290 pub topic_configs: Vec<WireKeyValue>,
291}
292
293#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
295pub struct WireKeyValue {
296 pub key: String,
297 pub value: String,
298}
299
300#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
303pub struct WireCopartitionGroup {
304 pub source_topics: Vec<i16>,
305 pub source_topic_regex: Vec<i16>,
306 pub repartition_source_topics: Vec<i16>,
307}
308
309impl From<&Topology> for WireTopology {
310 fn from(t: &Topology) -> Self {
311 WireTopology {
312 epoch: t.epoch,
313 subtopologies: t.subtopologies.iter().map(WireSubtopology::from).collect(),
314 }
315 }
316}
317
318impl From<&Subtopology> for WireSubtopology {
319 fn from(s: &Subtopology) -> Self {
320 WireSubtopology {
321 subtopology_id: s.subtopology_id.clone(),
322 source_topics: s.source_topics.clone(),
323 source_topic_regex: s.source_topic_regex.clone(),
324 repartition_sink_topics: s.repartition_sink_topics.clone(),
325 repartition_source_topics: s
326 .repartition_source_topics
327 .iter()
328 .map(WireTopicInfo::from)
329 .collect(),
330 state_changelog_topics: s
331 .state_changelog_topics
332 .iter()
333 .map(WireTopicInfo::from)
334 .collect(),
335 copartition_groups: s
336 .copartition_groups
337 .iter()
338 .map(WireCopartitionGroup::from)
339 .collect(),
340 }
341 }
342}
343
344impl From<&TopicInfo> for WireTopicInfo {
345 fn from(t: &TopicInfo) -> Self {
346 WireTopicInfo {
347 name: t.name.clone(),
348 partitions: t.partitions,
349 replication_factor: t.replication_factor,
350 topic_configs: t
351 .topic_configs
352 .iter()
353 .map(|kv| WireKeyValue {
354 key: kv.key.clone(),
355 value: kv.value.clone(),
356 })
357 .collect(),
358 }
359 }
360}
361
362impl From<&CopartitionGroup> for WireCopartitionGroup {
363 fn from(c: &CopartitionGroup) -> Self {
364 WireCopartitionGroup {
365 source_topics: c.source_topics.clone(),
366 source_topic_regex: c.source_topic_regex.clone(),
367 repartition_source_topics: c.repartition_source_topics.clone(),
368 }
369 }
370}
371
372pub(crate) fn copartition_group(
379 sources: &[String],
380 repartition: &[String],
381 members: &[String],
382) -> CopartitionGroup {
383 let mut source_topics = Vec::new();
384 let mut repartition_source_topics = Vec::new();
385 for m in members {
386 if let Some(i) = sources.iter().position(|s| s == m) {
387 source_topics.push(i16::try_from(i).unwrap_or(i16::MAX));
388 } else if let Some(i) = repartition.iter().position(|s| s == m) {
389 repartition_source_topics.push(i16::try_from(i).unwrap_or(i16::MAX));
390 }
391 }
392 source_topics.sort_unstable();
393 repartition_source_topics.sort_unstable();
394 CopartitionGroup {
395 source_topics,
396 source_topic_regex: Vec::new(),
397 repartition_source_topics,
398 ..Default::default()
399 }
400}
401
402#[cfg(test)]
403mod tests {
404 use assert2::check;
405
406 use super::*;
407 use crate::topology::{grouping::GroupTopics, node::ChangelogKind};
408
409 #[test]
410 fn wire_topology_serializes_to_fixture_shape_with_topic_info() {
411 use krabka_protocol::owned::common::streams_group_heartbeat_request::key_value::KeyValue;
412 let proto = Topology {
415 epoch: 0,
416 subtopologies: vec![Subtopology {
417 subtopology_id: "0".into(),
418 source_topics: vec!["in".into()],
419 state_changelog_topics: vec![TopicInfo {
420 name: "app-store-changelog".into(),
421 partitions: 0,
422 replication_factor: -1,
423 topic_configs: vec![KeyValue {
424 key: "cleanup.policy".into(),
425 value: "compact".into(),
426 ..Default::default()
427 }],
428 ..Default::default()
429 }],
430 ..Default::default()
431 }],
432 ..Default::default()
433 };
434 let view = WireTopology::from(&proto);
435 let json = serde_json::to_value(&view).unwrap();
436 check!(
439 json["subtopologies"][0]["state_changelog_topics"][0]["name"] == "app-store-changelog"
440 );
441 check!(json["subtopologies"][0]["state_changelog_topics"][0]["replication_factor"] == -1);
442 check!(
443 json["subtopologies"][0]["state_changelog_topics"][0]["topic_configs"][0]["key"]
444 == "cleanup.policy"
445 );
446 check!(
447 json["subtopologies"][0]
448 .get("unknown_tagged_fields")
449 .is_none()
450 );
451 check!(
452 json["subtopologies"][0]["state_changelog_topics"][0]
453 .get("unknown_tagged_fields")
454 .is_none()
455 );
456 }
457
458 #[test]
459 fn wire_copartition_group_projects_indices() {
460 let proto = CopartitionGroup {
461 source_topics: vec![0, 2],
462 source_topic_regex: Vec::new(),
463 repartition_source_topics: vec![1],
464 ..Default::default()
465 };
466 let view = WireCopartitionGroup::from(&proto);
467 check!(view.source_topics == vec![0i16, 2i16]);
468 check!(view.repartition_source_topics == vec![1i16]);
469 let json = serde_json::to_value(&view).unwrap();
470 check!(json.get("unknown_tagged_fields").is_none());
471 }
472
473 #[test]
474 fn epoch_is_zero_and_source_topics_sorted() {
475 let groups = vec![GroupTopics {
476 id: "0".into(),
477 source_topics: vec!["b".into(), "a".into()],
478 ..Default::default()
479 }];
480 let topo = to_wire(&groups, "app");
481 check!(topo.epoch == 0);
482 check!(topo.subtopologies[0].source_topics == vec!["a".to_string(), "b".to_string()]);
483 check!(topo.subtopologies[0].source_topic_regex.is_empty());
484 }
485
486 #[test]
487 fn subtopologies_sort_by_id_as_string_not_numeric() {
488 let groups = vec![
489 GroupTopics {
490 id: "2".into(),
491 source_topics: vec!["x".into()],
492 ..Default::default()
493 },
494 GroupTopics {
495 id: "10".into(),
496 source_topics: vec!["x".into()],
497 ..Default::default()
498 },
499 GroupTopics {
500 id: "1".into(),
501 source_topics: vec!["x".into()],
502 ..Default::default()
503 },
504 ];
505 let topo = to_wire(&groups, "app");
506 let ids: Vec<&str> = topo
507 .subtopologies
508 .iter()
509 .map(|s| s.subtopology_id.as_str())
510 .collect();
511 check!(ids == vec!["1", "10", "2"]);
512 }
513
514 #[test]
515 fn changelog_topics_named_zero_partitions_default_rf_and_configs() {
516 let groups = vec![GroupTopics {
517 id: "0".into(),
518 source_topics: vec!["in".into()],
519 changelog_stores: vec![("store".into(), None, ChangelogKind::Kv)],
520 ..Default::default()
521 }];
522 let topo = to_wire(&groups, "my-app");
523 let cl = &topo.subtopologies[0].state_changelog_topics;
524 check!(cl.len() == 1);
525 check!(cl[0].name == "my-app-store-changelog");
526 check!(cl[0].partitions == 0);
527 check!(cl[0].replication_factor == -1);
530 let configs: Vec<(&str, &str)> = cl[0]
531 .topic_configs
532 .iter()
533 .map(|kv| (kv.key.as_str(), kv.value.as_str()))
534 .collect();
535 check!(
536 configs
537 == vec![
538 ("cleanup.policy", "compact"),
539 ("message.timestamp.type", "CreateTime"),
540 ]
541 );
542 }
543
544 #[test]
545 fn changelog_override_uses_source_topic_name_verbatim() {
546 let groups = vec![GroupTopics {
550 id: "0".into(),
551 source_topics: vec!["in".into()],
552 changelog_stores: vec![("store".into(), Some("in".into()), ChangelogKind::Kv)],
553 ..Default::default()
554 }];
555 let topo = to_wire(&groups, "my-app");
556 let cl = &topo.subtopologies[0].state_changelog_topics;
557 check!(cl.len() == 1);
558 check!(cl[0].name == "in");
559 check!(cl[0].replication_factor == -1);
560 let configs: Vec<(&str, &str)> = cl[0]
561 .topic_configs
562 .iter()
563 .map(|kv| (kv.key.as_str(), kv.value.as_str()))
564 .collect();
565 check!(
566 configs
567 == vec![
568 ("cleanup.policy", "compact"),
569 ("message.timestamp.type", "CreateTime"),
570 ]
571 );
572 }
573
574 #[test]
575 fn repartition_source_topics_carry_default_rf_and_sorted_configs() {
576 let groups = vec![GroupTopics {
577 id: "1".into(),
578 repartition_source_topics: vec!["my-app-store-repartition".into()],
579 ..Default::default()
580 }];
581 let topo = to_wire(&groups, "my-app");
582 let rp = &topo.subtopologies[0].repartition_source_topics;
583 check!(rp.len() == 1);
584 check!(rp[0].name == "my-app-store-repartition");
585 check!(rp[0].partitions == 0);
586 check!(rp[0].replication_factor == -1);
587 let configs: Vec<(&str, &str)> = rp[0]
588 .topic_configs
589 .iter()
590 .map(|kv| (kv.key.as_str(), kv.value.as_str()))
591 .collect();
592 check!(
593 configs
594 == vec![
595 ("cleanup.policy", "delete"),
596 ("message.timestamp.type", "CreateTime"),
597 ("retention.ms", "-1"),
598 ("segment.bytes", "52428800"),
599 ]
600 );
601 }
602
603 #[test]
604 fn copartition_indices_point_into_sorted_arrays() {
605 let sources = vec!["a".to_string(), "b".to_string(), "c".to_string()];
606 let repartition: Vec<String> = vec![];
607 let cg = copartition_group(&sources, &repartition, &["c".into(), "a".into()]);
608 check!(cg.source_topics == vec![0i16, 2i16]);
609 check!(cg.repartition_source_topics.is_empty());
610 check!(cg.source_topic_regex.is_empty());
611 }
612
613 #[test]
614 fn copartition_indices_into_repartition_array() {
615 let sources = vec!["a".to_string()];
616 let repartition = vec!["rp0".to_string(), "rp1".to_string()];
617 let cg = copartition_group(&sources, &repartition, &["rp1".into(), "a".into()]);
618 check!(cg.source_topics == vec![0i16]);
619 check!(cg.repartition_source_topics == vec![1i16]);
620 }
621
622 #[test]
623 fn windowed_store_changelog_config_is_compact_delete_with_retention() {
624 let groups = vec![GroupTopics {
626 id: "0".into(),
627 source_topics: vec!["in".into()],
628 changelog_stores: vec![(
629 "w".into(),
630 None,
631 ChangelogKind::AggWindow {
632 retention: Time::from_millis(86_460_000),
633 },
634 )],
635 ..Default::default()
636 }];
637 let topo = to_wire(&groups, "app");
638 let cl = &topo.subtopologies[0].state_changelog_topics;
639 check!(cl.len() == 1);
640 check!(cl[0].name == "app-w-changelog");
641 check!(cl[0].partitions == 0);
642 check!(cl[0].replication_factor == -1);
643 let configs: Vec<(&str, &str)> = cl[0]
644 .topic_configs
645 .iter()
646 .map(|kv| (kv.key.as_str(), kv.value.as_str()))
647 .collect();
648 check!(
649 configs
650 == vec![
651 ("cleanup.policy", "compact,delete"),
652 ("message.timestamp.type", "CreateTime"),
653 ("retention.ms", "86460000"),
654 ]
655 );
656 }
657
658 #[test]
659 fn kv_store_changelog_config_unchanged_after_windowed_change() {
660 let groups = vec![GroupTopics {
662 id: "0".into(),
663 source_topics: vec!["in".into()],
664 changelog_stores: vec![("store".into(), None, ChangelogKind::Kv)],
665 ..Default::default()
666 }];
667 let topo = to_wire(&groups, "my-app");
668 let cl = &topo.subtopologies[0].state_changelog_topics;
669 check!(cl.len() == 1);
670 let configs: Vec<(&str, &str)> = cl[0]
671 .topic_configs
672 .iter()
673 .map(|kv| (kv.key.as_str(), kv.value.as_str()))
674 .collect();
675 check!(
676 configs
677 == vec![
678 ("cleanup.policy", "compact"),
679 ("message.timestamp.type", "CreateTime"),
680 ]
681 );
682 }
683
684 #[test]
685 fn copartition_unknown_member_is_silently_skipped() {
686 let sources = vec!["a".to_string()];
687 let repartition: Vec<String> = vec![];
688 let cg = copartition_group(&sources, &repartition, &["unknown".into()]);
689 check!(cg.source_topics.is_empty());
690 check!(cg.repartition_source_topics.is_empty());
691 }
692
693 #[test]
694 fn join_window_changelog_is_delete_only_with_retention() {
695 use crate::topology::node::ChangelogKind;
696 let groups = vec![GroupTopics {
699 id: "0".into(),
700 source_topics: vec!["in".into()],
701 changelog_stores: vec![(
702 "j".into(),
703 None,
704 ChangelogKind::JoinWindow {
705 retention: Time::from_millis(86_520_000),
706 },
707 )],
708 ..Default::default()
709 }];
710 let topo = to_wire(&groups, "app");
711 let cl = &topo.subtopologies[0].state_changelog_topics[0];
712 check!(cl.name == "app-j-changelog");
713 check!(cl.topic_configs[0].key == "cleanup.policy");
714 check!(cl.topic_configs[0].value == "delete"); check!(cl.topic_configs[1].key == "message.timestamp.type");
716 check!(cl.topic_configs[2].key == "retention.ms");
717 check!(cl.topic_configs[2].value == "86520000");
718 }
719
720 #[test]
721 fn versioned_store_changelog_config_is_compact_with_min_compaction_lag() {
722 let cfgs = versioned_changelog_topic_configs(Time::from_millis(686_400_000));
723 let get = |k: &str| cfgs.iter().find(|c| c.key == k).map(|c| c.value.clone());
724 check!(get("cleanup.policy").as_deref() == Some("compact"));
725 check!(get("message.timestamp.type").as_deref() == Some("CreateTime"));
726 check!(get("min.compaction.lag.ms").as_deref() == Some("686400000"));
727 }
728
729 #[test]
736 fn changelog_kinds_render_kafka_config_strings_verbatim() {
737 let cases = [
738 (
739 ChangelogKind::Kv,
740 vec![
741 ("cleanup.policy", "compact"),
742 ("message.timestamp.type", "CreateTime"),
743 ],
744 ),
745 (
746 ChangelogKind::AggWindow { retention: days(7) },
747 vec![
748 ("cleanup.policy", "compact,delete"),
749 ("message.timestamp.type", "CreateTime"),
750 ("retention.ms", "604800000"),
751 ],
752 ),
753 (
754 ChangelogKind::JoinWindow {
755 retention: minutes(90),
756 },
757 vec![
758 ("cleanup.policy", "delete"),
759 ("message.timestamp.type", "CreateTime"),
760 ("retention.ms", "5400000"),
761 ],
762 ),
763 (
764 ChangelogKind::Versioned {
765 min_compaction_lag: millis(1),
766 },
767 vec![
768 ("cleanup.policy", "compact"),
769 ("message.timestamp.type", "CreateTime"),
770 ("min.compaction.lag.ms", "1"),
771 ],
772 ),
773 ];
774
775 for (kind, expected) in cases {
776 let groups = vec![GroupTopics {
777 id: "0".into(),
778 source_topics: vec!["in".into()],
779 changelog_stores: vec![("s".into(), None, kind)],
780 ..Default::default()
781 }];
782 let topo = to_wire(&groups, "app");
783 let rendered: Vec<(&str, &str)> = topo.subtopologies[0].state_changelog_topics[0]
784 .topic_configs
785 .iter()
786 .map(|kv| (kv.key.as_str(), kv.value.as_str()))
787 .collect();
788 check!(rendered == expected, "{kind:?}");
789 }
790 }
791
792 #[test]
796 fn repartition_topic_configs_render_verbatim() {
797 let configs = repartition_topic_configs();
798 let rendered: Vec<(&str, &str)> = configs
799 .iter()
800 .map(|kv| (kv.key.as_str(), kv.value.as_str()))
801 .collect();
802 check!(
803 rendered
804 == vec![
805 ("cleanup.policy", "delete"),
806 ("message.timestamp.type", "CreateTime"),
807 ("retention.ms", "-1"),
808 ("segment.bytes", "52428800"),
809 ]
810 );
811 }
812
813 #[test]
814 fn repartition_sink_and_source_topics_included_in_wire() {
815 let groups = vec![GroupTopics {
816 id: "0".into(),
817 source_topics: vec!["in".into()],
818 repartition_sink_topics: vec!["rp".into()],
819 repartition_source_topics: vec!["rp".into()],
820 ..Default::default()
821 }];
822 let topo = to_wire(&groups, "app");
823 let st = &topo.subtopologies[0];
824 check!(st.repartition_sink_topics == vec!["rp".to_string()]);
825 check!(st.repartition_source_topics.len() == 1);
826 check!(st.repartition_source_topics[0].name == "rp");
827 }
828}