From 61bf18e47f6dc73d374f94c1f30d51ca06df32ed Mon Sep 17 00:00:00 2001 From: hauke Date: Thu, 5 Nov 2026 15:20:43 +0100 Subject: [PATCH] feat #14740: merge mic and output transcripts Co-Authored-By: Claude Sonnet 5 --- crates/core/src/lib.rs | 1 + crates/core/src/merge.rs | 526 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 527 insertions(+) create mode 100644 crates/core/src/merge.rs diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 7b94fe0..304ff4f 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -7,6 +7,7 @@ pub mod audio; pub mod capture; pub mod detect; +pub mod merge; pub mod model; pub mod recorder; pub mod retention; diff --git a/crates/core/src/merge.rs b/crates/core/src/merge.rs new file mode 100644 index 0000000..9270d88 --- /dev/null +++ b/crates/core/src/merge.rs @@ -0,0 +1,526 @@ +//! Pure merge of the two per-call transcripts — the mic track (always the +//! local user) and the output track (the remote side, diarized into raw +//! speaker ids) — into one time-ordered conversation. Does no I/O; callers +//! feed in words already produced by transcription and consume the +//! resulting [`Segment`]s. + +use std::cmp::Ordering; +use std::collections::BTreeMap; + +use crate::model::{Segment, Speaker, Word}; + +/// Longest gap, in seconds, that still belongs to the same segment. A gap +/// strictly greater than this starts a new segment while grouping a track's +/// words (step 2); a gap at most this long lets two same-speaker segments +/// that ended up adjacent after sorting be merged back together (step 4). +const MAX_GAP_SECONDS: f64 = 1.5; + +/// One word already attributed to a [`Speaker`], in the shape the grouping +/// step needs (the mic track's own `speaker` field is never consulted — +/// mic words are always [`Speaker::Me`]). +struct WordEvent { + speaker: Speaker, + start: f64, + end: f64, + text: String, +} + +/// Merges the mic track and the output track into one time-ordered list of +/// segments. +/// +/// 1. Mic words become [`Speaker::Me`] (their own `speaker` field is +/// ignored). Output words become [`Speaker::Remote`], numbered from 1 by +/// first appearance (in time order) of the raw `speaker` id; `None` +/// counts as one id like any other. +/// 2. Per track, consecutive words from the same speaker are grouped into a +/// segment, splitting on a pause longer than 1.5 s. +/// 3. Both tracks' segments are sorted by `start` (stable; ties put the mic +/// first, then remote speakers in ascending number order). +/// 4. Segments left adjacent in that sorted list are merged when they share +/// a speaker and the gap between them is at most 1.5 s. +/// 5. A segment's text is its words' text, trimmed and joined with single +/// spaces, skipping words whose trimmed text is empty. +/// +/// A track's words need not arrive sorted; they are stably sorted by +/// `start` first. A NaN `start`/`end` is treated as `0.0` for ordering so +/// malformed input never panics. +pub fn merge(mic: &[Word], out: &[Word]) -> Vec { + let mic_events: Vec = sort_words(mic) + .into_iter() + .map(|w| WordEvent { + speaker: Speaker::Me, + start: w.start, + end: w.end, + text: w.text, + }) + .collect(); + + let out_events: Vec = { + let mut seen_ids: Vec> = Vec::new(); + sort_words(out) + .into_iter() + .map(|w| { + let number = match seen_ids.iter().position(|id| id == &w.speaker) { + Some(index) => index, + None => { + seen_ids.push(w.speaker.clone()); + seen_ids.len() - 1 + } + } as u32 + + 1; + WordEvent { + speaker: Speaker::Remote(number), + start: w.start, + end: w.end, + text: w.text, + } + }) + .collect() + }; + + let mut segments = group_segments(&mic_events); + segments.extend(group_segments(&out_events)); + segments.sort_by(cmp_for_merge_order); + + merge_adjacent(segments) +} + +/// The renamed label for `speaker`, or its [`Speaker::default_label`] if it +/// was never renamed, or the rename is present but blank. +pub fn label(speaker: &Speaker, names: &BTreeMap) -> String { + match names.get(&speaker.key()) { + Some(name) if !name.trim().is_empty() => name.clone(), + _ => speaker.default_label(), + } +} + +/// Orders by `start` (NaN treated as `0.0`), then — for ties — puts the mic +/// first and orders remote speakers by ascending number. +fn cmp_for_merge_order(a: &Segment, b: &Segment) -> Ordering { + order_key(a.start) + .total_cmp(&order_key(b.start)) + .then_with(|| speaker_rank(a.speaker).cmp(&speaker_rank(b.speaker))) +} + +fn speaker_rank(speaker: Speaker) -> (u8, u32) { + match speaker { + Speaker::Me => (0, 0), + Speaker::Remote(n) => (1, n), + } +} + +/// `start`/`end` values ordered as-is, except NaN sorts as `0.0`. +fn order_key(value: f64) -> f64 { + if value.is_nan() { + 0.0 + } else { + value + } +} + +fn sort_words(words: &[Word]) -> Vec { + let mut sorted = words.to_vec(); + sorted.sort_by(|a, b| order_key(a.start).total_cmp(&order_key(b.start))); + sorted +} + +/// Accumulates one segment's words while grouping a track. +struct Building { + speaker: Speaker, + start: f64, + /// Max end seen across the segment's words so far (words need not be + /// sorted by end, only by start). + max_end: f64, + /// End of the literal previous word, used for the pause-split check — + /// deliberately not `max_end`, so an earlier long word does not paper + /// over a real pause after a later, shorter one. + prev_word_end: f64, + texts: Vec, +} + +impl Building { + fn start_new(event: &WordEvent) -> Self { + let mut texts = Vec::new(); + push_text(&mut texts, &event.text); + Building { + speaker: event.speaker, + start: event.start, + max_end: event.end, + prev_word_end: event.end, + texts, + } + } + + fn extend(&mut self, event: &WordEvent) { + self.max_end = self.max_end.max(event.end); + self.prev_word_end = event.end; + push_text(&mut self.texts, &event.text); + } + + fn finish(self) -> Segment { + Segment { + speaker: self.speaker, + start: self.start, + end: self.max_end, + text: self.texts.join(" "), + } + } +} + +fn push_text(texts: &mut Vec, raw: &str) { + let trimmed = raw.trim(); + if !trimmed.is_empty() { + texts.push(trimmed.to_string()); + } +} + +/// Groups a single track's words (already sorted by `start`) into segments, +/// splitting whenever the speaker changes or the pause since the previous +/// word exceeds [`MAX_GAP_SECONDS`]. +fn group_segments(events: &[WordEvent]) -> Vec { + let mut result = Vec::new(); + let mut current: Option = None; + + for event in events { + let continues = current.as_ref().is_some_and(|building| { + building.speaker == event.speaker + && event.start - building.prev_word_end <= MAX_GAP_SECONDS + }); + + if continues { + current.as_mut().unwrap().extend(event); + } else { + if let Some(building) = current.take() { + result.push(building.finish()); + } + current = Some(Building::start_new(event)); + } + } + if let Some(building) = current { + result.push(building.finish()); + } + + result +} + +/// Merges segments left adjacent after sorting when they share a speaker +/// and the gap between them is at most [`MAX_GAP_SECONDS`]. Unlike the +/// per-track grouping in [`group_segments`], the gap here is measured +/// against the previous segment's recorded `end` (its words' max end), so a +/// pause that only looked long at the word level can still be closed back +/// up once segments are considered as a whole. +fn merge_adjacent(segments: Vec) -> Vec { + let mut merged: Vec = Vec::with_capacity(segments.len()); + + for segment in segments { + let should_merge = merged.last().is_some_and(|last: &Segment| { + last.speaker == segment.speaker && segment.start - last.end <= MAX_GAP_SECONDS + }); + + if should_merge { + let last = merged.last_mut().expect("just checked non-empty"); + last.end = last.end.max(segment.end); + last.text = join_text(&last.text, &segment.text); + } else { + merged.push(segment); + } + } + + merged +} + +fn join_text(a: &str, b: &str) -> String { + match (a.is_empty(), b.is_empty()) { + (true, true) => String::new(), + (true, false) => b.to_string(), + (false, true) => a.to_string(), + (false, false) => format!("{a} {b}"), + } +} + +#[cfg(test)] +mod tests { + use super::{label, merge}; + use crate::model::{Segment, Speaker, Word}; + use std::collections::BTreeMap; + + fn w(text: &str, start: f64, end: f64, speaker: Option<&str>) -> Word { + Word { + text: text.to_string(), + start, + end, + speaker: speaker.map(str::to_string), + } + } + + #[test] + fn interleaves_by_time() { + let mic = vec![ + w("hallo", 0.0, 1.0, None), + w("wie gehts", 1.0, 2.0, None), + w("tschuess", 4.5, 6.0, None), + ]; + let out = vec![w("mir gehts gut", 2.5, 4.0, Some("SPEAKER_00"))]; + + assert_eq!( + merge(&mic, &out), + vec![ + Segment { + speaker: Speaker::Me, + start: 0.0, + end: 2.0, + text: "hallo wie gehts".to_string(), + }, + Segment { + speaker: Speaker::Remote(1), + start: 2.5, + end: 4.0, + text: "mir gehts gut".to_string(), + }, + Segment { + speaker: Speaker::Me, + start: 4.5, + end: 6.0, + text: "tschuess".to_string(), + }, + ] + ); + } + + #[test] + fn numbers_remote_speakers_by_first_appearance() { + let out = vec![ + w("drei", 0.0, 1.0, Some("SPEAKER_03")), + w("null", 5.0, 6.0, Some("SPEAKER_00")), + ]; + + assert_eq!( + merge(&[], &out), + vec![ + Segment { + speaker: Speaker::Remote(1), + start: 0.0, + end: 1.0, + text: "drei".to_string(), + }, + Segment { + speaker: Speaker::Remote(2), + start: 5.0, + end: 6.0, + text: "null".to_string(), + }, + ] + ); + } + + #[test] + fn splits_same_speaker_on_long_pause() { + let mic = vec![ + w("eins", 0.0, 1.0, None), + // Gap since the previous word is 2.0 s, over the 1.5 s limit. + w("zwei", 3.0, 4.0, None), + ]; + + assert_eq!( + merge(&mic, &[]), + vec![ + Segment { + speaker: Speaker::Me, + start: 0.0, + end: 1.0, + text: "eins".to_string(), + }, + Segment { + speaker: Speaker::Me, + start: 3.0, + end: 4.0, + text: "zwei".to_string(), + }, + ] + ); + } + + #[test] + fn null_speaker_ids_are_one_remote() { + let out = vec![w("a", 0.0, 1.0, None), w("b", 1.0, 2.0, None)]; + + assert_eq!( + merge(&[], &out), + vec![Segment { + speaker: Speaker::Remote(1), + start: 0.0, + end: 2.0, + text: "a b".to_string(), + }] + ); + } + + #[test] + fn empty_tracks() { + assert_eq!(merge(&[], &[]), Vec::::new()); + + let mic = vec![w("hallo", 0.0, 1.0, None)]; + assert_eq!( + merge(&mic, &[]), + vec![Segment { + speaker: Speaker::Me, + start: 0.0, + end: 1.0, + text: "hallo".to_string(), + }] + ); + + let out = vec![w("hi", 0.0, 1.0, Some("SPEAKER_00"))]; + assert_eq!( + merge(&[], &out), + vec![Segment { + speaker: Speaker::Remote(1), + start: 0.0, + end: 1.0, + text: "hi".to_string(), + }] + ); + } + + #[test] + fn label_uses_renames() { + let mut names = BTreeMap::new(); + names.insert("remote:1".to_string(), "Frau Müller".to_string()); + + assert_eq!(label(&Speaker::Remote(1), &names), "Frau Müller"); + assert_eq!(label(&Speaker::Me, &names), "Ich"); + } + + #[test] + fn label_blank_rename_falls_back_to_default() { + let mut names = BTreeMap::new(); + names.insert("remote:2".to_string(), " ".to_string()); + + assert_eq!(label(&Speaker::Remote(2), &names), "Sprecher 2"); + } + + #[test] + fn ties_at_same_start_put_mic_first() { + let mic = vec![w("hallo", 0.0, 1.0, None)]; + let out = vec![w("hi", 0.0, 1.0, Some("SPEAKER_00"))]; + + let segments = merge(&mic, &out); + + assert_eq!(segments[0].speaker, Speaker::Me); + assert_eq!(segments[1].speaker, Speaker::Remote(1)); + } + + #[test] + fn unsorted_input_words_are_handled() { + let mic = vec![w("zwei", 1.0, 2.0, None), w("eins", 0.0, 1.0, None)]; + + assert_eq!( + merge(&mic, &[]), + vec![Segment { + speaker: Speaker::Me, + start: 0.0, + end: 2.0, + text: "eins zwei".to_string(), + }] + ); + } + + #[test] + fn nan_timestamps_do_not_panic_and_sort_as_zero() { + // A NaN start/end must not make `merge` panic, and for ordering + // purposes it is treated as 0.0 — so this word, despite textually + // "arriving" after the 5.0 s one, sorts before it. + let mic = vec![ + w("spaet", 5.0, 5.5, None), + w("nan", f64::NAN, f64::NAN, None), + ]; + + let segments = merge(&mic, &[]); + + assert_eq!(segments.len(), 2); + assert_eq!(segments[0].text, "nan"); + assert_eq!(segments[1].text, "spaet"); + } + + #[test] + fn empty_text_words_are_skipped_but_still_count_for_timing() { + let mic = vec![ + w("hallo", 0.0, 1.0, None), + w(" ", 1.0, 1.4, None), + w("welt", 1.4, 2.0, None), + ]; + + assert_eq!( + merge(&mic, &[]), + vec![Segment { + speaker: Speaker::Me, + start: 0.0, + end: 2.0, + text: "hallo welt".to_string(), + }] + ); + } + + #[test] + fn interleaved_other_speaker_prevents_merge() { + // Remote(1) at 0-1, Remote(2) at 1.2-2, Remote(1) again at 2.2-3: + // the second Remote(1) run is not adjacent (in the sorted list) to + // the first, so it must stay a separate segment even though the + // gap alone would be small enough to merge. + let out = vec![ + w("a", 0.0, 1.0, Some("SPEAKER_00")), + w("b", 1.2, 2.0, Some("SPEAKER_01")), + w("c", 2.2, 3.0, Some("SPEAKER_00")), + ]; + + assert_eq!( + merge(&[], &out), + vec![ + Segment { + speaker: Speaker::Remote(1), + start: 0.0, + end: 1.0, + text: "a".to_string(), + }, + Segment { + speaker: Speaker::Remote(2), + start: 1.2, + end: 2.0, + text: "b".to_string(), + }, + Segment { + speaker: Speaker::Remote(1), + start: 2.2, + end: 3.0, + text: "c".to_string(), + }, + ] + ); + } + + #[test] + fn step_four_merges_using_segment_end_not_last_word_end() { + // w1 overlaps far past w2: the segment's recorded end (max of its + // words) is 5.0, even though the literal previous word (w2) ends + // at 2.0. The pause check in step 2 uses the literal previous + // word's end, so w3 (start 6.0) splits off into its own segment + // (6.0 - 2.0 = 4.0 > 1.5). But step 4 re-checks adjacent segments + // using the *segment's* end (5.0), so 6.0 - 5.0 = 1.0 <= 1.5 merges + // them back together. + let mic = vec![ + w("w1", 0.0, 5.0, None), + w("w2", 1.0, 2.0, None), + w("w3", 6.0, 7.0, None), + ]; + + assert_eq!( + merge(&mic, &[]), + vec![Segment { + speaker: Speaker::Me, + start: 0.0, + end: 7.0, + text: "w1 w2 w3".to_string(), + }] + ); + } +}