feat #14740: merge mic and output transcripts
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,526 @@
|
||||
//! Pure merge of the two per-call transcripts — the mic track (always the
|
||||
//! local user) and the output track (the remote side, diarized into raw
|
||||
//! speaker ids) — into one time-ordered conversation. Does no I/O; callers
|
||||
//! feed in words already produced by transcription and consume the
|
||||
//! resulting [`Segment`]s.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::model::{Segment, Speaker, Word};
|
||||
|
||||
/// Longest gap, in seconds, that still belongs to the same segment. A gap
|
||||
/// strictly greater than this starts a new segment while grouping a track's
|
||||
/// words (step 2); a gap at most this long lets two same-speaker segments
|
||||
/// that ended up adjacent after sorting be merged back together (step 4).
|
||||
const MAX_GAP_SECONDS: f64 = 1.5;
|
||||
|
||||
/// One word already attributed to a [`Speaker`], in the shape the grouping
|
||||
/// step needs (the mic track's own `speaker` field is never consulted —
|
||||
/// mic words are always [`Speaker::Me`]).
|
||||
struct WordEvent {
|
||||
speaker: Speaker,
|
||||
start: f64,
|
||||
end: f64,
|
||||
text: String,
|
||||
}
|
||||
|
||||
/// Merges the mic track and the output track into one time-ordered list of
|
||||
/// segments.
|
||||
///
|
||||
/// 1. Mic words become [`Speaker::Me`] (their own `speaker` field is
|
||||
/// ignored). Output words become [`Speaker::Remote`], numbered from 1 by
|
||||
/// first appearance (in time order) of the raw `speaker` id; `None`
|
||||
/// counts as one id like any other.
|
||||
/// 2. Per track, consecutive words from the same speaker are grouped into a
|
||||
/// segment, splitting on a pause longer than 1.5 s.
|
||||
/// 3. Both tracks' segments are sorted by `start` (stable; ties put the mic
|
||||
/// first, then remote speakers in ascending number order).
|
||||
/// 4. Segments left adjacent in that sorted list are merged when they share
|
||||
/// a speaker and the gap between them is at most 1.5 s.
|
||||
/// 5. A segment's text is its words' text, trimmed and joined with single
|
||||
/// spaces, skipping words whose trimmed text is empty.
|
||||
///
|
||||
/// A track's words need not arrive sorted; they are stably sorted by
|
||||
/// `start` first. A NaN `start`/`end` is treated as `0.0` for ordering so
|
||||
/// malformed input never panics.
|
||||
pub fn merge(mic: &[Word], out: &[Word]) -> Vec<Segment> {
|
||||
let mic_events: Vec<WordEvent> = sort_words(mic)
|
||||
.into_iter()
|
||||
.map(|w| WordEvent {
|
||||
speaker: Speaker::Me,
|
||||
start: w.start,
|
||||
end: w.end,
|
||||
text: w.text,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let out_events: Vec<WordEvent> = {
|
||||
let mut seen_ids: Vec<Option<String>> = Vec::new();
|
||||
sort_words(out)
|
||||
.into_iter()
|
||||
.map(|w| {
|
||||
let number = match seen_ids.iter().position(|id| id == &w.speaker) {
|
||||
Some(index) => index,
|
||||
None => {
|
||||
seen_ids.push(w.speaker.clone());
|
||||
seen_ids.len() - 1
|
||||
}
|
||||
} as u32
|
||||
+ 1;
|
||||
WordEvent {
|
||||
speaker: Speaker::Remote(number),
|
||||
start: w.start,
|
||||
end: w.end,
|
||||
text: w.text,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
let mut segments = group_segments(&mic_events);
|
||||
segments.extend(group_segments(&out_events));
|
||||
segments.sort_by(cmp_for_merge_order);
|
||||
|
||||
merge_adjacent(segments)
|
||||
}
|
||||
|
||||
/// The renamed label for `speaker`, or its [`Speaker::default_label`] if it
|
||||
/// was never renamed, or the rename is present but blank.
|
||||
pub fn label(speaker: &Speaker, names: &BTreeMap<String, String>) -> String {
|
||||
match names.get(&speaker.key()) {
|
||||
Some(name) if !name.trim().is_empty() => name.clone(),
|
||||
_ => speaker.default_label(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Orders by `start` (NaN treated as `0.0`), then — for ties — puts the mic
|
||||
/// first and orders remote speakers by ascending number.
|
||||
fn cmp_for_merge_order(a: &Segment, b: &Segment) -> Ordering {
|
||||
order_key(a.start)
|
||||
.total_cmp(&order_key(b.start))
|
||||
.then_with(|| speaker_rank(a.speaker).cmp(&speaker_rank(b.speaker)))
|
||||
}
|
||||
|
||||
fn speaker_rank(speaker: Speaker) -> (u8, u32) {
|
||||
match speaker {
|
||||
Speaker::Me => (0, 0),
|
||||
Speaker::Remote(n) => (1, n),
|
||||
}
|
||||
}
|
||||
|
||||
/// `start`/`end` values ordered as-is, except NaN sorts as `0.0`.
|
||||
fn order_key(value: f64) -> f64 {
|
||||
if value.is_nan() {
|
||||
0.0
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
fn sort_words(words: &[Word]) -> Vec<Word> {
|
||||
let mut sorted = words.to_vec();
|
||||
sorted.sort_by(|a, b| order_key(a.start).total_cmp(&order_key(b.start)));
|
||||
sorted
|
||||
}
|
||||
|
||||
/// Accumulates one segment's words while grouping a track.
|
||||
struct Building {
|
||||
speaker: Speaker,
|
||||
start: f64,
|
||||
/// Max end seen across the segment's words so far (words need not be
|
||||
/// sorted by end, only by start).
|
||||
max_end: f64,
|
||||
/// End of the literal previous word, used for the pause-split check —
|
||||
/// deliberately not `max_end`, so an earlier long word does not paper
|
||||
/// over a real pause after a later, shorter one.
|
||||
prev_word_end: f64,
|
||||
texts: Vec<String>,
|
||||
}
|
||||
|
||||
impl Building {
|
||||
fn start_new(event: &WordEvent) -> Self {
|
||||
let mut texts = Vec::new();
|
||||
push_text(&mut texts, &event.text);
|
||||
Building {
|
||||
speaker: event.speaker,
|
||||
start: event.start,
|
||||
max_end: event.end,
|
||||
prev_word_end: event.end,
|
||||
texts,
|
||||
}
|
||||
}
|
||||
|
||||
fn extend(&mut self, event: &WordEvent) {
|
||||
self.max_end = self.max_end.max(event.end);
|
||||
self.prev_word_end = event.end;
|
||||
push_text(&mut self.texts, &event.text);
|
||||
}
|
||||
|
||||
fn finish(self) -> Segment {
|
||||
Segment {
|
||||
speaker: self.speaker,
|
||||
start: self.start,
|
||||
end: self.max_end,
|
||||
text: self.texts.join(" "),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn push_text(texts: &mut Vec<String>, raw: &str) {
|
||||
let trimmed = raw.trim();
|
||||
if !trimmed.is_empty() {
|
||||
texts.push(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
/// Groups a single track's words (already sorted by `start`) into segments,
|
||||
/// splitting whenever the speaker changes or the pause since the previous
|
||||
/// word exceeds [`MAX_GAP_SECONDS`].
|
||||
fn group_segments(events: &[WordEvent]) -> Vec<Segment> {
|
||||
let mut result = Vec::new();
|
||||
let mut current: Option<Building> = None;
|
||||
|
||||
for event in events {
|
||||
let continues = current.as_ref().is_some_and(|building| {
|
||||
building.speaker == event.speaker
|
||||
&& event.start - building.prev_word_end <= MAX_GAP_SECONDS
|
||||
});
|
||||
|
||||
if continues {
|
||||
current.as_mut().unwrap().extend(event);
|
||||
} else {
|
||||
if let Some(building) = current.take() {
|
||||
result.push(building.finish());
|
||||
}
|
||||
current = Some(Building::start_new(event));
|
||||
}
|
||||
}
|
||||
if let Some(building) = current {
|
||||
result.push(building.finish());
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Merges segments left adjacent after sorting when they share a speaker
|
||||
/// and the gap between them is at most [`MAX_GAP_SECONDS`]. Unlike the
|
||||
/// per-track grouping in [`group_segments`], the gap here is measured
|
||||
/// against the previous segment's recorded `end` (its words' max end), so a
|
||||
/// pause that only looked long at the word level can still be closed back
|
||||
/// up once segments are considered as a whole.
|
||||
fn merge_adjacent(segments: Vec<Segment>) -> Vec<Segment> {
|
||||
let mut merged: Vec<Segment> = Vec::with_capacity(segments.len());
|
||||
|
||||
for segment in segments {
|
||||
let should_merge = merged.last().is_some_and(|last: &Segment| {
|
||||
last.speaker == segment.speaker && segment.start - last.end <= MAX_GAP_SECONDS
|
||||
});
|
||||
|
||||
if should_merge {
|
||||
let last = merged.last_mut().expect("just checked non-empty");
|
||||
last.end = last.end.max(segment.end);
|
||||
last.text = join_text(&last.text, &segment.text);
|
||||
} else {
|
||||
merged.push(segment);
|
||||
}
|
||||
}
|
||||
|
||||
merged
|
||||
}
|
||||
|
||||
fn join_text(a: &str, b: &str) -> String {
|
||||
match (a.is_empty(), b.is_empty()) {
|
||||
(true, true) => String::new(),
|
||||
(true, false) => b.to_string(),
|
||||
(false, true) => a.to_string(),
|
||||
(false, false) => format!("{a} {b}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{label, merge};
|
||||
use crate::model::{Segment, Speaker, Word};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn w(text: &str, start: f64, end: f64, speaker: Option<&str>) -> Word {
|
||||
Word {
|
||||
text: text.to_string(),
|
||||
start,
|
||||
end,
|
||||
speaker: speaker.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interleaves_by_time() {
|
||||
let mic = vec![
|
||||
w("hallo", 0.0, 1.0, None),
|
||||
w("wie gehts", 1.0, 2.0, None),
|
||||
w("tschuess", 4.5, 6.0, None),
|
||||
];
|
||||
let out = vec![w("mir gehts gut", 2.5, 4.0, Some("SPEAKER_00"))];
|
||||
|
||||
assert_eq!(
|
||||
merge(&mic, &out),
|
||||
vec![
|
||||
Segment {
|
||||
speaker: Speaker::Me,
|
||||
start: 0.0,
|
||||
end: 2.0,
|
||||
text: "hallo wie gehts".to_string(),
|
||||
},
|
||||
Segment {
|
||||
speaker: Speaker::Remote(1),
|
||||
start: 2.5,
|
||||
end: 4.0,
|
||||
text: "mir gehts gut".to_string(),
|
||||
},
|
||||
Segment {
|
||||
speaker: Speaker::Me,
|
||||
start: 4.5,
|
||||
end: 6.0,
|
||||
text: "tschuess".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbers_remote_speakers_by_first_appearance() {
|
||||
let out = vec![
|
||||
w("drei", 0.0, 1.0, Some("SPEAKER_03")),
|
||||
w("null", 5.0, 6.0, Some("SPEAKER_00")),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
merge(&[], &out),
|
||||
vec![
|
||||
Segment {
|
||||
speaker: Speaker::Remote(1),
|
||||
start: 0.0,
|
||||
end: 1.0,
|
||||
text: "drei".to_string(),
|
||||
},
|
||||
Segment {
|
||||
speaker: Speaker::Remote(2),
|
||||
start: 5.0,
|
||||
end: 6.0,
|
||||
text: "null".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splits_same_speaker_on_long_pause() {
|
||||
let mic = vec![
|
||||
w("eins", 0.0, 1.0, None),
|
||||
// Gap since the previous word is 2.0 s, over the 1.5 s limit.
|
||||
w("zwei", 3.0, 4.0, None),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
merge(&mic, &[]),
|
||||
vec![
|
||||
Segment {
|
||||
speaker: Speaker::Me,
|
||||
start: 0.0,
|
||||
end: 1.0,
|
||||
text: "eins".to_string(),
|
||||
},
|
||||
Segment {
|
||||
speaker: Speaker::Me,
|
||||
start: 3.0,
|
||||
end: 4.0,
|
||||
text: "zwei".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_speaker_ids_are_one_remote() {
|
||||
let out = vec![w("a", 0.0, 1.0, None), w("b", 1.0, 2.0, None)];
|
||||
|
||||
assert_eq!(
|
||||
merge(&[], &out),
|
||||
vec![Segment {
|
||||
speaker: Speaker::Remote(1),
|
||||
start: 0.0,
|
||||
end: 2.0,
|
||||
text: "a b".to_string(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_tracks() {
|
||||
assert_eq!(merge(&[], &[]), Vec::<Segment>::new());
|
||||
|
||||
let mic = vec![w("hallo", 0.0, 1.0, None)];
|
||||
assert_eq!(
|
||||
merge(&mic, &[]),
|
||||
vec![Segment {
|
||||
speaker: Speaker::Me,
|
||||
start: 0.0,
|
||||
end: 1.0,
|
||||
text: "hallo".to_string(),
|
||||
}]
|
||||
);
|
||||
|
||||
let out = vec![w("hi", 0.0, 1.0, Some("SPEAKER_00"))];
|
||||
assert_eq!(
|
||||
merge(&[], &out),
|
||||
vec![Segment {
|
||||
speaker: Speaker::Remote(1),
|
||||
start: 0.0,
|
||||
end: 1.0,
|
||||
text: "hi".to_string(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_uses_renames() {
|
||||
let mut names = BTreeMap::new();
|
||||
names.insert("remote:1".to_string(), "Frau Müller".to_string());
|
||||
|
||||
assert_eq!(label(&Speaker::Remote(1), &names), "Frau Müller");
|
||||
assert_eq!(label(&Speaker::Me, &names), "Ich");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_blank_rename_falls_back_to_default() {
|
||||
let mut names = BTreeMap::new();
|
||||
names.insert("remote:2".to_string(), " ".to_string());
|
||||
|
||||
assert_eq!(label(&Speaker::Remote(2), &names), "Sprecher 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ties_at_same_start_put_mic_first() {
|
||||
let mic = vec![w("hallo", 0.0, 1.0, None)];
|
||||
let out = vec![w("hi", 0.0, 1.0, Some("SPEAKER_00"))];
|
||||
|
||||
let segments = merge(&mic, &out);
|
||||
|
||||
assert_eq!(segments[0].speaker, Speaker::Me);
|
||||
assert_eq!(segments[1].speaker, Speaker::Remote(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsorted_input_words_are_handled() {
|
||||
let mic = vec![w("zwei", 1.0, 2.0, None), w("eins", 0.0, 1.0, None)];
|
||||
|
||||
assert_eq!(
|
||||
merge(&mic, &[]),
|
||||
vec![Segment {
|
||||
speaker: Speaker::Me,
|
||||
start: 0.0,
|
||||
end: 2.0,
|
||||
text: "eins zwei".to_string(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nan_timestamps_do_not_panic_and_sort_as_zero() {
|
||||
// A NaN start/end must not make `merge` panic, and for ordering
|
||||
// purposes it is treated as 0.0 — so this word, despite textually
|
||||
// "arriving" after the 5.0 s one, sorts before it.
|
||||
let mic = vec![
|
||||
w("spaet", 5.0, 5.5, None),
|
||||
w("nan", f64::NAN, f64::NAN, None),
|
||||
];
|
||||
|
||||
let segments = merge(&mic, &[]);
|
||||
|
||||
assert_eq!(segments.len(), 2);
|
||||
assert_eq!(segments[0].text, "nan");
|
||||
assert_eq!(segments[1].text, "spaet");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_text_words_are_skipped_but_still_count_for_timing() {
|
||||
let mic = vec![
|
||||
w("hallo", 0.0, 1.0, None),
|
||||
w(" ", 1.0, 1.4, None),
|
||||
w("welt", 1.4, 2.0, None),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
merge(&mic, &[]),
|
||||
vec![Segment {
|
||||
speaker: Speaker::Me,
|
||||
start: 0.0,
|
||||
end: 2.0,
|
||||
text: "hallo welt".to_string(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interleaved_other_speaker_prevents_merge() {
|
||||
// Remote(1) at 0-1, Remote(2) at 1.2-2, Remote(1) again at 2.2-3:
|
||||
// the second Remote(1) run is not adjacent (in the sorted list) to
|
||||
// the first, so it must stay a separate segment even though the
|
||||
// gap alone would be small enough to merge.
|
||||
let out = vec![
|
||||
w("a", 0.0, 1.0, Some("SPEAKER_00")),
|
||||
w("b", 1.2, 2.0, Some("SPEAKER_01")),
|
||||
w("c", 2.2, 3.0, Some("SPEAKER_00")),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
merge(&[], &out),
|
||||
vec![
|
||||
Segment {
|
||||
speaker: Speaker::Remote(1),
|
||||
start: 0.0,
|
||||
end: 1.0,
|
||||
text: "a".to_string(),
|
||||
},
|
||||
Segment {
|
||||
speaker: Speaker::Remote(2),
|
||||
start: 1.2,
|
||||
end: 2.0,
|
||||
text: "b".to_string(),
|
||||
},
|
||||
Segment {
|
||||
speaker: Speaker::Remote(1),
|
||||
start: 2.2,
|
||||
end: 3.0,
|
||||
text: "c".to_string(),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_four_merges_using_segment_end_not_last_word_end() {
|
||||
// w1 overlaps far past w2: the segment's recorded end (max of its
|
||||
// words) is 5.0, even though the literal previous word (w2) ends
|
||||
// at 2.0. The pause check in step 2 uses the literal previous
|
||||
// word's end, so w3 (start 6.0) splits off into its own segment
|
||||
// (6.0 - 2.0 = 4.0 > 1.5). But step 4 re-checks adjacent segments
|
||||
// using the *segment's* end (5.0), so 6.0 - 5.0 = 1.0 <= 1.5 merges
|
||||
// them back together.
|
||||
let mic = vec![
|
||||
w("w1", 0.0, 5.0, None),
|
||||
w("w2", 1.0, 2.0, None),
|
||||
w("w3", 6.0, 7.0, None),
|
||||
];
|
||||
|
||||
assert_eq!(
|
||||
merge(&mic, &[]),
|
||||
vec![Segment {
|
||||
speaker: Speaker::Me,
|
||||
start: 0.0,
|
||||
end: 7.0,
|
||||
text: "w1 w2 w3".to_string(),
|
||||
}]
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user