feat #14740: play back calls

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
hauke
2026-11-10 13:37:47 +01:00
co-authored by Claude Opus 5.5
parent 8d757fd639
commit a74cfa5864
16 changed files with 1335 additions and 12 deletions
Generated
+7
View File
@@ -1803,6 +1803,12 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
] ]
[[package]]
name = "http-range"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2149c4b4b83390c71fab669730347d578ec6bdf759f2160c581ad8b8819733b4"
[[package]] [[package]]
name = "httparse" name = "httparse"
version = "1.10.1" version = "1.10.1"
@@ -4576,6 +4582,7 @@ dependencies = [
"gtk", "gtk",
"heck 0.5.0", "heck 0.5.0",
"http", "http",
"http-range",
"jni 0.21.1", "jni 0.21.1",
"libc", "libc",
"log", "log",
+313
View File
@@ -0,0 +1,313 @@
//! Mixes the two 16 kHz mono tracks of a call into one mono track for
//! playback.
//!
//! The tracks are summed at full level and passed through a soft clipper,
//! so a passage where only one side speaks keeps its original loudness
//! while both sides speaking loudly at once never wraps around or clips
//! hard. [`mix_wavs`] streams both files in chunks, so even a long call
//! never sits in memory.
use std::fs::{self, File};
use std::io::{self, BufReader, ErrorKind};
use std::path::Path;
use hound::{SampleFormat, WavReader, WavSpec, WavWriter};
use super::wav::SAMPLE_RATE;
/// Samples read per track and step in [`mix_wavs`] (about one second).
const CHUNK_SAMPLES: usize = 16_384;
/// Up to this level (half of full scale) the sum passes unchanged; above,
/// the soft clipper compresses it towards full scale.
const KNEE: f32 = 0.5;
/// The format of both tracks and of the mixed file.
fn spec() -> WavSpec {
WavSpec {
channels: 1,
sample_rate: SAMPLE_RATE,
bits_per_sample: 16,
sample_format: SampleFormat::Int,
}
}
/// Mixes `a` and `b` sample by sample and appends the result to `out`.
/// The output is as long as the longer input; the shorter one counts as
/// silence after its end. Never clips hard: see [`soft_clip`].
pub fn mix_samples(a: &[i16], b: &[i16], out: &mut Vec<i16>) {
let len = a.len().max(b.len());
out.reserve(len);
for i in 0..len {
let sum = to_f32(a.get(i).copied().unwrap_or(0)) + to_f32(b.get(i).copied().unwrap_or(0));
out.push(to_i16(soft_clip(sum)));
}
}
fn to_f32(sample: i16) -> f32 {
sample as f32 / i16::MAX as f32
}
fn to_i16(sample: f32) -> i16 {
(sample.clamp(-1.0, 1.0) * i16::MAX as f32).round() as i16
}
/// Linear up to [`KNEE`]; above, `tanh` bends the level smoothly (same
/// slope at the knee) towards, but never beyond, full scale.
fn soft_clip(x: f32) -> f32 {
let level = x.abs();
if level <= KNEE {
return x;
}
let room = 1.0 - KNEE;
x.signum() * (KNEE + room * ((level - KNEE) / room).tanh())
}
/// One source track; `None` if the file is missing or holds no audio.
type Track = Option<hound::WavIntoSamples<BufReader<File>, i16>>;
/// Opens a track for [`mix_wavs`]. A missing or header-only file counts as
/// no track; anything else that is not 16 kHz mono 16-bit PCM is an error.
fn open_track(path: &Path) -> io::Result<Track> {
if path.as_os_str().is_empty() {
return Ok(None);
}
let reader = match WavReader::open(path) {
Ok(reader) => reader,
Err(hound::Error::IoError(e)) if e.kind() == ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(invalid(path, e)),
};
if reader.spec() != spec() {
return Err(invalid(path, "not 16 kHz mono 16-bit PCM"));
}
if reader.len() == 0 {
return Ok(None);
}
Ok(Some(reader.into_samples()))
}
fn invalid(path: &Path, e: impl std::fmt::Display) -> io::Error {
io::Error::new(ErrorKind::InvalidData, format!("{}: {e}", path.display()))
}
/// Reads up to [`CHUNK_SAMPLES`] samples of `track` into `buf` (cleared
/// first).
fn read_chunk(track: &mut Track, buf: &mut Vec<i16>) -> io::Result<()> {
buf.clear();
let Some(samples) = track else {
return Ok(());
};
for sample in samples.take(CHUNK_SAMPLES) {
buf.push(sample.map_err(|e| io::Error::new(ErrorKind::InvalidData, e))?);
}
Ok(())
}
/// Mixes the WAV tracks at `a` and `b` (16 kHz mono 16-bit PCM, as written
/// by [`super::TrackWriter`]) into a new WAV at `dest` and returns its
/// length in samples. A missing or empty track is left out; with neither
/// track holding audio nothing is written and the error is
/// [`ErrorKind::NotFound`].
///
/// Writes `dest` with a `.part` suffix first and renames it at the end, so
/// `dest` is never a half-written file.
pub fn mix_wavs(a: &Path, b: &Path, dest: &Path) -> io::Result<u64> {
let mut track_a = open_track(a)?;
let mut track_b = open_track(b)?;
if track_a.is_none() && track_b.is_none() {
return Err(io::Error::new(
ErrorKind::NotFound,
"no audio in either track",
));
}
let mut part = dest.as_os_str().to_owned();
part.push(".part");
let part = Path::new(&part);
let result = write_mix(&mut track_a, &mut track_b, part);
match result {
Ok(samples) => {
fs::rename(part, dest)?;
Ok(samples)
}
Err(e) => {
let _ = fs::remove_file(part);
Err(e)
}
}
}
fn write_mix(a: &mut Track, b: &mut Track, path: &Path) -> io::Result<u64> {
let mut writer = WavWriter::create(path, spec()).map_err(hound_to_io)?;
let (mut buf_a, mut buf_b, mut mixed) = (Vec::new(), Vec::new(), Vec::new());
let mut written = 0u64;
loop {
read_chunk(a, &mut buf_a)?;
read_chunk(b, &mut buf_b)?;
if buf_a.is_empty() && buf_b.is_empty() {
break;
}
mixed.clear();
mix_samples(&buf_a, &buf_b, &mut mixed);
for &sample in &mixed {
writer.write_sample(sample).map_err(hound_to_io)?;
}
written += mixed.len() as u64;
}
writer.finalize().map_err(hound_to_io)?;
Ok(written)
}
fn hound_to_io(e: hound::Error) -> io::Error {
match e {
hound::Error::IoError(e) => e,
other => io::Error::other(other),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::audio::TrackWriter;
fn mix(a: &[i16], b: &[i16]) -> Vec<i16> {
let mut out = Vec::new();
mix_samples(a, b, &mut out);
out
}
#[test]
fn quiet_samples_of_equal_length_are_summed() {
assert_eq!(mix(&[100, -200, 0], &[50, 300, -7]), vec![150, 100, -7]);
}
#[test]
fn the_longer_track_continues_alone() {
assert_eq!(mix(&[1, 2], &[10, 20, 30, 40]), vec![11, 22, 30, 40]);
assert_eq!(mix(&[10, 20, 30], &[1]), vec![11, 20, 30]);
}
#[test]
fn an_empty_track_leaves_the_other_unchanged_up_to_the_knee() {
let half = (KNEE * i16::MAX as f32) as i16;
let track = [0, 1234, -1234, half, -half, i16::MIN + 1];
let mut expected = track.to_vec();
// Beyond the knee even a single track is compressed (but stays
// loud and keeps its sign).
let last = mix(&[], &track)[5];
expected[5] = last;
assert!(last < -28_000, "{last}");
assert_eq!(mix(&[], &track), expected);
assert_eq!(mix(&track, &[]), expected);
assert_eq!(mix(&[], &[]), Vec::<i16>::new());
}
#[test]
fn full_scale_on_both_sides_stays_within_full_scale() {
let out = mix(
&[i16::MAX, i16::MIN, i16::MAX],
&[i16::MAX, i16::MIN, i16::MIN],
);
// Loud, with the right sign, and softly short of full scale.
assert!(out[0] > 30_000 && out[0] < i16::MAX, "{}", out[0]);
assert!(out[1] < -30_000 && out[1] > -i16::MAX, "{}", out[1]);
// Opposite full-scale samples cancel instead of wrapping around.
assert!(out[2].abs() <= 1, "{}", out[2]);
}
#[test]
fn the_soft_clip_is_monotonic_and_bounded() {
let mut last = -1.0f32;
for step in -400..=400 {
let x = step as f32 / 200.0; // -2.0 ..= 2.0
let y = soft_clip(x);
assert!(y >= last, "not monotonic at {x}");
assert!(y.abs() < 1.0, "{x} -> {y}");
last = y;
}
}
fn write_track(path: &Path, samples: &[f32]) {
let mut writer = TrackWriter::create(path).unwrap();
writer.write(samples).unwrap();
writer.finalize().unwrap();
}
fn read(path: &Path) -> Vec<i16> {
WavReader::open(path)
.unwrap()
.into_samples::<i16>()
.map(Result::unwrap)
.collect()
}
#[test]
fn two_files_are_mixed_across_several_chunks() {
let dir = tempfile::tempdir().unwrap();
let (a, b, dest) = (
dir.path().join("a.wav"),
dir.path().join("b.wav"),
dir.path().join("m.wav"),
);
let long = CHUNK_SAMPLES * 2 + 100;
write_track(&a, &vec![0.1; long]);
write_track(&b, &vec![0.2; CHUNK_SAMPLES + 7]);
assert_eq!(mix_wavs(&a, &b, &dest).unwrap(), long as u64);
let mixed = read(&dest);
assert_eq!(mixed.len(), long);
let both = to_i16(0.1) + to_i16(0.2);
assert!((mixed[0] - both).abs() <= 1);
assert!((mixed[CHUNK_SAMPLES + 6] - both).abs() <= 1);
assert_eq!(mixed[CHUNK_SAMPLES + 7], to_i16(0.1));
assert_eq!(mixed[long - 1], to_i16(0.1));
assert!(!dir.path().join("m.wav.part").exists());
}
#[test]
fn a_missing_or_empty_file_is_left_out() {
let dir = tempfile::tempdir().unwrap();
let (a, empty, dest) = (
dir.path().join("a.wav"),
dir.path().join("e.wav"),
dir.path().join("m.wav"),
);
write_track(&a, &[0.25; 10]);
write_track(&empty, &[]);
assert_eq!(
mix_wavs(&dir.path().join("gone.wav"), &a, &dest).unwrap(),
10
);
assert_eq!(read(&dest), vec![to_i16(0.25); 10]);
assert_eq!(mix_wavs(&a, &empty, &dest).unwrap(), 10);
assert_eq!(read(&dest), vec![to_i16(0.25); 10]);
}
#[test]
fn without_audio_in_either_file_nothing_is_written() {
let dir = tempfile::tempdir().unwrap();
let (empty, dest) = (dir.path().join("e.wav"), dir.path().join("m.wav"));
write_track(&empty, &[]);
let err = mix_wavs(&empty, &dir.path().join("gone.wav"), &dest).unwrap_err();
assert_eq!(err.kind(), ErrorKind::NotFound);
assert!(!dest.exists());
}
#[test]
fn a_file_that_is_not_a_track_is_an_error() {
let dir = tempfile::tempdir().unwrap();
let (a, junk, dest) = (
dir.path().join("a.wav"),
dir.path().join("j.wav"),
dir.path().join("m.wav"),
);
write_track(&a, &[0.25; 10]);
fs::write(&junk, b"not a wav file at all").unwrap();
let err = mix_wavs(&a, &junk, &dest).unwrap_err();
assert_eq!(err.kind(), ErrorKind::InvalidData);
assert!(!dest.exists());
}
}
+3 -1
View File
@@ -11,10 +11,12 @@ mod conditioner;
mod flac; mod flac;
mod gap; mod gap;
mod level; mod level;
mod mix;
mod wav; mod wav;
pub use conditioner::Conditioner; pub use conditioner::Conditioner;
pub use flac::encode_flac; pub use flac::encode_flac;
pub use gap::GapFiller; pub use gap::GapFiller;
pub use level::rms_level; pub use level::rms_level;
pub use wav::{repair_wav, TrackWriter}; pub use mix::{mix_samples, mix_wavs};
pub use wav::{repair_wav, TrackWriter, SAMPLE_RATE};
+6
View File
@@ -172,6 +172,10 @@ pub struct CallDetail {
/// Whether the transcript is being uploaded right now. Set by the app, /// Whether the transcript is being uploaded right now. Set by the app,
/// which tracks its uploads; `false` from [`CallDetail::of`]. /// which tracks its uploads; `false` from [`CallDetail::of`].
pub uploading: bool, pub uploading: bool,
/// Whether the call's audio is still on disk to be played
/// ([`crate::playback::audio_available`]). Set by the app; `false` from
/// [`CallDetail::of`], which does not look at the files.
pub audio_available: bool,
} }
impl CallDetail { impl CallDetail {
@@ -209,6 +213,7 @@ impl CallDetail {
segments, segments,
merges: merge_views(merges, &speaker_names), merges: merge_views(merges, &speaker_names),
uploading: false, uploading: false,
audio_available: false,
speaker_names, speaker_names,
} }
} }
@@ -646,6 +651,7 @@ mod tests {
"into_label": "Sprecher 1", "into_label": "Sprecher 1",
}], }],
"uploading": false, "uploading": false,
"audio_available": false,
}) })
); );
} }
+1
View File
@@ -14,6 +14,7 @@ pub mod http;
pub mod merge; pub mod merge;
pub mod model; pub mod model;
pub mod pipeline; pub mod pipeline;
pub mod playback;
pub mod recorder; pub mod recorder;
pub mod retention; pub mod retention;
pub mod secrets; pub mod secrets;
+343
View File
@@ -0,0 +1,343 @@
//! Playback of a call in the detail view while its audio is still on disk:
//! both tracks are mixed into one cached mono WAV
//! (`<playback dir>/<id>.wav`), which the UI plays through the webview's
//! asset protocol. The cache is rebuilt only when a track is newer than it,
//! and removed with the call's audio.
use std::fs;
use std::io::{self, ErrorKind};
use std::path::{Path, PathBuf};
use std::time::SystemTime;
use serde::Serialize;
use crate::audio::{mix_wavs, SAMPLE_RATE};
use crate::model::{Call, CallId, CallStatus};
/// The call's audio was deleted (retention, or the files are gone).
pub const AUDIO_GONE: &str = "Audio nicht mehr vorhanden";
pub const RECORDING: &str = "Das Gespräch wird gerade aufgenommen";
pub const PROCESSING: &str = "Das Gespräch wird gerade verarbeitet – Wiedergabe danach möglich";
/// A WAV file of this size or less holds no audio (canonical header only).
const HEADER_LEN: u64 = 44;
/// What the UI needs to play a call: the mixed file (turned into an asset
/// URL by the UI) and its length.
#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct PlaybackInfo {
pub path: PathBuf,
pub duration_secs: f64,
}
/// Why `call` cannot be played, if it cannot: its tracks are still being
/// written (recording) or read for transcription (processing), or its
/// audio was deleted.
pub fn refusal(call: &Call) -> Option<&'static str> {
match call.status {
CallStatus::Recording => Some(RECORDING),
CallStatus::Processing => Some(PROCESSING),
_ if call.audio_deleted => Some(AUDIO_GONE),
_ => None,
}
}
/// Whether `call` still has audio to play: not deleted, and at least one
/// track holds more than a bare header.
pub fn audio_available(call: &Call) -> bool {
!call.audio_deleted
&& [&call.mic_path, &call.out_path]
.iter()
.any(|path| fs::metadata(path).is_ok_and(|m| m.len() > HEADER_LEN))
}
/// Whether a cached mix last written at `cache` can be reused for tracks
/// last written at `sources` (`None`: that track does not exist). It can if
/// it exists and is newer than every existing track; with no track left
/// there is nothing it could still stand for.
pub fn is_fresh(cache: Option<SystemTime>, sources: &[Option<SystemTime>]) -> bool {
let Some(cache) = cache else {
return false;
};
let mut any = false;
for source in sources.iter().flatten() {
if *source >= cache {
return false;
}
any = true;
}
any
}
/// Where call `id`'s mix is cached in the playback directory `dir`.
pub fn cache_path(dir: &Path, id: CallId) -> PathBuf {
dir.join(format!("{id}.wav"))
}
/// Returns call `call`'s mix in the playback directory `dir`, building it
/// first unless a fresh one is cached ([`is_fresh`]). Refused (see
/// [`refusal`]) while the call records or is processed and once its audio
/// is gone; a cached mix of a call whose audio is gone is removed.
pub fn prepare(call: &Call, dir: &Path) -> Result<PlaybackInfo, String> {
let path = cache_path(dir, call.id);
if let Some(msg) = refusal(call) {
if msg == AUDIO_GONE {
remove_cached(dir, call.id);
}
return Err(msg.to_string());
}
let sources = [modified(&call.mic_path), modified(&call.out_path)];
if is_fresh(modified(&path), &sources) {
match duration_secs(&path) {
Ok(duration_secs) => {
return Ok(PlaybackInfo {
path,
duration_secs,
})
}
Err(e) => log::warn!("call {}: rebuilding unreadable playback file: {e}", call.id),
}
}
fs::create_dir_all(dir).map_err(|e| format!("Wiedergabe-Ordner fehlt: {e}"))?;
match mix_wavs(&call.mic_path, &call.out_path, &path) {
Ok(samples) => Ok(PlaybackInfo {
path,
duration_secs: samples as f64 / SAMPLE_RATE as f64,
}),
Err(e) if e.kind() == ErrorKind::NotFound => {
remove_cached(dir, call.id);
Err(AUDIO_GONE.to_string())
}
Err(e) => Err(format!("Audio konnte nicht vorbereitet werden: {e}")),
}
}
/// Removes call `id`'s cached mix, if any (with the call or its audio).
/// Failures are logged: the playback directory is emptied at every start.
pub fn remove_cached(dir: &Path, id: CallId) {
let path = cache_path(dir, id);
match fs::remove_file(&path) {
Ok(()) => {}
Err(e) if e.kind() == ErrorKind::NotFound => {}
Err(e) => log::warn!("could not remove {}: {e}", path.display()),
}
}
/// Removes every cached mix (at startup, so customer audio never lingers
/// in the cache).
pub fn clear_cache(dir: &Path) -> io::Result<()> {
match fs::remove_dir_all(dir) {
Err(e) if e.kind() == ErrorKind::NotFound => Ok(()),
other => other,
}
}
fn modified(path: &Path) -> Option<SystemTime> {
fs::metadata(path).and_then(|m| m.modified()).ok()
}
fn duration_secs(path: &Path) -> Result<f64, hound::Error> {
let reader = hound::WavReader::open(path)?;
Ok(reader.duration() as f64 / reader.spec().sample_rate as f64)
}
#[cfg(test)]
mod tests {
use std::fs::File;
use std::time::Duration;
use chrono::Local;
use super::*;
use crate::audio::TrackWriter;
fn call(dir: &Path, status: CallStatus) -> Call {
Call {
id: 7,
app: "Microsoft Teams".to_string(),
started_at: Local::now(),
ended_at: Some(Local::now()),
status,
interrupted: false,
error: None,
mic_path: dir.join("7-mic.wav"),
out_path: dir.join("7-out.wav"),
transcribed_at: None,
audio_deleted: false,
title: None,
crm_transcript_id: None,
crm_base_url: None,
uploaded_at: None,
}
}
fn write_track(path: &Path, samples: &[f32]) {
let mut writer = TrackWriter::create(path).unwrap();
writer.write(samples).unwrap();
writer.finalize().unwrap();
}
/// A call with one second on the mic and half a second on the output.
fn recorded_call(dir: &Path, status: CallStatus) -> Call {
let call = call(dir, status);
write_track(&call.mic_path, &vec![0.1; 16_000]);
write_track(&call.out_path, &vec![0.2; 8_000]);
call
}
fn set_modified(path: &Path, time: SystemTime) {
File::options()
.write(true)
.open(path)
.unwrap()
.set_modified(time)
.unwrap();
}
#[test]
fn a_cache_newer_than_every_track_is_fresh() {
let t = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000);
let s = Duration::from_secs(1);
assert!(is_fresh(Some(t), &[Some(t - s), Some(t - s)]));
assert!(is_fresh(Some(t), &[None, Some(t - s)]));
assert!(!is_fresh(Some(t), &[Some(t - s), Some(t + s)]));
assert!(!is_fresh(Some(t), &[Some(t), Some(t - s)]), "same time");
assert!(!is_fresh(None, &[Some(t - s), Some(t - s)]), "no cache");
assert!(!is_fresh(Some(t), &[None, None]), "no tracks");
}
#[test]
fn recording_processing_and_deleted_calls_are_refused() {
let dir = Path::new("/nonexistent");
assert_eq!(refusal(&call(dir, CallStatus::Recording)), Some(RECORDING));
assert_eq!(
refusal(&call(dir, CallStatus::Processing)),
Some(PROCESSING)
);
let mut deleted = call(dir, CallStatus::Done);
deleted.audio_deleted = true;
assert_eq!(refusal(&deleted), Some(AUDIO_GONE));
for status in [
CallStatus::AwaitingDecision,
CallStatus::Done,
CallStatus::Failed,
] {
assert_eq!(refusal(&call(dir, status)), None, "{status:?}");
}
}
#[test]
fn audio_is_available_while_a_track_holds_audio() {
let dir = tempfile::tempdir().unwrap();
let mut c = call(dir.path(), CallStatus::Done);
assert!(!audio_available(&c), "no files");
write_track(&c.out_path, &[]);
assert!(!audio_available(&c), "header only");
write_track(&c.mic_path, &[0.1; 10]);
assert!(audio_available(&c));
c.audio_deleted = true;
assert!(!audio_available(&c), "deleted by retention");
}
#[test]
fn the_mix_is_built_once_and_reused() {
let dir = tempfile::tempdir().unwrap();
let playback = dir.path().join("playback");
let c = recorded_call(dir.path(), CallStatus::Done);
let info = prepare(&c, &playback).unwrap();
assert_eq!(info.path, playback.join("7.wav"));
assert!((info.duration_secs - 1.0).abs() < 1e-9);
// Marked as written long ago: a rebuild would show a new time.
let old = SystemTime::now() - Duration::from_secs(60);
set_modified(&c.mic_path, old - Duration::from_secs(60));
set_modified(&c.out_path, old - Duration::from_secs(60));
set_modified(&info.path, old);
assert_eq!(prepare(&c, &playback).unwrap(), info);
assert_eq!(modified(&info.path), Some(old), "reused, not rebuilt");
}
#[test]
fn the_mix_is_rebuilt_when_a_track_is_newer() {
let dir = tempfile::tempdir().unwrap();
let playback = dir.path().join("playback");
let c = recorded_call(dir.path(), CallStatus::AwaitingDecision);
let first = prepare(&c, &playback).unwrap();
let old = SystemTime::now() - Duration::from_secs(60);
set_modified(&first.path, old);
// Two seconds now; the file is newer than the cache.
write_track(&c.mic_path, &vec![0.1; 32_000]);
let second = prepare(&c, &playback).unwrap();
assert!((second.duration_secs - 2.0).abs() < 1e-9);
assert_ne!(modified(&second.path), Some(old));
}
#[test]
fn a_call_with_one_track_plays_that_track() {
let dir = tempfile::tempdir().unwrap();
let c = call(dir.path(), CallStatus::Failed);
write_track(&c.out_path, &vec![0.2; 4_000]);
let info = prepare(&c, &dir.path().join("playback")).unwrap();
assert!((info.duration_secs - 0.25).abs() < 1e-9);
}
#[test]
fn a_refused_call_builds_nothing() {
let dir = tempfile::tempdir().unwrap();
let playback = dir.path().join("playback");
for status in [CallStatus::Recording, CallStatus::Processing] {
let c = recorded_call(dir.path(), status);
assert!(prepare(&c, &playback).is_err());
assert!(!cache_path(&playback, c.id).exists());
}
}
#[test]
fn deleted_audio_removes_the_cached_mix() {
let dir = tempfile::tempdir().unwrap();
let playback = dir.path().join("playback");
let mut c = recorded_call(dir.path(), CallStatus::Done);
let info = prepare(&c, &playback).unwrap();
// Retention deleted the audio.
fs::remove_file(&c.mic_path).unwrap();
fs::remove_file(&c.out_path).unwrap();
c.audio_deleted = true;
assert_eq!(prepare(&c, &playback), Err(AUDIO_GONE.to_string()));
assert!(!info.path.exists());
}
#[test]
fn missing_files_remove_the_cached_mix() {
let dir = tempfile::tempdir().unwrap();
let playback = dir.path().join("playback");
let c = recorded_call(dir.path(), CallStatus::Done);
let info = prepare(&c, &playback).unwrap();
fs::remove_file(&c.mic_path).unwrap();
fs::remove_file(&c.out_path).unwrap();
assert_eq!(prepare(&c, &playback), Err(AUDIO_GONE.to_string()));
assert!(!info.path.exists());
}
#[test]
fn the_cache_is_removed_per_call_and_cleared_as_a_whole() {
let dir = tempfile::tempdir().unwrap();
let playback = dir.path().join("playback");
let c = recorded_call(dir.path(), CallStatus::Done);
let info = prepare(&c, &playback).unwrap();
fs::write(playback.join("8.wav"), b"other").unwrap();
remove_cached(&playback, c.id);
assert!(!info.path.exists());
assert!(playback.join("8.wav").exists());
remove_cached(&playback, c.id); // already gone: no error
clear_cache(&playback).unwrap();
assert!(!playback.exists());
clear_cache(&playback).unwrap(); // already gone: no error
}
}
+1 -1
View File
@@ -17,7 +17,7 @@ tauri-build = { version = "2.7.4", features = [] }
serde_json = "1.0" serde_json = "1.0"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
log = "0.4" log = "0.4"
tauri = { version = "2.12.5", features = ["tray-icon"] } tauri = { version = "2.12.5", features = ["tray-icon", "protocol-asset"] }
tauri-plugin-log = "2" tauri-plugin-log = "2"
tauri-plugin-autostart = "2" tauri-plugin-autostart = "2"
tauri-plugin-single-instance = "2" tauri-plugin-single-instance = "2"
+8
View File
@@ -11,6 +11,7 @@ use std::time::Duration;
use recorder_core::call_view::{CallDetail, CallListItem}; use recorder_core::call_view::{CallDetail, CallListItem};
use recorder_core::capture::Devices; use recorder_core::capture::Devices;
use recorder_core::model::CallId; use recorder_core::model::CallId;
use recorder_core::playback::PlaybackInfo;
use recorder_core::secrets::Service; use recorder_core::secrets::Service;
use recorder_core::settings::Settings; use recorder_core::settings::Settings;
use serde::Serialize; use serde::Serialize;
@@ -208,6 +209,13 @@ pub fn rename_call(
Ok(()) Ok(())
} }
/// Mixes the call's two tracks into one file to play in the detail view
/// (or reuses the cached mix); the UI plays it through the asset protocol.
#[tauri::command]
pub async fn prepare_playback(app: AppHandle, id: CallId) -> CmdResult<PlaybackInfo> {
blocking(app, move |app| app.state::<AppState>().prepare_playback(id)).await
}
/// Deletes a finished call's audio and row. Closes the prompt if no call /// Deletes a finished call's audio and row. Closes the prompt if no call
/// waits for it any more; otherwise the prompt reloads and shows the next. /// waits for it any more; otherwise the prompt reloads and shows the next.
/// While its DeepScript jobs are still on record, nothing is deleted and a /// While its DeepScript jobs are still on record, nothing is deleted and a
+7
View File
@@ -72,6 +72,9 @@ pub fn run() {
let data_dir = app.path().app_data_dir()?; let data_dir = app.path().app_data_dir()?;
let config_dir = app.path().app_config_dir()?; let config_dir = app.path().app_config_dir()?;
// The asset protocol may read this directory, and only this one
// (`app.security.assetProtocol.scope` in tauri.conf.json).
let playback_dir = app.path().app_cache_dir()?.join("playback");
std::fs::create_dir_all(&data_dir)?; std::fs::create_dir_all(&data_dir)?;
let store = Store::open(&data_dir.join("calls.sqlite"))?; let store = Store::open(&data_dir.join("calls.sqlite"))?;
let recovery = pipeline::recover_on_startup(&store)?; let recovery = pipeline::recover_on_startup(&store)?;
@@ -82,6 +85,7 @@ pub fn run() {
Box::new(KeyringSecrets), Box::new(KeyringSecrets),
config_dir.join("settings.json"), config_dir.join("settings.json"),
data_dir.join("recordings"), data_dir.join("recordings"),
playback_dir,
); );
let leftover_jobs = state.store.calls_with_remote_jobs().unwrap_or_else(|e| { let leftover_jobs = state.store.calls_with_remote_jobs().unwrap_or_else(|e| {
log::error!("could not read leftover remote jobs: {e}"); log::error!("could not read leftover remote jobs: {e}");
@@ -96,6 +100,8 @@ pub fn run() {
std::thread::spawn(KeyringSecrets::remove_unused_keys); std::thread::spawn(KeyringSecrets::remove_unused_keys);
// Before anything can start a run. // Before anything can start a run.
processing::clear_upload_dir(app.handle()); processing::clear_upload_dir(app.handle());
// Before any window can ask for a mix.
state.clear_playback_cache();
app.manage(state); app.manage(state);
app.manage(PendingUpdate::default()); app.manage(PendingUpdate::default());
app.manage(UpdateSchedule::default()); app.manage(UpdateSchedule::default());
@@ -154,6 +160,7 @@ pub fn run() {
commands::merge_speakers, commands::merge_speakers,
commands::unmerge_speaker, commands::unmerge_speaker,
commands::rename_call, commands::rename_call,
commands::prepare_playback,
backend::backend_server, backend::backend_server,
backend::backend_status, backend::backend_status,
backend::backend_login, backend::backend_login,
+162 -3
View File
@@ -2,9 +2,10 @@
//! //!
//! Holds the store, the settings, the secret store, the HTTP transport, the //! Holds the store, the settings, the secret store, the HTTP transport, the
//! recording that is currently running, the call detector, the calls //! recording that is currently running, the call detector, the calls
//! awaiting a decision in the end-of-call prompt and the calls being //! awaiting a decision in the end-of-call prompt, the calls being
//! processed. Everything here is independent of windows and //! processed and the cached mixes for playback. Everything here is
//! events; the command layer turns results into UI updates. //! independent of windows and events; the command layer turns results into
//! UI updates.
use std::collections::{HashSet, VecDeque}; use std::collections::{HashSet, VecDeque};
use std::fs; use std::fs;
@@ -23,6 +24,7 @@ use recorder_core::capture::{CaptureSource, OutputTarget};
use recorder_core::detect::{CallDetector, DetectorEvent, MicSession}; use recorder_core::detect::{CallDetector, DetectorEvent, MicSession};
use recorder_core::http::{HttpTransport, ReqwestTransport}; use recorder_core::http::{HttpTransport, ReqwestTransport};
use recorder_core::model::{Call, CallId, CallStatus, Speaker}; use recorder_core::model::{Call, CallId, CallStatus, Speaker};
use recorder_core::playback::{self, PlaybackInfo};
use recorder_core::recorder::Recorder; use recorder_core::recorder::Recorder;
use recorder_core::secrets::{SecretStore, Service}; use recorder_core::secrets::{SecretStore, Service};
use recorder_core::settings::Settings; use recorder_core::settings::Settings;
@@ -147,6 +149,12 @@ pub struct AppState {
settings: Mutex<Settings>, settings: Mutex<Settings>,
settings_path: PathBuf, settings_path: PathBuf,
recordings_dir: PathBuf, recordings_dir: PathBuf,
/// Where the mixes for playback are cached (`<app cache>/playback`,
/// the only directory the webview's asset protocol may read).
playback_dir: PathBuf,
/// Held while a mix is built or removed, so a call deleted meanwhile
/// never leaves its mix behind.
playback_lock: Mutex<()>,
current: Mutex<Option<CurrentRecording>>, current: Mutex<Option<CurrentRecording>>,
detector: Mutex<CallDetector>, detector: Mutex<CallDetector>,
/// Stopped calls awaiting the end-of-call prompt's decision, oldest /// Stopped calls awaiting the end-of-call prompt's decision, oldest
@@ -201,6 +209,7 @@ impl AppState {
secrets: Box<dyn SecretStore>, secrets: Box<dyn SecretStore>,
settings_path: PathBuf, settings_path: PathBuf,
recordings_dir: PathBuf, recordings_dir: PathBuf,
playback_dir: PathBuf,
) -> Self { ) -> Self {
let settings = Settings::load(&settings_path); let settings = Settings::load(&settings_path);
let detector = CallDetector::new(detector_config(&settings)); let detector = CallDetector::new(detector_config(&settings));
@@ -211,6 +220,8 @@ impl AppState {
settings: Mutex::new(settings), settings: Mutex::new(settings),
settings_path, settings_path,
recordings_dir, recordings_dir,
playback_dir,
playback_lock: Mutex::new(()),
current: Mutex::new(None), current: Mutex::new(None),
detector: Mutex::new(detector), detector: Mutex::new(detector),
pending_prompts: Mutex::new(VecDeque::new()), pending_prompts: Mutex::new(VecDeque::new()),
@@ -656,9 +667,44 @@ impl AppState {
&self.settings().language, &self.settings().language,
); );
detail.uploading = self.is_uploading(id); detail.uploading = self.is_uploading(id);
detail.audio_available = playback::audio_available(&call);
Ok(detail) Ok(detail)
} }
/// Mixes call `id`'s tracks for playback (or reuses the cached mix,
/// see [`playback::prepare`]). Refused while the call is recorded or
/// processed, and once its audio is gone.
pub fn prepare_playback(&self, id: CallId) -> Result<PlaybackInfo, String> {
if lock(&self.current)
.as_ref()
.is_some_and(|cur| cur.call_id == id)
{
return Err(playback::RECORDING.to_string());
}
let _building = lock(&self.playback_lock);
// Read under the lock: a call deleted before it is not mixed, one
// deleted meanwhile has its mix removed once the lock is free.
let call = self.find_call(id)?;
playback::prepare(&call, &self.playback_dir)
}
/// Removes call `id`'s cached mix, once the call or its audio is
/// deleted.
pub fn remove_playback(&self, id: CallId) {
let _building = lock(&self.playback_lock);
playback::remove_cached(&self.playback_dir, id);
}
/// Empties the playback cache (at startup), so mixed customer audio
/// never lingers from an earlier run.
pub fn clear_playback_cache(&self) {
let dir = &self.playback_dir;
match playback::clear_cache(dir) {
Ok(()) => log::info!("cleared the playback cache {}", dir.display()),
Err(e) => log::warn!("could not empty {}: {e}", dir.display()),
}
}
/// Renames a speaker of call `id`; a blank `name` restores the default /// Renames a speaker of call `id`; a blank `name` restores the default
/// label ("Ich", "Sprecher N"). /// label ("Ich", "Sprecher N").
pub fn rename_speaker(&self, id: CallId, speaker_key: &str, name: &str) -> Result<(), String> { pub fn rename_speaker(&self, id: CallId, speaker_key: &str, name: &str) -> Result<(), String> {
@@ -789,6 +835,7 @@ impl AppState {
} }
drop(uploading); drop(uploading);
drop(processing); drop(processing);
self.remove_playback(id);
Ok(self.prompts_waiting()) Ok(self.prompts_waiting())
} }
@@ -967,6 +1014,7 @@ mod tests {
Box::new(MemorySecrets::default()), Box::new(MemorySecrets::default()),
dir.path().join("settings.json"), dir.path().join("settings.json"),
dir.path().join("recordings"), dir.path().join("recordings"),
dir.path().join("playback"),
); );
Fixture { state, dir } Fixture { state, dir }
} }
@@ -1897,4 +1945,115 @@ mod tests {
OutputTarget::Device("id-Lautsprecher".into()) OutputTarget::Device("id-Lautsprecher".into())
); );
} }
/// A finished call whose tracks hold real audio: one second on the mic,
/// half a second on the output.
fn playable_call(f: &Fixture, name: &str) -> CallId {
let (id, mic, out) = finished_call(f, name);
for (path, samples) in [(&mic, 16_000), (&out, 8_000)] {
let mut writer = recorder_core::audio::TrackWriter::create(path).unwrap();
writer.write(&vec![0.1; samples]).unwrap();
writer.finalize().unwrap();
}
id
}
fn playback_file(f: &Fixture, id: CallId) -> PathBuf {
f.dir.path().join("playback").join(format!("{id}.wav"))
}
#[test]
fn a_finished_call_is_mixed_into_the_playback_cache() {
let f = fixture();
let id = playable_call(&f, "a");
assert!(f.state.call_detail(id).unwrap().audio_available);
let info = f.state.prepare_playback(id).unwrap();
assert_eq!(info.path, playback_file(&f, id));
assert!((info.duration_secs - 1.0).abs() < 1e-9);
assert!(info.path.exists());
}
#[test]
fn recording_and_processing_calls_are_not_played() {
let f = fixture();
let dir = f.dir.path();
let recording = f
.state
.store
.create_call(
"Microsoft Teams",
at(10, 0, 0),
&dir.join("r-mic.wav"),
&dir.join("r-out.wav"),
)
.unwrap();
assert_eq!(
f.state.prepare_playback(recording),
Err(playback::RECORDING.to_string())
);
let processing = playable_call(&f, "p");
f.state
.store
.set_status(processing, CallStatus::Processing, None)
.unwrap();
assert_eq!(
f.state.prepare_playback(processing),
Err(playback::PROCESSING.to_string())
);
assert!(!playback_file(&f, processing).exists());
assert_eq!(
f.state.prepare_playback(999),
Err("Gespräch nicht gefunden".to_string())
);
}
#[test]
fn deleted_audio_is_not_played_and_its_mix_is_removed() {
let f = fixture();
let id = playable_call(&f, "a");
f.state.prepare_playback(id).unwrap();
// What the retention sweep does to a transcribed call's audio.
let call = f.state.store.get_call(id).unwrap().unwrap();
fs::remove_file(&call.mic_path).unwrap();
fs::remove_file(&call.out_path).unwrap();
f.state.store.mark_audio_deleted(id).unwrap();
assert!(!f.state.call_detail(id).unwrap().audio_available);
assert_eq!(
f.state.prepare_playback(id),
Err(playback::AUDIO_GONE.to_string())
);
assert!(!playback_file(&f, id).exists());
}
#[test]
fn deleting_a_call_removes_its_mix() {
let f = fixture();
let id = playable_call(&f, "a");
let other = playable_call(&f, "b");
f.state.prepare_playback(id).unwrap();
f.state.prepare_playback(other).unwrap();
f.state.discard_call(id).unwrap();
assert!(!playback_file(&f, id).exists());
assert!(playback_file(&f, other).exists());
assert_eq!(
f.state.prepare_playback(id),
Err("Gespräch nicht gefunden".to_string())
);
}
#[test]
fn the_playback_cache_is_emptied_at_startup() {
let f = fixture();
let id = playable_call(&f, "a");
f.state.prepare_playback(id).unwrap();
f.state.clear_playback_cache();
assert!(!f.dir.path().join("playback").exists());
f.state.clear_playback_cache(); // nothing left: no error
}
} }
+7 -1
View File
@@ -44,7 +44,13 @@
} }
], ],
"security": { "security": {
"csp": null "csp": null,
"assetProtocol": {
"enable": true,
"scope": [
"$APPCACHE/playback/**"
]
}
} }
}, },
"bundle": { "bundle": {
+220
View File
@@ -0,0 +1,220 @@
<script lang="ts">
import { convertFileSrc } from '@tauri-apps/api/core'
import { onDestroy } from 'svelte'
import { errorText, preparePlayback, type CallId } from './api'
import { formatOffset } from './format'
let {
callId,
estimate,
ontime,
onplaying,
}: {
callId: CallId
/** The call's length from its times, shown until the mix is ready. */
estimate: number
/** The playback position, once the mix is loaded (also while paused). */
ontime?: (secs: number) => void
onplaying?: (playing: boolean) => void
} = $props()
let audio: HTMLAudioElement
/** Resolves once the mix is loaded; `false` if preparing it failed. */
let loading: Promise<boolean> | null = null
let preparing = $state(false)
let playing = $state(false)
// svelte-ignore state_referenced_locally
let duration = $state(estimate)
let position = $state(0)
/** While the slider is dragged it shows the drag, not the audio. */
let dragging = $state(false)
let error = $state('')
/** The mix held in memory where the webview cannot stream it (Linux). */
let blobUrl: string | null = null
/** Mixes the call (first use only) and loads the mix into the player. */
function load(): Promise<boolean> {
loading ??= (async () => {
preparing = true
error = ''
try {
const info = await preparePlayback(callId)
duration = info.duration_secs
const url = convertFileSrc(info.path)
try {
await loadSource(url)
} catch (err) {
if (!(err instanceof MediaError) || err.code !== MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED) {
throw err
}
// WebKitGTK (Linux) plays no media from a custom scheme such as
// asset:// (WebView2 gets http://asset.localhost instead), but
// fetches it fine: there the mix is played from memory.
const response = await fetch(url)
if (!response.ok) throw new Error(`Audio konnte nicht geladen werden (${response.status}).`)
blobUrl = URL.createObjectURL(await response.blob())
await loadSource(blobUrl)
}
return true
} catch (err) {
error = err instanceof MediaError ? 'Audio konnte nicht geladen werden.' : errorText(err)
loading = null // a later click tries again
return false
} finally {
preparing = false
}
})()
return loading
}
/** Loads `src` into the player; rejects with its `MediaError`. */
function loadSource(src: string): Promise<void> {
const ready = new Promise<void>((resolve, reject) => {
const ok = () => {
audio.removeEventListener('error', fail)
resolve()
}
const fail = () => {
audio.removeEventListener('loadedmetadata', ok)
reject(audio.error)
}
audio.addEventListener('loadedmetadata', ok, { once: true })
audio.addEventListener('error', fail, { once: true })
})
audio.src = src
return ready
}
async function start() {
try {
await audio.play()
} catch (err) {
error = `Wiedergabe nicht möglich: ${errorText(err)}`
}
}
async function toggle() {
if (playing) {
audio.pause()
} else if (await load()) {
await start()
}
}
/** Jumps to `secs` and plays from there (a click on a transcript line). */
export async function playFrom(secs: number) {
if (!(await load())) return
audio.currentTime = Math.min(secs, duration)
position = audio.currentTime
if (audio.paused) await start()
}
async function seekTo(secs: number) {
dragging = false
position = secs
if (!(await load())) return
audio.currentTime = secs
}
function ontimeupdate() {
if (!dragging) position = audio.currentTime
ontime?.(audio.currentTime)
}
function setPlaying(value: boolean) {
playing = value
onplaying?.(value)
}
onDestroy(() => {
// Lets go of the file, so the call can be deleted right away.
audio?.pause()
audio?.removeAttribute('src')
audio?.load()
if (blobUrl) URL.revokeObjectURL(blobUrl)
onplaying?.(false)
})
</script>
<div class="player">
<audio
bind:this={audio}
preload="metadata"
{ontimeupdate}
onplay={() => setPlaying(true)}
onpause={() => setPlaying(false)}
onended={() => setPlaying(false)}
></audio>
<button
class="play"
disabled={preparing}
title={playing ? 'Pause' : 'Abspielen'}
aria-label={playing ? 'Pause' : 'Abspielen'}
onclick={toggle}
>
{#if preparing}…{:else if playing}❚❚{:else}▶{/if}
</button>
<input
class="timeline"
type="range"
min="0"
max={duration || 1}
step="0.1"
value={position}
disabled={preparing}
aria-label="Position"
oninput={(e) => {
dragging = true
position = Number(e.currentTarget.value)
}}
onchange={(e) => seekTo(Number(e.currentTarget.value))}
/>
<span class="time">{formatOffset(position)} / {formatOffset(duration)}</span>
{#if preparing}
<span class="muted">Audio wird vorbereitet…</span>
{:else if error}
<span class="error">{error}</span>
{/if}
</div>
<style>
.player {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 6px 10px;
min-width: 0;
flex: 1;
}
.play {
flex-shrink: 0;
width: 30px;
height: 30px;
border-radius: 50%;
border: none;
background: var(--primary);
color: #fff;
font-size: 11px;
line-height: 1;
cursor: pointer;
}
.play:disabled {
opacity: 0.6;
cursor: default;
}
.timeline {
flex: 1 1 160px;
min-width: 120px;
accent-color: var(--primary);
}
.time {
color: var(--muted);
font-size: 12px;
font-variant-numeric: tabular-nums;
white-space: nowrap;
}
</style>
+100 -4
View File
@@ -8,6 +8,9 @@
onrename, onrename,
onmerge, onmerge,
onunmerge, onunmerge,
activeIndex = null,
follow = false,
onseek,
}: { }: {
/** The transcript with merges applied: merged speakers no longer appear. */ /** The transcript with merges applied: merged speakers no longer appear. */
segments: SegmentView[] segments: SegmentView[]
@@ -18,8 +21,45 @@
onmerge: (fromKey: string, intoKey: string) => Promise<void> onmerge: (fromKey: string, intoKey: string) => Promise<void>
/** Undoes the merge of `fromKey`. */ /** Undoes the merge of `fromKey`. */
onunmerge: (fromKey: string) => Promise<void> onunmerge: (fromKey: string) => Promise<void>
/** The line being heard; it is highlighted. */
activeIndex?: number | null
/** Scrolls the line being heard into view when it leaves the view. */
follow?: boolean
/** Plays from `secs`, a line's start (only while the audio is there). */
onseek?: (secs: number) => void
} = $props() } = $props()
let lineElements: HTMLElement[] = $state([])
/** A click on a line's text plays from it, unless text was selected. */
function lineClick(start: number) {
if (window.getSelection()?.toString()) return
onseek?.(start)
}
/** The nearest ancestor that scrolls vertically. */
function scrollParent(node: HTMLElement): HTMLElement | null {
for (let el = node.parentElement; el; el = el.parentElement) {
if (/(auto|scroll)/.test(getComputedStyle(el).overflowY)) return el
}
return null
}
// Following the playback scrolls only once the line has left the view
// (also when it hides under the sticky player), and then centres it.
$effect(() => {
if (!follow || activeIndex === null) return
const line = lineElements[activeIndex]
const parent = line && scrollParent(line)
if (!line || !parent) return
const view = parent.getBoundingClientRect()
const box = line.getBoundingClientRect()
const covered = parseFloat(getComputedStyle(line).scrollMarginTop) || 0
if (box.top < view.top + covered || box.bottom > view.bottom) {
line.scrollIntoView({ block: 'center' })
}
})
// "Ich" is always the primary blue; remote speakers cycle through the rest. // "Ich" is always the primary blue; remote speakers cycle through the rest.
const ME_COLOR = '#3b5bdb' const ME_COLOR = '#3b5bdb'
const REMOTE_COLORS = ['#2b8a3e', '#c2255c', '#e8590c', '#7048e8', '#0c8599', '#5c940d'] const REMOTE_COLORS = ['#2b8a3e', '#c2255c', '#e8590c', '#7048e8', '#0c8599', '#5c940d']
@@ -186,8 +226,27 @@
<div class="lines"> <div class="lines">
{#each segments as segment, i (i)} {#each segments as segment, i (i)}
<div class="line"> <!-- The time is the keyboard-accessible way to play a line. -->
<!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_static_element_interactions -->
<div
class="line"
class:active={i === activeIndex}
class:seekable={onseek !== undefined}
bind:this={lineElements[i]}
onclick={() => lineClick(segment.start)}
>
{#if onseek}
<button
class="time"
title="Ab hier abspielen"
onclick={(e) => {
e.stopPropagation()
onseek(segment.start)
}}>{formatOffset(segment.start)}</button
>
{:else}
<span class="time">{formatOffset(segment.start)}</span> <span class="time">{formatOffset(segment.start)}</span>
{/if}
<span class="speaker" style:color={speakerColor(segment.speaker_key)}>{segment.label}</span> <span class="speaker" style:color={speakerColor(segment.speaker_key)}>{segment.label}</span>
<span class="text">{segment.text}</span> <span class="text">{segment.text}</span>
</div> </div>
@@ -324,12 +383,33 @@
.lines { .lines {
display: grid; display: grid;
grid-template-columns: auto auto 1fr; grid-template-columns: auto auto 1fr;
gap: 8px 12px; column-gap: 12px;
margin-top: 14px; margin: 10px -8px 0;
} }
/* One row of the transcript grid that can be highlighted as a whole. */
.line { .line {
display: contents; display: grid;
grid-column: 1 / -1;
grid-template-columns: subgrid;
padding: 4px 8px;
border-radius: 6px;
/* Room for the sticky player when the line is scrolled into view. */
scroll-margin-top: 64px;
}
.line.seekable {
cursor: pointer;
}
.line.seekable:hover {
background: var(--bg);
}
.line.active,
.line.active:hover {
background: var(--primary-soft);
box-shadow: inset 3px 0 0 var(--primary);
} }
.time { .time {
@@ -339,6 +419,22 @@
padding-top: 1px; padding-top: 1px;
} }
button.time {
align-self: start;
border: none;
background: none;
padding: 1px 0 0;
font-family: inherit;
line-height: inherit;
text-align: left;
cursor: pointer;
}
button.time:hover {
color: var(--primary);
text-decoration: underline;
}
.speaker { .speaker {
font-weight: 600; font-weight: 600;
max-width: 160px; max-width: 160px;
+12
View File
@@ -140,6 +140,15 @@ export interface CallDetail {
merges: SpeakerMerge[] merges: SpeakerMerge[]
/** The transcript is being uploaded right now. */ /** The transcript is being uploaded right now. */
uploading: boolean uploading: boolean
/** The call's audio is still on this PC and can be played. */
audio_available: boolean
}
/** A call's two tracks mixed into one file for playback. */
export interface PlaybackInfo {
/** The mixed WAV in the playback cache; play it via `convertFileSrc`. */
path: string
duration_secs: number
} }
/** A newer version the release feed offers. */ /** A newer version the release feed offers. */
@@ -188,6 +197,9 @@ export const getCallBrief = (id: CallId) => invoke<CallBrief>('get_call_brief',
export const transcribeCall = (id: CallId) => invoke<void>('transcribe_call', { id }) export const transcribeCall = (id: CallId) => invoke<void>('transcribe_call', { id })
export const retryCall = (id: CallId) => invoke<void>('retry_call', { id }) export const retryCall = (id: CallId) => invoke<void>('retry_call', { id })
export const discardCall = (id: CallId) => invoke<void>('discard_call', { id }) export const discardCall = (id: CallId) => invoke<void>('discard_call', { id })
/** Mixes the call's tracks (or reuses the cached mix); refused while the
* call is recorded or processed and once its audio is gone. */
export const preparePlayback = (id: CallId) => invoke<PlaybackInfo>('prepare_playback', { id })
export const listCalls = () => invoke<CallListItem[]>('list_calls') export const listCalls = () => invoke<CallListItem[]>('list_calls')
export const getCall = (id: CallId) => invoke<CallDetail>('get_call', { id }) export const getCall = (id: CallId) => invoke<CallDetail>('get_call', { id })
/** An empty name restores the default label. */ /** An empty name restores the default label. */
+21
View File
@@ -0,0 +1,21 @@
// Couples playback to the transcript: which line is being heard.
/** The part of a transcript line the lookup needs. */
export interface Timed {
start: number
end: number
}
/** Index of the line heard at `secs`: the one whose `[start, end)` holds
* it — of overlapping lines (both sides talking at once) the one that
* started last. `null` before the first line, in a pause between lines and
* after the last one. */
export function segmentAt(segments: readonly Timed[], secs: number): number | null {
let found: number | null = null
segments.forEach((segment, i) => {
if (segment.start <= secs && secs < segment.end) {
if (found === null || segment.start >= segments[found].start) found = i
}
})
return found
}
+123 -1
View File
@@ -5,6 +5,7 @@
errorText, errorText,
getCall, getCall,
getRecordingState, getRecordingState,
getSettings,
listCalls, listCalls,
mergeSpeakers, mergeSpeakers,
onBackendLoginError, onBackendLoginError,
@@ -27,6 +28,8 @@
} from '../lib/api' } from '../lib/api'
import CallList from '../lib/CallList.svelte' import CallList from '../lib/CallList.svelte'
import { formatDateTime, formatDetailTime, formatMinutes } from '../lib/format' import { formatDateTime, formatDetailTime, formatMinutes } from '../lib/format'
import Player from '../lib/Player.svelte'
import { segmentAt } from '../lib/playback'
import StatusPill from '../lib/StatusPill.svelte' import StatusPill from '../lib/StatusPill.svelte'
import TranscriptView from '../lib/TranscriptView.svelte' import TranscriptView from '../lib/TranscriptView.svelte'
import UpdateBanner from '../lib/UpdateBanner.svelte' import UpdateBanner from '../lib/UpdateBanner.svelte'
@@ -56,6 +59,16 @@
let uploading = $state<CallId[]>([]) let uploading = $state<CallId[]>([])
let uploadErrors = $state<Record<CallId, string>>({}) let uploadErrors = $state<Record<CallId, string>>({})
// Playback of the shown call (the player itself lives in `Player`).
let player = $state<ReturnType<typeof Player>>()
/** The playback position once the mix is loaded, else `null`. */
let playerTime = $state<number | null>(null)
let playing = $state(false)
/** Whether the transcript scrolls along; scrolling by hand ends it. */
let follow = $state(true)
/** For "… werden N Tage nach der Transkription gelöscht". */
let retentionDays = $state<number | null>(null)
let editingTitle = $state(false) let editingTitle = $state(false)
let titleDraft = $state('') let titleDraft = $state('')
let savingTitle = $state(false) let savingTitle = $state(false)
@@ -103,9 +116,16 @@
} }
} }
function resetPlayback() {
playerTime = null
playing = false
follow = true
}
function select(id: CallId) { function select(id: CallId) {
if (id === selectedId) return if (id === selectedId) return
selectedId = id selectedId = id
resetPlayback()
detail = null detail = null
detailError = '' detailError = ''
actionError = '' actionError = ''
@@ -118,6 +138,7 @@
function clearSelection() { function clearSelection() {
selectedId = null selectedId = null
resetPlayback()
detail = null detail = null
detailError = '' detailError = ''
actionError = '' actionError = ''
@@ -145,6 +166,9 @@
}), }),
] ]
refreshList() refreshList()
getSettings()
.then((settings) => (retentionDays = settings.retention_days))
.catch(() => {})
getRecordingState() getRecordingState()
.then((state) => (recordingState = state)) .then((state) => (recordingState = state))
.catch((err) => (recordError = errorText(err))) .catch((err) => (recordError = errorText(err)))
@@ -310,6 +334,28 @@
node.select() node.select()
} }
/** A click on a transcript line: plays from there and follows again. */
function playFrom(secs: number) {
follow = true
player?.playFrom(secs)
}
const SCROLL_KEYS = ['ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' ']
/** Scrolling the detail by hand while playing stops following the
* playback (until "folgen" or a click on a line). Only user input counts:
* the transcript's own scrolling raises none of these events. */
function manualScroll(event: Event) {
if (!playing) return
if (event instanceof KeyboardEvent) {
const target = event.target as HTMLElement
if (!SCROLL_KEYS.includes(event.key) || target.closest('input, textarea, button')) return
}
// A press on the scroll bar (not on the content).
if (event instanceof PointerEvent && event.target !== event.currentTarget) return
follow = false
}
/** "Microsoft Teams · heute 10:14 · 32 min" (no duration while recording). */ /** "Microsoft Teams · heute 10:14 · 32 min" (no duration while recording). */
function defaultTitle(call: CallListItem): string { function defaultTitle(call: CallListItem): string {
const parts = [call.app, formatDetailTime(call.started_at)] const parts = [call.app, formatDetailTime(call.started_at)]
@@ -330,6 +376,11 @@
// Calls that still need the user's attention (not transcribed, or // Calls that still need the user's attention (not transcribed, or
// transcribed but not uploaded here) — shown as "Gespräche · N offen". // transcribed but not uploaded here) — shown as "Gespräche · N offen".
let openCount = $derived(calls.filter((c) => c.needs_action).length) let openCount = $derived(calls.filter((c) => c.needs_action).length)
/** The transcript line being heard, highlighted while the mix is loaded. */
let activeIndex = $derived(
detail && playerTime !== null ? segmentAt(detail.segments, playerTime) : null,
)
</script> </script>
<main class="main"> <main class="main">
@@ -364,7 +415,15 @@
</div> </div>
</aside> </aside>
<section class="detail"> <!-- The handlers only watch for scrolling by hand (see manualScroll). -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
<section
class="detail"
onwheel={manualScroll}
ontouchmove={manualScroll}
onkeydown={manualScroll}
onpointerdown={manualScroll}
>
{#if detail} {#if detail}
{@const call = detail.call} {@const call = detail.call}
<header class="detail-header"> <header class="detail-header">
@@ -433,6 +492,36 @@
{/if} {/if}
</header> </header>
{#if call.status !== 'recording' && call.status !== 'processing'}
{#if detail.audio_available}
<div class="playback-bar">
{#key call.id}
<Player
bind:this={player}
callId={call.id}
estimate={call.duration_secs}
ontime={(secs) => (playerTime = secs)}
onplaying={(value) => (playing = value)}
/>
{/key}
{#if call.status === 'done' && detail.segments.length > 0}
<button
class="btn small follow"
class:on={follow}
aria-pressed={follow}
title="Das Transkript scrollt mit der Wiedergabe mit"
onclick={() => (follow = !follow)}>folgen</button
>
{/if}
</div>
{:else}
<p class="muted no-audio">
Audio nicht mehr vorhanden – Aufnahmen werden {retentionDays ?? '…'} Tage nach der
Transkription gelöscht (<a href="#/settings">Einstellungen</a>).
</p>
{/if}
{/if}
<div class="body"> <div class="body">
{#if call.status === 'recording'} {#if call.status === 'recording'}
<p class="state rec">● Aufnahme läuft</p> <p class="state rec">● Aufnahme läuft</p>
@@ -456,6 +545,9 @@
onrename={rename} onrename={rename}
onmerge={merge} onmerge={merge}
onunmerge={unmerge} onunmerge={unmerge}
{activeIndex}
follow={follow && playing}
onseek={detail.audio_available ? playFrom : undefined}
/> />
{/if} {/if}
</div> </div>
@@ -681,6 +773,36 @@
margin: 10px 0 0; margin: 10px 0 0;
} }
/* Stays at the top while the transcript scrolls under it. */
.playback-bar {
position: sticky;
top: -18px; /* the detail's top padding: flush with its edge */
z-index: 2;
display: flex;
align-items: center;
gap: 10px;
margin: 0 -24px 14px;
padding: 8px 24px;
background: var(--bg);
border-bottom: 1px solid var(--border);
}
.follow {
flex-shrink: 0;
color: var(--muted);
font-weight: 600;
}
.follow.on {
border-color: var(--primary);
background: var(--primary-soft);
color: var(--primary);
}
.no-audio {
margin: 0 0 14px;
}
.state { .state {
color: var(--muted); color: var(--muted);
} }