feat #14740: transcription pipeline

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
hauke
2026-11-05 16:41:53 +01:00
co-authored by Claude Opus 5.5
parent 1420e56e77
commit e2828791da
13 changed files with 1066 additions and 17 deletions
Generated
+61
View File
@@ -457,6 +457,7 @@ dependencies = [
"tauri-build",
"tauri-plugin-autostart",
"tauri-plugin-log",
"tauri-plugin-notification",
"tauri-plugin-single-instance",
"tempfile",
]
@@ -2369,6 +2370,20 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f"
[[package]]
name = "mac-notification-sys"
version = "0.6.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd604973958ddcc11b561193c0fb96ba146506ef2f231ef2e7c35fd2cbc9beca"
dependencies = [
"cc",
"log",
"objc2",
"objc2-foundation",
"time",
"uuid",
]
[[package]]
name = "markup5ever"
version = "0.39.0"
@@ -2534,6 +2549,20 @@ dependencies = [
"libc",
]
[[package]]
name = "notify-rust"
version = "4.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4587364a9a0074333429b3df75a30a205340c56a536ca3eb6ca0e59b87bbf8af"
dependencies = [
"futures-lite",
"log",
"mac-notification-sys",
"serde",
"tauri-winrt-notification",
"zbus 5.19.0",
]
[[package]]
name = "num"
version = "0.4.3"
@@ -3382,6 +3411,7 @@ dependencies = [
"flacenc",
"hound",
"keyring",
"log",
"reqwest 0.12.28",
"rubato",
"rusqlite",
@@ -4447,6 +4477,26 @@ dependencies = [
"time",
]
[[package]]
name = "tauri-plugin-notification"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bc6c194a95a535abdfbe3bc1af9a43935b8b9b29c216b8e71af8b59132703cf"
dependencies = [
"log",
"notify-rust",
"rand 0.10.3",
"serde",
"serde_json",
"serde_repr",
"tauri",
"tauri-plugin",
"thiserror 2.0.21",
"time",
"url",
"zbus 5.19.0",
]
[[package]]
name = "tauri-plugin-single-instance"
version = "2.5.5"
@@ -4562,6 +4612,17 @@ dependencies = [
"toml 1.2.0+spec-1.1.0",
]
[[package]]
name = "tauri-winrt-notification"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f37a6c354fd28fc9e322ed9bd47e3959576dad28c9d58ea1cf888cce1c7ccb36"
dependencies = [
"thiserror 2.0.21",
"windows",
"windows-version",
]
[[package]]
name = "tempfile"
version = "3.27.0"
+1
View File
@@ -23,6 +23,7 @@ keyring = { version = "3", features = ["windows-native", "sync-secret-service",
async-trait = "0.1"
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "multipart", "json", "stream"] }
tokio = { version = "1", features = ["rt", "fs", "time", "macros"] }
log = "0.4"
[dev-dependencies]
tempfile = "3"
+1
View File
@@ -10,6 +10,7 @@ pub mod detect;
pub mod http;
pub mod merge;
pub mod model;
pub mod pipeline;
pub mod recorder;
pub mod retention;
pub mod secrets;
+683
View File
@@ -0,0 +1,683 @@
//! Turns a recorded call into a stored transcript: encode both tracks to
//! FLAC, transcribe them remotely, merge the results and store them. Also
//! the startup routine that repairs recordings cut off by a crash.
//!
//! The [`Store`] is shared with the rest of the app; it locks its connection
//! only inside each call, so no lock is ever held across an `.await`.
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::time::Duration;
use chrono::{Local, TimeDelta};
use thiserror::Error;
use crate::audio::{encode_flac, repair_wav};
use crate::merge::merge;
use crate::model::{Call, CallId, CallStatus, Word};
use crate::store::{RecoveryReport, Store, StoreError};
use crate::transcribe::{TranscribeError, TranscribeOptions, Transcriber};
/// Settings a pipeline run uses.
#[derive(Debug, Clone, PartialEq)]
pub struct PipelineOptions {
/// ISO 639-1 code, or `"auto"`.
pub language: String,
pub deepscript_model: String,
}
/// Everything a pipeline run can fail with.
#[derive(Debug, Error)]
pub enum PipelineError {
#[error("transcription failed: {0:?}")]
Transcribe(TranscribeError),
#[error(transparent)]
Store(#[from] StoreError),
#[error("cannot read audio file {}: {source}", path.display())]
Io { path: PathBuf, source: io::Error },
#[error("call {0} not found")]
CallNotFound(CallId),
}
impl From<TranscribeError> for PipelineError {
fn from(e: TranscribeError) -> Self {
PipelineError::Transcribe(e)
}
}
impl PipelineError {
/// German, user-facing text for this error. Never includes an API key
/// or transcript text.
pub fn user_message(&self) -> String {
match self {
PipelineError::Transcribe(e) => e.user_message(),
PipelineError::Store(e) => format!("Interner Fehler: {e}"),
PipelineError::Io { path, source } => {
format!("Audiodatei nicht lesbar: {} ({source})", path.display())
}
PipelineError::CallNotFound(_) => "Gespräch nicht gefunden".to_string(),
}
}
}
/// `track` value of the mic track's remote job.
const MIC_TRACK: &str = "mic";
/// `track` value of the output track's remote job.
const OUT_TRACK: &str = "out";
/// Transcribes call `id` and stores the merged transcript, ending in
/// `Done`. A call that already has a transcript (a retry after a later
/// step failed) is not transcribed again.
///
/// Both tracks are encoded to FLAC in `work_dir` (removed again afterwards)
/// and uploaded; the remote jobs are deleted once their results are in, or
/// the run failed. Any error sets the call to `Failed` with
/// [`PipelineError::user_message`]. The recordings themselves are never
/// deleted here.
pub async fn process_call(
store: &Store,
tr: &dyn Transcriber,
id: CallId,
opts: &PipelineOptions,
work_dir: &Path,
) -> Result<(), PipelineError> {
let result = run(store, tr, id, opts, work_dir).await;
if let Err(e) = &result {
log::warn!("call {id}: processing failed: {e}");
if let Err(se) = store.set_status(id, CallStatus::Failed, Some(&e.user_message())) {
log::error!("call {id}: could not mark as failed: {se}");
}
}
result
}
async fn run(
store: &Store,
tr: &dyn Transcriber,
id: CallId,
opts: &PipelineOptions,
work_dir: &Path,
) -> Result<(), PipelineError> {
let call = store.get_call(id)?.ok_or(PipelineError::CallNotFound(id))?;
if call.transcribed_at.is_none() && store.segments(id)?.is_empty() {
store.set_status(id, CallStatus::Processing, None)?;
transcribe(store, tr, &call, opts, work_dir).await?;
}
// SUMMARY SEAM (Task 14b): summarize here.
store.set_status(id, CallStatus::Done, None)?;
log::info!("call {id}: done");
Ok(())
}
/// Encodes, uploads and transcribes both tracks, then stores the merged
/// transcript.
async fn transcribe(
store: &Store,
tr: &dyn Transcriber,
call: &Call,
opts: &PipelineOptions,
work_dir: &Path,
) -> Result<(), PipelineError> {
let id = call.id;
fs::create_dir_all(work_dir).map_err(|source| PipelineError::Io {
path: work_dir.to_path_buf(),
source,
})?;
let mic_flac = work_dir.join(format!("{id}-mic.flac"));
let out_flac = work_dir.join(format!("{id}-out.flac"));
let _flac_files = RemoveOnDrop(vec![mic_flac.clone(), out_flac.clone()]);
tokio::try_join!(
encode(&call.mic_path, &mic_flac),
encode(&call.out_path, &out_flac)
)?;
log::info!("call {id}: uploading both tracks");
let track_opts = |single_speaker| TranscribeOptions {
language: opts.language.clone(),
model: opts.deepscript_model.clone(),
single_speaker,
};
let (mic_opts, out_opts) = (track_opts(true), track_opts(false));
let (mic_job, out_job) = tokio::join!(
submit(store, tr, id, MIC_TRACK, &mic_flac, &mic_opts),
submit(store, tr, id, OUT_TRACK, &out_flac, &out_opts)
);
let submitted: Vec<String> = [&mic_job, &out_job]
.into_iter()
.filter_map(|job| job.as_ref().ok().cloned())
.collect();
let words = wait_both(tr, id, mic_job, out_job).await;
// Also after a failed submit or wait: nothing stays at DeepScript.
delete_jobs(store, tr, id, &submitted).await;
let (mic, out) = words?;
let segments = merge(&mic, &out);
store.save_segments(id, &segments)?;
store.mark_transcribed(id, Local::now())?;
log::info!("call {id}: transcribed, {} segments", segments.len());
Ok(())
}
/// Waits for both jobs at once, once both were submitted.
async fn wait_both(
tr: &dyn Transcriber,
id: CallId,
mic_job: Result<String, PipelineError>,
out_job: Result<String, PipelineError>,
) -> Result<(Vec<Word>, Vec<Word>), PipelineError> {
let (mic_job, out_job) = (mic_job?, out_job?);
log::info!("call {id}: waiting for remote jobs {mic_job} and {out_job}");
let (mic, out) = tokio::join!(tr.wait(&mic_job), tr.wait(&out_job));
Ok((mic?, out?))
}
/// Encodes `wav` to FLAC at `flac` on the blocking pool.
async fn encode(wav: &Path, flac: &Path) -> Result<(), PipelineError> {
let (src, dst) = (wav.to_path_buf(), flac.to_path_buf());
tokio::task::spawn_blocking(move || encode_flac(&src, &dst))
.await
.unwrap_or_else(|e| Err(io::Error::other(e)))
.map_err(|source| PipelineError::Io {
path: wav.to_path_buf(),
source,
})
}
/// Uploads one track and records its remote job right away, so a crash
/// mid-wait still leaves a record of what to delete remotely.
async fn submit(
store: &Store,
tr: &dyn Transcriber,
id: CallId,
track: &str,
flac: &Path,
opts: &TranscribeOptions,
) -> Result<String, PipelineError> {
let remote_id = tr.submit(flac, opts).await?;
store.record_remote_job(id, track, &remote_id)?;
Ok(remote_id)
}
/// Best-effort deletes the remote jobs recorded for call `id` (left over
/// from a run the app did not finish), then forgets them.
pub async fn delete_remote_jobs(store: &Store, tr: &dyn Transcriber, id: CallId) {
delete_jobs(store, tr, id, &[]).await;
}
/// Deletes every remote job recorded for `id` plus `known` (jobs submitted
/// in this run, in case recording one failed). Forgets the recorded jobs
/// only if all of them were deleted, so a later run tries again.
async fn delete_jobs(store: &Store, tr: &dyn Transcriber, id: CallId, known: &[String]) {
let mut jobs: Vec<String> = match store.remote_jobs(id) {
Ok(jobs) => jobs.into_iter().map(|(_, remote_id)| remote_id).collect(),
Err(e) => {
log::error!("call {id}: could not read remote jobs: {e}");
Vec::new()
}
};
for job in known {
if !jobs.contains(job) {
jobs.push(job.clone());
}
}
let mut all_deleted = true;
for job in &jobs {
if let Err(e) = tr.delete(job).await {
all_deleted = false;
log::warn!(
"call {id}: deleting remote job {job} failed: {}",
e.user_message()
);
}
}
if all_deleted {
if let Err(e) = store.clear_remote_jobs(id) {
log::error!("call {id}: could not clear remote jobs: {e}");
}
}
}
/// Removes its files when dropped (missing files are fine).
struct RemoveOnDrop(Vec<PathBuf>);
impl Drop for RemoveOnDrop {
fn drop(&mut self) {
for path in &self.0 {
if let Err(e) = fs::remove_file(path) {
if e.kind() != io::ErrorKind::NotFound {
log::warn!("could not remove {}: {e}", path.display());
}
}
}
}
}
/// Reconciles call states left over from an unclean shutdown (see
/// [`Store::startup_recovery`]) and repairs the WAV headers of recordings
/// that were cut off. An interrupted call's `ended_at` becomes its start
/// plus the shorter track's length (a missing or unrepairable track counts
/// as empty).
///
/// The caller is responsible for deleting the remote jobs of the calls
/// reported as `failed` (see [`delete_remote_jobs`]).
pub fn recover_on_startup(store: &Store) -> Result<RecoveryReport, PipelineError> {
let report = store.startup_recovery()?;
for &id in &report.interrupted {
let Some(call) = store.get_call(id)? else {
continue;
};
let length = repaired_length(id, &call.mic_path).min(repaired_length(id, &call.out_path));
let length = TimeDelta::from_std(length).unwrap_or(TimeDelta::zero());
store.set_ended_at(id, call.started_at + length)?;
}
Ok(report)
}
/// Repairs one track's header and returns its length; zero if the file is
/// missing or cannot be repaired.
fn repaired_length(id: CallId, path: &Path) -> Duration {
match repair_wav(path) {
Ok(length) => length,
Err(e) => {
log::warn!("call {id}: could not repair {}: {e}", path.display());
Duration::ZERO
}
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::fs::{self, OpenOptions};
use std::io::{Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use async_trait::async_trait;
use chrono::{Duration, Local, TimeZone};
use super::*;
use crate::audio::TrackWriter;
use crate::merge::label;
use crate::model::{CallStatus, Segment, Speaker, Word};
use crate::transcribe::TranscribeOptions;
/// Scripted [`Transcriber`]: a file whose name contains `-mic.` gets the
/// job id `job-mic`, anything else `job-out`; `wait` returns the words
/// (or error) scripted for that job.
#[derive(Default)]
struct FakeTranscriber {
results: Mutex<HashMap<String, Result<Vec<Word>, TranscribeError>>>,
submitted: Mutex<Vec<(String, TranscribeOptions)>>,
deleted: Mutex<Vec<String>>,
/// Job whose upload fails with `Rejected`.
reject_upload: Option<&'static str>,
}
impl FakeTranscriber {
fn returning(mic: Vec<Word>, out: Vec<Word>) -> Self {
let fake = FakeTranscriber::default();
fake.script("job-mic", Ok(mic));
fake.script("job-out", Ok(out));
fake
}
fn script(&self, job: &str, result: Result<Vec<Word>, TranscribeError>) {
self.results.lock().unwrap().insert(job.to_string(), result);
}
fn submitted(&self) -> Vec<(String, TranscribeOptions)> {
let mut submitted = self.submitted.lock().unwrap().clone();
submitted.sort_by(|a, b| a.0.cmp(&b.0));
submitted
}
fn deleted(&self) -> Vec<String> {
let mut deleted = self.deleted.lock().unwrap().clone();
deleted.sort();
deleted
}
}
#[async_trait]
impl Transcriber for FakeTranscriber {
async fn submit(
&self,
audio: &Path,
opts: &TranscribeOptions,
) -> Result<String, TranscribeError> {
let bytes = fs::read(audio).expect("submitted file exists");
assert_eq!(&bytes[0..4], b"fLaC", "uploads are FLAC");
let name = audio.file_name().unwrap().to_string_lossy();
let job = if name.contains("-mic.") {
"job-mic"
} else {
"job-out"
};
if self.reject_upload == Some(job) {
return Err(TranscribeError::Rejected("unsupported-format".to_string()));
}
self.submitted
.lock()
.unwrap()
.push((job.to_string(), opts.clone()));
Ok(job.to_string())
}
async fn wait(&self, remote_id: &str) -> Result<Vec<Word>, TranscribeError> {
self.results
.lock()
.unwrap()
.get(remote_id)
.cloned()
.unwrap_or_else(|| panic!("no result scripted for {remote_id}"))
}
async fn delete(&self, remote_id: &str) -> Result<(), TranscribeError> {
self.deleted.lock().unwrap().push(remote_id.to_string());
Ok(())
}
}
fn word(text: &str, start: f64, end: f64, speaker: Option<&str>) -> Word {
Word {
text: text.to_string(),
start,
end,
speaker: speaker.map(str::to_string),
}
}
fn opts() -> PipelineOptions {
PipelineOptions {
language: "de".to_string(),
deepscript_model: "premium".to_string(),
}
}
/// Writes a mono 16 kHz WAV of `secs` seconds of a quiet tone.
fn write_wav(path: &Path, secs: f32) {
let mut writer = TrackWriter::create(path).unwrap();
let samples: Vec<f32> = (0..(16_000.0 * secs) as usize)
.map(|i| 0.1 * (i as f32 * 0.05).sin())
.collect();
writer.write(&samples).unwrap();
writer.finalize().unwrap();
}
struct Fixture {
store: Store,
dir: tempfile::TempDir,
id: CallId,
mic: PathBuf,
out: PathBuf,
}
impl Fixture {
fn work_dir(&self) -> PathBuf {
self.dir.path().join("upload")
}
async fn process(&self, tr: &dyn Transcriber) -> Result<(), PipelineError> {
process_call(&self.store, tr, self.id, &opts(), &self.work_dir()).await
}
fn status(&self) -> CallStatus {
self.store.get_call(self.id).unwrap().unwrap().status
}
/// Files left in the upload directory (it may not exist at all).
fn upload_files(&self) -> Vec<PathBuf> {
fs::read_dir(self.work_dir())
.map(|entries| entries.map(|e| e.unwrap().path()).collect())
.unwrap_or_default()
}
}
/// A call awaiting the decision, with one second of audio per track.
fn fixture() -> Fixture {
let store = Store::open_in_memory().unwrap();
let dir = tempfile::tempdir().unwrap();
let mic = dir.path().join("1-mic.wav");
let out = dir.path().join("1-out.wav");
write_wav(&mic, 1.0);
write_wav(&out, 1.0);
let started = Local::now() - Duration::minutes(5);
let id = store
.create_call("Microsoft Teams", started, &mic, &out)
.unwrap();
store.finish_recording(id, Local::now(), false).unwrap();
Fixture {
store,
dir,
id,
mic,
out,
}
}
fn transcript(f: &Fixture) -> Vec<String> {
let names = Default::default();
f.store
.segments(f.id)
.unwrap()
.iter()
.map(|s| format!("{}: {}", label(&s.speaker, &names), s.text))
.collect()
}
#[tokio::test]
async fn pipeline_happy_path_sets_done_and_stores_segments() {
let f = fixture();
let fake = FakeTranscriber::returning(
vec![word("Guten", 0.0, 0.4, None), word("Tag", 0.4, 0.8, None)],
vec![word("Hallo", 2.0, 2.5, Some("A"))],
);
f.process(&fake).await.unwrap();
let call = f.store.get_call(f.id).unwrap().unwrap();
assert_eq!(call.status, CallStatus::Done);
assert_eq!(call.error, None);
assert!(call.transcribed_at.is_some());
assert_eq!(
transcript(&f),
vec![
"Ich: Guten Tag".to_string(),
"Sprecher 1: Hallo".to_string()
]
);
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]);
assert_eq!(f.upload_files(), Vec::<PathBuf>::new());
}
#[tokio::test]
async fn pipeline_mic_submitted_as_single_speaker() {
let f = fixture();
let fake = FakeTranscriber::returning(vec![], vec![]);
f.process(&fake).await.unwrap();
let expected = |single_speaker| TranscribeOptions {
language: "de".to_string(),
model: "premium".to_string(),
single_speaker,
};
assert_eq!(
fake.submitted(),
vec![
("job-mic".to_string(), expected(true)),
("job-out".to_string(), expected(false)),
]
);
}
#[tokio::test]
async fn pipeline_transcription_failure_sets_failed_keeps_audio() {
let f = fixture();
let fake = FakeTranscriber::returning(vec![word("Hallo", 0.0, 0.5, None)], vec![]);
fake.script("job-out", Err(TranscribeError::InsufficientBalance));
let err = f.process(&fake).await.unwrap_err();
assert_eq!(err.user_message(), "DeepScript-Guthaben aufgebraucht.");
let call = f.store.get_call(f.id).unwrap().unwrap();
assert_eq!(call.status, CallStatus::Failed);
assert_eq!(call.error, Some(err.user_message()));
assert!(call.transcribed_at.is_none());
assert!(f.mic.exists());
assert!(f.out.exists());
assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]);
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
assert_eq!(f.store.segments(f.id).unwrap(), vec![]);
assert_eq!(f.upload_files(), Vec::<PathBuf>::new());
}
#[tokio::test]
async fn pipeline_failed_upload_deletes_the_other_job() {
let f = fixture();
let fake = FakeTranscriber {
reject_upload: Some("job-out"),
..FakeTranscriber::returning(vec![], vec![])
};
let err = f.process(&fake).await.unwrap_err();
assert_eq!(
err.user_message(),
"DeepScript hat die Datei abgelehnt (unsupported-format)."
);
assert_eq!(f.status(), CallStatus::Failed);
assert_eq!(fake.deleted(), vec!["job-mic"]);
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
}
#[tokio::test]
async fn leftover_remote_jobs_are_deleted_and_forgotten() {
let f = fixture();
f.store.record_remote_job(f.id, "mic", "old-mic").unwrap();
f.store.record_remote_job(f.id, "out", "old-out").unwrap();
let fake = FakeTranscriber::default();
delete_remote_jobs(&f.store, &fake, f.id).await;
assert_eq!(fake.deleted(), vec!["old-mic", "old-out"]);
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
}
#[tokio::test]
async fn pipeline_unreadable_audio_sets_failed() {
let f = fixture();
fs::remove_file(&f.out).unwrap();
let fake = FakeTranscriber::returning(vec![], vec![]);
let err = f.process(&fake).await.unwrap_err();
assert!(
err.user_message().starts_with("Audiodatei nicht lesbar: "),
"{}",
err.user_message()
);
assert_eq!(f.status(), CallStatus::Failed);
assert_eq!(fake.submitted(), vec![]);
assert_eq!(f.upload_files(), Vec::<PathBuf>::new());
}
#[tokio::test]
async fn pipeline_tolerates_empty_track() {
let f = fixture();
let fake = FakeTranscriber::returning(vec![word("Hallo", 0.0, 0.5, None)], vec![]);
f.process(&fake).await.unwrap();
assert_eq!(f.status(), CallStatus::Done);
assert_eq!(transcript(&f), vec!["Ich: Hallo".to_string()]);
}
#[tokio::test]
async fn pipeline_skips_transcription_when_segments_exist() {
let f = fixture();
let segments = vec![Segment {
speaker: Speaker::Me,
start: 0.0,
end: 1.0,
text: "Schon da".to_string(),
}];
f.store.save_segments(f.id, &segments).unwrap();
f.store
.set_status(f.id, CallStatus::Failed, Some("vorher"))
.unwrap();
let fake = FakeTranscriber::default();
f.process(&fake).await.unwrap();
let call = f.store.get_call(f.id).unwrap().unwrap();
assert_eq!(call.status, CallStatus::Done);
assert_eq!(call.error, None);
assert_eq!(fake.submitted(), vec![]);
assert_eq!(f.store.segments(f.id).unwrap(), segments);
}
/// Zeroes the RIFF and data size fields, as left by a recording killed
/// before its first header refresh.
fn zero_header_sizes(path: &Path) {
let mut file = OpenOptions::new().write(true).open(path).unwrap();
for offset in [4, 40] {
file.seek(SeekFrom::Start(offset)).unwrap();
file.write_all(&0u32.to_le_bytes()).unwrap();
}
}
fn wav_duration(path: &Path) -> f64 {
let reader = hound::WavReader::open(path).unwrap();
reader.duration() as f64 / reader.spec().sample_rate as f64
}
#[test]
fn startup_repairs_interrupted_recordings() {
let store = Store::open_in_memory().unwrap();
let dir = tempfile::tempdir().unwrap();
let mic = dir.path().join("1-mic.wav");
let out = dir.path().join("1-out.wav");
write_wav(&mic, 3.0);
write_wav(&out, 2.5);
zero_header_sizes(&mic);
zero_header_sizes(&out);
let started = Local.with_ymd_and_hms(2026, 11, 5, 10, 14, 0).unwrap();
let id = store
.create_call("Microsoft Teams", started, &mic, &out)
.unwrap();
let report = recover_on_startup(&store).unwrap();
assert_eq!(report.interrupted, vec![id]);
let call = store.get_call(id).unwrap().unwrap();
assert_eq!(call.status, CallStatus::AwaitingDecision);
assert!(call.interrupted);
assert_eq!(wav_duration(&mic), 3.0);
assert_eq!(wav_duration(&out), 2.5);
assert_eq!(call.ended_at, Some(started + Duration::milliseconds(2500)));
}
#[test]
fn startup_treats_a_missing_track_as_empty() {
let store = Store::open_in_memory().unwrap();
let dir = tempfile::tempdir().unwrap();
let mic = dir.path().join("1-mic.wav");
write_wav(&mic, 3.0);
let started = Local.with_ymd_and_hms(2026, 11, 5, 10, 14, 0).unwrap();
let id = store
.create_call("Manuell", started, &mic, &dir.path().join("1-out.wav"))
.unwrap();
recover_on_startup(&store).unwrap();
let call = store.get_call(id).unwrap().unwrap();
assert_eq!(call.status, CallStatus::AwaitingDecision);
assert_eq!(call.ended_at, Some(started));
}
}
+1 -1
View File
@@ -216,7 +216,7 @@ impl Transcriber for DeepScript {
"queued" | "processing" => {}
other => {
if !self.warned_unknown_status.swap(true, Ordering::Relaxed) {
eprintln!(
log::warn!(
"DeepScript: unbekannter Transkriptionsstatus '{other}', warte weiter"
);
}
+1
View File
@@ -21,6 +21,7 @@ tauri = { version = "2.12.5", features = ["tray-icon"] }
tauri-plugin-log = "2"
tauri-plugin-autostart = "2"
tauri-plugin-single-instance = "2"
tauri-plugin-notification = "2"
chrono = { version = "0.4", features = ["serde"] }
tempfile = "3"
recorder-core = { path = "../crates/core" }
+2 -1
View File
@@ -8,6 +8,7 @@
"prompt"
],
"permissions": [
"core:default"
"core:default",
"notification:default"
]
}
+24 -7
View File
@@ -16,6 +16,7 @@ use serde::Serialize;
use tauri::{AppHandle, Emitter, Manager, State};
use tauri_plugin_autostart::ManagerExt;
use crate::processing;
use crate::state::{AppState, CallBrief, DeviceLevels, Ended, PendingPrompt, RecordingState};
const LEVELS_INTERVAL: Duration = Duration::from_millis(100);
@@ -120,19 +121,35 @@ pub fn get_call_brief(state: State<'_, AppState>, id: CallId) -> CmdResult<CallB
#[tauri::command]
pub async fn discard_call(app: AppHandle, id: CallId) -> CmdResult<()> {
blocking(app, move |app| {
if app.state::<AppState>().discard_call(id)? == 0 {
set_window_visible(app, "prompt", false);
}
app.state::<AppState>().discard_call(id)?;
prompt_queue_changed(app);
Ok(())
})
.await
}
/// Placeholder until transcription exists; the prompt shows the error.
/// Transkribieren in the end-of-call prompt: starts processing in the
/// background and returns at once; the call leaves the prompt.
#[tauri::command]
pub fn transcribe_call(id: CallId) -> CmdResult<()> {
let _ = id;
Err("Transkription ist noch nicht verfügbar".to_string())
pub async fn transcribe_call(app: AppHandle, id: CallId) -> CmdResult<()> {
blocking(app, move |app| processing::start(app, id, false)).await
}
/// Processes a failed (or still undecided) call again.
#[tauri::command]
pub async fn retry_call(app: AppHandle, id: CallId) -> CmdResult<()> {
blocking(app, move |app| processing::start(app, id, true)).await
}
/// After a call left the prompt queue: hides the prompt if no call waits
/// any more, and tells it to reload (`prompt-changed`) otherwise.
pub fn prompt_queue_changed(app: &AppHandle) {
if app.state::<AppState>().prompts_waiting() == 0 {
set_window_visible(app, "prompt", false);
}
if let Err(e) = app.emit("prompt-changed", ()) {
log::error!("could not emit prompt-changed: {e}");
}
}
/// Starts a manual recording, shows the floating window and streams levels.
+6 -1
View File
@@ -1,8 +1,10 @@
mod commands;
mod detection;
mod processing;
mod state;
mod tray;
use recorder_core::pipeline;
use recorder_core::secrets::KeyringSecrets;
use recorder_core::store::Store;
use tauri::{Manager, WindowEvent};
@@ -28,6 +30,7 @@ pub fn run() {
MacosLauncher::LaunchAgent,
Some(vec![AUTOSTART_ARG]),
))
.plugin(tauri_plugin_notification::init())
.setup(|app| {
// Release builds log to a file in the app log directory (on
// Windows %LOCALAPPDATA%\eu.euvino.callrecorder\logs), debug
@@ -50,7 +53,7 @@ pub fn run() {
let config_dir = app.path().app_config_dir()?;
std::fs::create_dir_all(&data_dir)?;
let store = Store::open(&data_dir.join("calls.sqlite"))?;
let recovery = store.startup_recovery()?;
let recovery = pipeline::recover_on_startup(&store)?;
log::info!("startup recovery: {recovery:?}");
let state = AppState::new(
@@ -62,6 +65,7 @@ pub fn run() {
let autostart = state.settings().autostart;
app.manage(state);
commands::apply_autostart(app.handle(), autostart);
processing::delete_leftover_jobs(app.handle(), recovery.failed);
tray::create(app.handle())?;
detection::spawn(app.handle().clone());
@@ -93,6 +97,7 @@ pub fn run() {
commands::get_call_brief,
commands::discard_call,
commands::transcribe_call,
commands::retry_call,
])
.run(tauri::generate_context!())
.expect("error while building tauri application");
+167
View File
@@ -0,0 +1,167 @@
//! Runs the processing pipeline for calls in the background and tells the
//! user about it: a `call-updated` event on every status change and a
//! system notification when a run ends. Notifications never contain
//! transcript text, and the API key is never logged.
use recorder_core::model::{CallId, CallStatus};
use recorder_core::pipeline::{self, PipelineOptions};
use recorder_core::secrets::Service;
use recorder_core::transcribe::deepscript::DeepScript;
use recorder_core::transcribe::TranscribeError;
use serde::Serialize;
use tauri::{AppHandle, Emitter, Manager};
use tauri_plugin_notification::NotificationExt;
use crate::commands;
use crate::state::AppState;
/// Payload of the `call-updated` event: the call's status changed.
#[derive(Serialize, Clone)]
struct CallUpdated {
id: CallId,
}
/// A call reserved with [`AppState::claim_processing`]; released when
/// dropped, also if the run panics.
struct Claim {
app: AppHandle,
id: CallId,
}
impl Claim {
fn take(app: &AppHandle, id: CallId, retry: bool) -> Result<Self, String> {
app.state::<AppState>().claim_processing(id, retry)?;
Ok(Self {
app: app.clone(),
id,
})
}
}
impl Drop for Claim {
fn drop(&mut self) {
self.app.state::<AppState>().release_processing(self.id);
}
}
/// Starts processing call `id` in the background and returns at once; the
/// call leaves the end-of-call prompt. `retry` also allows failed calls.
/// Without a DeepScript key nothing starts: the call is marked failed and
/// the error returned. Reads the keyring, so call it off the UI thread.
pub fn start(app: &AppHandle, id: CallId, retry: bool) -> Result<(), String> {
let claim = Claim::take(app, id, retry)?;
let state = app.state::<AppState>();
let Some(key) = deepscript_key(&state) else {
let msg = TranscribeError::MissingKey.user_message();
if let Err(e) = state.store.set_status(id, CallStatus::Failed, Some(&msg)) {
log::error!("call {id}: could not mark as failed: {e}");
}
emit_updated(app, id);
return Err(msg);
};
let work_dir = app
.path()
.app_cache_dir()
.map_err(|e| format!("Cache-Ordner nicht gefunden: {e}"))?
.join("upload");
let settings = state.settings();
let opts = PipelineOptions {
language: settings.language,
deepscript_model: settings.deepscript_model,
};
// A fresh client per run, so a key changed in the settings applies to
// the next run.
let transcriber = DeepScript::new(state.transport.clone(), key);
// Shown as processing right away, before the run gets going.
state
.store
.set_status(id, CallStatus::Processing, None)
.map_err(|e| format!("Interner Fehler: {e}"))?;
emit_updated(app, id);
if state.remove_queued_prompt(id).is_some() {
commands::prompt_queue_changed(app);
}
log::info!("call {id}: processing started");
tauri::async_runtime::spawn(async move {
let app = &claim.app;
let state = app.state::<AppState>();
let result = pipeline::process_call(&state.store, &transcriber, id, &opts, &work_dir).await;
emit_updated(app, id);
notify(app, id, result.err().map(|e| e.user_message()));
});
Ok(())
}
/// Best-effort deletes the DeepScript jobs of calls whose processing the
/// last run of the app did not finish ([`RecoveryReport::failed`]), in the
/// background. Skipped without a key.
///
/// [`RecoveryReport::failed`]: recorder_core::store::RecoveryReport::failed
pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec<CallId>) {
if calls.is_empty() {
return;
}
let app = app.clone();
tauri::async_runtime::spawn(async move {
let key = {
let app = app.clone();
tauri::async_runtime::spawn_blocking(move || deepscript_key(&app.state())).await
};
let Ok(Some(key)) = key else {
log::info!("no DeepScript key: remote jobs of calls {calls:?} stay for now");
return;
};
let state = app.state::<AppState>();
let transcriber = DeepScript::new(state.transport.clone(), key);
for id in calls {
// A retry the user started meanwhile cleans up itself.
let Ok(_claim) = Claim::take(&app, id, true) else {
continue;
};
pipeline::delete_remote_jobs(&state.store, &transcriber, id).await;
}
});
}
fn deepscript_key(state: &AppState) -> Option<String> {
state
.secrets
.get(Service::DeepScript)
.filter(|key| !key.trim().is_empty())
}
fn emit_updated(app: &AppHandle, id: CallId) {
if let Err(e) = app.emit("call-updated", CallUpdated { id }) {
log::error!("could not emit call-updated: {e}");
}
}
/// "Transkript bereit" with "<app> · <N> min", or the failure's message.
fn notify(app: &AppHandle, id: CallId, error: Option<String>) {
let (title, body) = match error {
None => ("Transkript bereit", call_summary_line(app, id)),
Some(msg) => ("Transkription fehlgeschlagen", msg),
};
let result = app.notification().builder().title(title).body(body).show();
if let Err(e) = result {
log::warn!("call {id}: could not show the notification: {e}");
}
}
/// "Microsoft Teams · 32 min" (whole minutes, at least 1, like the UI).
fn call_summary_line(app: &AppHandle, id: CallId) -> String {
match app.state::<AppState>().call_brief(id) {
Ok(brief) => format!(
"{} · {} min",
brief.app,
((brief.duration_secs as f64 / 60.0).round() as i64).max(1)
),
Err(e) => {
log::error!("call {id}: could not load it for the notification: {e}");
String::new()
}
}
}
+109 -5
View File
@@ -1,22 +1,24 @@
//! Application state shared by commands, the tray and background threads.
//!
//! Holds the store, the settings, the secret store, the recording that is
//! currently running, the call detector and the call awaiting a decision in
//! the end-of-call prompt. Everything here is independent of windows and
//! Holds the store, the settings, the secret store, the HTTP transport, the
//! recording that is currently running, the call detector, the calls
//! awaiting a decision in the end-of-call prompt and the calls being
//! processed. Everything here is independent of windows and
//! events; the command layer turns results into UI updates.
use std::collections::VecDeque;
use std::collections::{HashSet, VecDeque};
use std::fs;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Mutex, MutexGuard};
use std::sync::{Arc, Mutex, MutexGuard};
use std::thread;
use std::time::{Duration, Instant};
use chrono::{DateTime, Local};
use recorder_core::capture::{CaptureSource, OutputTarget};
use recorder_core::detect::{CallDetector, DetectorEvent, MicSession};
use recorder_core::http::{HttpTransport, ReqwestTransport};
use recorder_core::model::{Call, CallId, CallStatus};
use recorder_core::recorder::Recorder;
use recorder_core::secrets::SecretStore;
@@ -31,6 +33,7 @@ const MANUAL_APP: &str = "Manuell";
/// How long [`AppState::test_devices`] records.
const TEST_DURATION: Duration = Duration::from_secs(5);
const TEST_POLL: Duration = Duration::from_millis(50);
const ALREADY_PROCESSING: &str = "Wird bereits verarbeitet";
/// Payload of the `recording-state` event.
#[derive(Serialize, Clone, Debug)]
@@ -111,6 +114,8 @@ struct CurrentRecording {
pub struct AppState {
pub store: Store,
pub secrets: Box<dyn SecretStore>,
/// The one HTTP client all API calls share.
pub transport: Arc<dyn HttpTransport>,
settings: Mutex<Settings>,
settings_path: PathBuf,
recordings_dir: PathBuf,
@@ -120,6 +125,8 @@ pub struct AppState {
/// first. A call leaves the queue when it is decided (discarded or sent
/// to transcription).
pending_prompts: Mutex<VecDeque<CallId>>,
/// Calls a pipeline run is under way for; at most one run per call.
processing: Mutex<HashSet<CallId>>,
/// Set while a recording starts or the devices are tested, so neither
/// runs twice and `current` is never locked for longer than a moment.
devices_busy: AtomicBool,
@@ -146,12 +153,14 @@ impl AppState {
Self {
store,
secrets,
transport: Arc::new(ReqwestTransport::new()),
settings: Mutex::new(settings),
settings_path,
recordings_dir,
current: Mutex::new(None),
detector: Mutex::new(detector),
pending_prompts: Mutex::new(VecDeque::new()),
processing: Mutex::new(HashSet::new()),
devices_busy: AtomicBool::new(false),
}
}
@@ -432,6 +441,40 @@ impl AppState {
Ok(self.prompts_waiting())
}
/// Reserves `id` for one pipeline run; release it with
/// [`AppState::release_processing`] when the run ends. A call that waits
/// for the decision can be processed; a failed one only on `retry`, or
/// while the prompt still shows it (its first attempt failed before
/// anything started, e.g. for lack of an API key).
pub fn claim_processing(&self, id: CallId, retry: bool) -> Result<(), String> {
let mut processing = lock(&self.processing);
if processing.contains(&id) {
return Err(ALREADY_PROCESSING.to_string());
}
let call = self
.store
.get_call(id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "Gespräch nicht gefunden".to_string())?;
match call.status {
CallStatus::AwaitingDecision => {}
CallStatus::Failed if retry || lock(&self.pending_prompts).contains(&id) => {}
CallStatus::Processing => return Err(ALREADY_PROCESSING.to_string()),
CallStatus::Recording => return Err("Das Gespräch wird gerade aufgenommen".to_string()),
CallStatus::Done => return Err("Das Gespräch ist bereits transkribiert".to_string()),
CallStatus::Failed => {
return Err("Das Gespräch wartet nicht auf eine Entscheidung".to_string())
}
}
processing.insert(id);
Ok(())
}
/// Ends the pipeline run claimed by [`AppState::claim_processing`].
pub fn release_processing(&self, id: CallId) {
lock(&self.processing).remove(&id);
}
/// Records both configured devices into a temporary directory for five
/// seconds and returns the peak level of each track.
pub fn test_devices(&self) -> Result<DeviceLevels, String> {
@@ -664,6 +707,67 @@ mod tests {
assert_eq!(f.state.prompts_waiting(), 0);
}
#[test]
fn a_call_is_processed_once_at_a_time() {
let f = fixture();
let (id, _, _) = finished_call(&f, "a");
assert_eq!(f.state.claim_processing(id, false), Ok(()));
assert_eq!(
f.state.claim_processing(id, true),
Err("Wird bereits verarbeitet".to_string())
);
f.state.release_processing(id);
assert_eq!(f.state.claim_processing(id, false), Ok(()));
}
#[test]
fn a_processing_call_is_not_claimed_again() {
let f = fixture();
let (id, _, _) = finished_call(&f, "a");
f.state
.store
.set_status(id, CallStatus::Processing, None)
.unwrap();
assert_eq!(
f.state.claim_processing(id, true),
Err("Wird bereits verarbeitet".to_string())
);
}
#[test]
fn a_failed_call_is_processed_again_on_retry_or_from_the_prompt() {
let f = fixture();
let (id, _, _) = finished_call(&f, "a");
f.state
.store
.set_status(id, CallStatus::Failed, Some("Fehler"))
.unwrap();
assert!(f.state.claim_processing(id, false).is_err());
assert_eq!(f.state.claim_processing(id, true), Ok(()));
f.state.release_processing(id);
f.state.queue_prompt(id);
assert_eq!(f.state.claim_processing(id, false), Ok(()));
}
#[test]
fn done_and_recording_calls_are_not_processed() {
let f = fixture();
let (done, _, _) = finished_call(&f, "a");
f.state
.store
.set_status(done, CallStatus::Done, None)
.unwrap();
assert!(f.state.claim_processing(done, true).is_err());
let recording = f
.state
.store
.create_call("Manuell", at(9, 0, 0), Path::new(""), Path::new(""))
.unwrap();
assert!(f.state.claim_processing(recording, true).is_err());
assert!(f.state.claim_processing(recording + 1, true).is_err());
}
#[test]
fn invalid_detection_delays_are_not_saved() {
let f = fixture();
+7
View File
@@ -74,12 +74,19 @@ export const discardCurrentRecording = () => invoke<void>('discard_current_recor
export const getPendingPrompt = () => invoke<PendingPrompt | null>('get_pending_prompt')
export const getCallBrief = (id: CallId) => invoke<CallBrief>('get_call_brief', { id })
export const transcribeCall = (id: CallId) => invoke<void>('transcribe_call', { id })
export const retryCall = (id: CallId) => invoke<void>('retry_call', { id })
export const discardCall = (id: CallId) => invoke<void>('discard_call', { id })
export const onRecordingState = (handler: (state: RecordingState) => void): Promise<UnlistenFn> =>
listen<RecordingState>('recording-state', (event) => handler(event.payload))
export const onCallEnded = (handler: (id: CallId) => void): Promise<UnlistenFn> =>
listen<{ id: CallId }>('call-ended', (event) => handler(event.payload.id))
/** The prompt queue changed (a call was decided elsewhere). */
export const onPromptChanged = (handler: () => void): Promise<UnlistenFn> =>
listen<null>('prompt-changed', () => handler())
/** A call's status changed (processing started, done or failed). */
export const onCallUpdated = (handler: (id: CallId) => void): Promise<UnlistenFn> =>
listen<{ id: CallId }>('call-updated', (event) => handler(event.payload.id))
export const onLevels = (handler: (levels: Levels) => void): Promise<UnlistenFn> =>
listen<Levels>('levels', (event) => handler(event.payload))
+3 -2
View File
@@ -5,6 +5,7 @@
errorText,
getPendingPrompt,
onCallEnded,
onPromptChanged,
transcribeCall,
type CallId,
type PendingPrompt,
@@ -29,12 +30,12 @@
.catch((err) => (error = errorText(err)))
onMount(() => {
const unlisten = onCallEnded(refresh)
const unlisten = [onCallEnded(refresh), onPromptChanged(refresh)]
refresh()
window.addEventListener('focus', refresh)
return () => {
window.removeEventListener('focus', refresh)
unlisten.then((off) => off())
unlisten.forEach((u) => u.then((off) => off()))
}
})