From e2828791daa16b2a6a1d6733291e2cf3a46fbd05 Mon Sep 17 00:00:00 2001 From: hauke Date: Thu, 5 Nov 2026 16:41:53 +0100 Subject: [PATCH] feat #14740: transcription pipeline Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 61 ++ crates/core/Cargo.toml | 1 + crates/core/src/lib.rs | 1 + crates/core/src/pipeline.rs | 683 +++++++++++++++++++++++ crates/core/src/transcribe/deepscript.rs | 2 +- src-tauri/Cargo.toml | 1 + src-tauri/capabilities/default.json | 3 +- src-tauri/src/commands.rs | 31 +- src-tauri/src/lib.rs | 7 +- src-tauri/src/processing.rs | 167 ++++++ src-tauri/src/state.rs | 114 +++- ui/src/lib/api.ts | 7 + ui/src/routes/Prompt.svelte | 5 +- 13 files changed, 1066 insertions(+), 17 deletions(-) create mode 100644 crates/core/src/pipeline.rs create mode 100644 src-tauri/src/processing.rs diff --git a/Cargo.lock b/Cargo.lock index d3ab70c..ad7c7e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,6 +457,7 @@ dependencies = [ "tauri-build", "tauri-plugin-autostart", "tauri-plugin-log", + "tauri-plugin-notification", "tauri-plugin-single-instance", "tempfile", ] @@ -2369,6 +2370,20 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +[[package]] +name = "mac-notification-sys" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd604973958ddcc11b561193c0fb96ba146506ef2f231ef2e7c35fd2cbc9beca" +dependencies = [ + "cc", + "log", + "objc2", + "objc2-foundation", + "time", + "uuid", +] + [[package]] name = "markup5ever" version = "0.39.0" @@ -2534,6 +2549,20 @@ dependencies = [ "libc", ] +[[package]] +name = "notify-rust" +version = "4.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4587364a9a0074333429b3df75a30a205340c56a536ca3eb6ca0e59b87bbf8af" +dependencies = [ + "futures-lite", + "log", + "mac-notification-sys", + "serde", + "tauri-winrt-notification", + "zbus 5.19.0", +] + [[package]] name = "num" version = "0.4.3" @@ -3382,6 +3411,7 @@ dependencies = [ "flacenc", "hound", "keyring", + "log", "reqwest 0.12.28", "rubato", "rusqlite", @@ -4447,6 +4477,26 @@ dependencies = [ "time", ] +[[package]] +name = "tauri-plugin-notification" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc6c194a95a535abdfbe3bc1af9a43935b8b9b29c216b8e71af8b59132703cf" +dependencies = [ + "log", + "notify-rust", + "rand 0.10.3", + "serde", + "serde_json", + "serde_repr", + "tauri", + "tauri-plugin", + "thiserror 2.0.21", + "time", + "url", + "zbus 5.19.0", +] + [[package]] name = "tauri-plugin-single-instance" version = "2.5.5" @@ -4562,6 +4612,17 @@ dependencies = [ "toml 1.2.0+spec-1.1.0", ] +[[package]] +name = "tauri-winrt-notification" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37a6c354fd28fc9e322ed9bd47e3959576dad28c9d58ea1cf888cce1c7ccb36" +dependencies = [ + "thiserror 2.0.21", + "windows", + "windows-version", +] + [[package]] name = "tempfile" version = "3.27.0" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 8f6fc62..5f66713 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -23,6 +23,7 @@ keyring = { version = "3", features = ["windows-native", "sync-secret-service", async-trait = "0.1" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "multipart", "json", "stream"] } tokio = { version = "1", features = ["rt", "fs", "time", "macros"] } +log = "0.4" [dev-dependencies] tempfile = "3" diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 087fad1..415b7eb 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -10,6 +10,7 @@ pub mod detect; pub mod http; pub mod merge; pub mod model; +pub mod pipeline; pub mod recorder; pub mod retention; pub mod secrets; diff --git a/crates/core/src/pipeline.rs b/crates/core/src/pipeline.rs new file mode 100644 index 0000000..29a7cec --- /dev/null +++ b/crates/core/src/pipeline.rs @@ -0,0 +1,683 @@ +//! Turns a recorded call into a stored transcript: encode both tracks to +//! FLAC, transcribe them remotely, merge the results and store them. Also +//! the startup routine that repairs recordings cut off by a crash. +//! +//! The [`Store`] is shared with the rest of the app; it locks its connection +//! only inside each call, so no lock is ever held across an `.await`. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use chrono::{Local, TimeDelta}; +use thiserror::Error; + +use crate::audio::{encode_flac, repair_wav}; +use crate::merge::merge; +use crate::model::{Call, CallId, CallStatus, Word}; +use crate::store::{RecoveryReport, Store, StoreError}; +use crate::transcribe::{TranscribeError, TranscribeOptions, Transcriber}; + +/// Settings a pipeline run uses. +#[derive(Debug, Clone, PartialEq)] +pub struct PipelineOptions { + /// ISO 639-1 code, or `"auto"`. + pub language: String, + pub deepscript_model: String, +} + +/// Everything a pipeline run can fail with. +#[derive(Debug, Error)] +pub enum PipelineError { + #[error("transcription failed: {0:?}")] + Transcribe(TranscribeError), + #[error(transparent)] + Store(#[from] StoreError), + #[error("cannot read audio file {}: {source}", path.display())] + Io { path: PathBuf, source: io::Error }, + #[error("call {0} not found")] + CallNotFound(CallId), +} + +impl From for PipelineError { + fn from(e: TranscribeError) -> Self { + PipelineError::Transcribe(e) + } +} + +impl PipelineError { + /// German, user-facing text for this error. Never includes an API key + /// or transcript text. + pub fn user_message(&self) -> String { + match self { + PipelineError::Transcribe(e) => e.user_message(), + PipelineError::Store(e) => format!("Interner Fehler: {e}"), + PipelineError::Io { path, source } => { + format!("Audiodatei nicht lesbar: {} ({source})", path.display()) + } + PipelineError::CallNotFound(_) => "Gespräch nicht gefunden".to_string(), + } + } +} + +/// `track` value of the mic track's remote job. +const MIC_TRACK: &str = "mic"; +/// `track` value of the output track's remote job. +const OUT_TRACK: &str = "out"; + +/// Transcribes call `id` and stores the merged transcript, ending in +/// `Done`. A call that already has a transcript (a retry after a later +/// step failed) is not transcribed again. +/// +/// Both tracks are encoded to FLAC in `work_dir` (removed again afterwards) +/// and uploaded; the remote jobs are deleted once their results are in, or +/// the run failed. Any error sets the call to `Failed` with +/// [`PipelineError::user_message`]. The recordings themselves are never +/// deleted here. +pub async fn process_call( + store: &Store, + tr: &dyn Transcriber, + id: CallId, + opts: &PipelineOptions, + work_dir: &Path, +) -> Result<(), PipelineError> { + let result = run(store, tr, id, opts, work_dir).await; + if let Err(e) = &result { + log::warn!("call {id}: processing failed: {e}"); + if let Err(se) = store.set_status(id, CallStatus::Failed, Some(&e.user_message())) { + log::error!("call {id}: could not mark as failed: {se}"); + } + } + result +} + +async fn run( + store: &Store, + tr: &dyn Transcriber, + id: CallId, + opts: &PipelineOptions, + work_dir: &Path, +) -> Result<(), PipelineError> { + let call = store.get_call(id)?.ok_or(PipelineError::CallNotFound(id))?; + if call.transcribed_at.is_none() && store.segments(id)?.is_empty() { + store.set_status(id, CallStatus::Processing, None)?; + transcribe(store, tr, &call, opts, work_dir).await?; + } + // SUMMARY SEAM (Task 14b): summarize here. + store.set_status(id, CallStatus::Done, None)?; + log::info!("call {id}: done"); + Ok(()) +} + +/// Encodes, uploads and transcribes both tracks, then stores the merged +/// transcript. +async fn transcribe( + store: &Store, + tr: &dyn Transcriber, + call: &Call, + opts: &PipelineOptions, + work_dir: &Path, +) -> Result<(), PipelineError> { + let id = call.id; + fs::create_dir_all(work_dir).map_err(|source| PipelineError::Io { + path: work_dir.to_path_buf(), + source, + })?; + let mic_flac = work_dir.join(format!("{id}-mic.flac")); + let out_flac = work_dir.join(format!("{id}-out.flac")); + let _flac_files = RemoveOnDrop(vec![mic_flac.clone(), out_flac.clone()]); + + tokio::try_join!( + encode(&call.mic_path, &mic_flac), + encode(&call.out_path, &out_flac) + )?; + log::info!("call {id}: uploading both tracks"); + + let track_opts = |single_speaker| TranscribeOptions { + language: opts.language.clone(), + model: opts.deepscript_model.clone(), + single_speaker, + }; + let (mic_opts, out_opts) = (track_opts(true), track_opts(false)); + let (mic_job, out_job) = tokio::join!( + submit(store, tr, id, MIC_TRACK, &mic_flac, &mic_opts), + submit(store, tr, id, OUT_TRACK, &out_flac, &out_opts) + ); + let submitted: Vec = [&mic_job, &out_job] + .into_iter() + .filter_map(|job| job.as_ref().ok().cloned()) + .collect(); + let words = wait_both(tr, id, mic_job, out_job).await; + // Also after a failed submit or wait: nothing stays at DeepScript. + delete_jobs(store, tr, id, &submitted).await; + let (mic, out) = words?; + + let segments = merge(&mic, &out); + store.save_segments(id, &segments)?; + store.mark_transcribed(id, Local::now())?; + log::info!("call {id}: transcribed, {} segments", segments.len()); + Ok(()) +} + +/// Waits for both jobs at once, once both were submitted. +async fn wait_both( + tr: &dyn Transcriber, + id: CallId, + mic_job: Result, + out_job: Result, +) -> Result<(Vec, Vec), PipelineError> { + let (mic_job, out_job) = (mic_job?, out_job?); + log::info!("call {id}: waiting for remote jobs {mic_job} and {out_job}"); + let (mic, out) = tokio::join!(tr.wait(&mic_job), tr.wait(&out_job)); + Ok((mic?, out?)) +} + +/// Encodes `wav` to FLAC at `flac` on the blocking pool. +async fn encode(wav: &Path, flac: &Path) -> Result<(), PipelineError> { + let (src, dst) = (wav.to_path_buf(), flac.to_path_buf()); + tokio::task::spawn_blocking(move || encode_flac(&src, &dst)) + .await + .unwrap_or_else(|e| Err(io::Error::other(e))) + .map_err(|source| PipelineError::Io { + path: wav.to_path_buf(), + source, + }) +} + +/// Uploads one track and records its remote job right away, so a crash +/// mid-wait still leaves a record of what to delete remotely. +async fn submit( + store: &Store, + tr: &dyn Transcriber, + id: CallId, + track: &str, + flac: &Path, + opts: &TranscribeOptions, +) -> Result { + let remote_id = tr.submit(flac, opts).await?; + store.record_remote_job(id, track, &remote_id)?; + Ok(remote_id) +} + +/// Best-effort deletes the remote jobs recorded for call `id` (left over +/// from a run the app did not finish), then forgets them. +pub async fn delete_remote_jobs(store: &Store, tr: &dyn Transcriber, id: CallId) { + delete_jobs(store, tr, id, &[]).await; +} + +/// Deletes every remote job recorded for `id` plus `known` (jobs submitted +/// in this run, in case recording one failed). Forgets the recorded jobs +/// only if all of them were deleted, so a later run tries again. +async fn delete_jobs(store: &Store, tr: &dyn Transcriber, id: CallId, known: &[String]) { + let mut jobs: Vec = match store.remote_jobs(id) { + Ok(jobs) => jobs.into_iter().map(|(_, remote_id)| remote_id).collect(), + Err(e) => { + log::error!("call {id}: could not read remote jobs: {e}"); + Vec::new() + } + }; + for job in known { + if !jobs.contains(job) { + jobs.push(job.clone()); + } + } + + let mut all_deleted = true; + for job in &jobs { + if let Err(e) = tr.delete(job).await { + all_deleted = false; + log::warn!( + "call {id}: deleting remote job {job} failed: {}", + e.user_message() + ); + } + } + if all_deleted { + if let Err(e) = store.clear_remote_jobs(id) { + log::error!("call {id}: could not clear remote jobs: {e}"); + } + } +} + +/// Removes its files when dropped (missing files are fine). +struct RemoveOnDrop(Vec); + +impl Drop for RemoveOnDrop { + fn drop(&mut self) { + for path in &self.0 { + if let Err(e) = fs::remove_file(path) { + if e.kind() != io::ErrorKind::NotFound { + log::warn!("could not remove {}: {e}", path.display()); + } + } + } + } +} + +/// Reconciles call states left over from an unclean shutdown (see +/// [`Store::startup_recovery`]) and repairs the WAV headers of recordings +/// that were cut off. An interrupted call's `ended_at` becomes its start +/// plus the shorter track's length (a missing or unrepairable track counts +/// as empty). +/// +/// The caller is responsible for deleting the remote jobs of the calls +/// reported as `failed` (see [`delete_remote_jobs`]). +pub fn recover_on_startup(store: &Store) -> Result { + let report = store.startup_recovery()?; + for &id in &report.interrupted { + let Some(call) = store.get_call(id)? else { + continue; + }; + let length = repaired_length(id, &call.mic_path).min(repaired_length(id, &call.out_path)); + let length = TimeDelta::from_std(length).unwrap_or(TimeDelta::zero()); + store.set_ended_at(id, call.started_at + length)?; + } + Ok(report) +} + +/// Repairs one track's header and returns its length; zero if the file is +/// missing or cannot be repaired. +fn repaired_length(id: CallId, path: &Path) -> Duration { + match repair_wav(path) { + Ok(length) => length, + Err(e) => { + log::warn!("call {id}: could not repair {}: {e}", path.display()); + Duration::ZERO + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::fs::{self, OpenOptions}; + use std::io::{Seek, SeekFrom, Write}; + use std::path::{Path, PathBuf}; + use std::sync::Mutex; + + use async_trait::async_trait; + use chrono::{Duration, Local, TimeZone}; + + use super::*; + use crate::audio::TrackWriter; + use crate::merge::label; + use crate::model::{CallStatus, Segment, Speaker, Word}; + use crate::transcribe::TranscribeOptions; + + /// Scripted [`Transcriber`]: a file whose name contains `-mic.` gets the + /// job id `job-mic`, anything else `job-out`; `wait` returns the words + /// (or error) scripted for that job. + #[derive(Default)] + struct FakeTranscriber { + results: Mutex, TranscribeError>>>, + submitted: Mutex>, + deleted: Mutex>, + /// Job whose upload fails with `Rejected`. + reject_upload: Option<&'static str>, + } + + impl FakeTranscriber { + fn returning(mic: Vec, out: Vec) -> Self { + let fake = FakeTranscriber::default(); + fake.script("job-mic", Ok(mic)); + fake.script("job-out", Ok(out)); + fake + } + + fn script(&self, job: &str, result: Result, TranscribeError>) { + self.results.lock().unwrap().insert(job.to_string(), result); + } + + fn submitted(&self) -> Vec<(String, TranscribeOptions)> { + let mut submitted = self.submitted.lock().unwrap().clone(); + submitted.sort_by(|a, b| a.0.cmp(&b.0)); + submitted + } + + fn deleted(&self) -> Vec { + let mut deleted = self.deleted.lock().unwrap().clone(); + deleted.sort(); + deleted + } + } + + #[async_trait] + impl Transcriber for FakeTranscriber { + async fn submit( + &self, + audio: &Path, + opts: &TranscribeOptions, + ) -> Result { + let bytes = fs::read(audio).expect("submitted file exists"); + assert_eq!(&bytes[0..4], b"fLaC", "uploads are FLAC"); + let name = audio.file_name().unwrap().to_string_lossy(); + let job = if name.contains("-mic.") { + "job-mic" + } else { + "job-out" + }; + if self.reject_upload == Some(job) { + return Err(TranscribeError::Rejected("unsupported-format".to_string())); + } + self.submitted + .lock() + .unwrap() + .push((job.to_string(), opts.clone())); + Ok(job.to_string()) + } + + async fn wait(&self, remote_id: &str) -> Result, TranscribeError> { + self.results + .lock() + .unwrap() + .get(remote_id) + .cloned() + .unwrap_or_else(|| panic!("no result scripted for {remote_id}")) + } + + async fn delete(&self, remote_id: &str) -> Result<(), TranscribeError> { + self.deleted.lock().unwrap().push(remote_id.to_string()); + Ok(()) + } + } + + fn word(text: &str, start: f64, end: f64, speaker: Option<&str>) -> Word { + Word { + text: text.to_string(), + start, + end, + speaker: speaker.map(str::to_string), + } + } + + fn opts() -> PipelineOptions { + PipelineOptions { + language: "de".to_string(), + deepscript_model: "premium".to_string(), + } + } + + /// Writes a mono 16 kHz WAV of `secs` seconds of a quiet tone. + fn write_wav(path: &Path, secs: f32) { + let mut writer = TrackWriter::create(path).unwrap(); + let samples: Vec = (0..(16_000.0 * secs) as usize) + .map(|i| 0.1 * (i as f32 * 0.05).sin()) + .collect(); + writer.write(&samples).unwrap(); + writer.finalize().unwrap(); + } + + struct Fixture { + store: Store, + dir: tempfile::TempDir, + id: CallId, + mic: PathBuf, + out: PathBuf, + } + + impl Fixture { + fn work_dir(&self) -> PathBuf { + self.dir.path().join("upload") + } + + async fn process(&self, tr: &dyn Transcriber) -> Result<(), PipelineError> { + process_call(&self.store, tr, self.id, &opts(), &self.work_dir()).await + } + + fn status(&self) -> CallStatus { + self.store.get_call(self.id).unwrap().unwrap().status + } + + /// Files left in the upload directory (it may not exist at all). + fn upload_files(&self) -> Vec { + fs::read_dir(self.work_dir()) + .map(|entries| entries.map(|e| e.unwrap().path()).collect()) + .unwrap_or_default() + } + } + + /// A call awaiting the decision, with one second of audio per track. + fn fixture() -> Fixture { + let store = Store::open_in_memory().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mic = dir.path().join("1-mic.wav"); + let out = dir.path().join("1-out.wav"); + write_wav(&mic, 1.0); + write_wav(&out, 1.0); + let started = Local::now() - Duration::minutes(5); + let id = store + .create_call("Microsoft Teams", started, &mic, &out) + .unwrap(); + store.finish_recording(id, Local::now(), false).unwrap(); + Fixture { + store, + dir, + id, + mic, + out, + } + } + + fn transcript(f: &Fixture) -> Vec { + let names = Default::default(); + f.store + .segments(f.id) + .unwrap() + .iter() + .map(|s| format!("{}: {}", label(&s.speaker, &names), s.text)) + .collect() + } + + #[tokio::test] + async fn pipeline_happy_path_sets_done_and_stores_segments() { + let f = fixture(); + let fake = FakeTranscriber::returning( + vec![word("Guten", 0.0, 0.4, None), word("Tag", 0.4, 0.8, None)], + vec![word("Hallo", 2.0, 2.5, Some("A"))], + ); + + f.process(&fake).await.unwrap(); + + let call = f.store.get_call(f.id).unwrap().unwrap(); + assert_eq!(call.status, CallStatus::Done); + assert_eq!(call.error, None); + assert!(call.transcribed_at.is_some()); + assert_eq!( + transcript(&f), + vec![ + "Ich: Guten Tag".to_string(), + "Sprecher 1: Hallo".to_string() + ] + ); + assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]); + assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]); + assert_eq!(f.upload_files(), Vec::::new()); + } + + #[tokio::test] + async fn pipeline_mic_submitted_as_single_speaker() { + let f = fixture(); + let fake = FakeTranscriber::returning(vec![], vec![]); + + f.process(&fake).await.unwrap(); + + let expected = |single_speaker| TranscribeOptions { + language: "de".to_string(), + model: "premium".to_string(), + single_speaker, + }; + assert_eq!( + fake.submitted(), + vec![ + ("job-mic".to_string(), expected(true)), + ("job-out".to_string(), expected(false)), + ] + ); + } + + #[tokio::test] + async fn pipeline_transcription_failure_sets_failed_keeps_audio() { + let f = fixture(); + let fake = FakeTranscriber::returning(vec![word("Hallo", 0.0, 0.5, None)], vec![]); + fake.script("job-out", Err(TranscribeError::InsufficientBalance)); + + let err = f.process(&fake).await.unwrap_err(); + + assert_eq!(err.user_message(), "DeepScript-Guthaben aufgebraucht."); + let call = f.store.get_call(f.id).unwrap().unwrap(); + assert_eq!(call.status, CallStatus::Failed); + assert_eq!(call.error, Some(err.user_message())); + assert!(call.transcribed_at.is_none()); + assert!(f.mic.exists()); + assert!(f.out.exists()); + assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]); + assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]); + assert_eq!(f.store.segments(f.id).unwrap(), vec![]); + assert_eq!(f.upload_files(), Vec::::new()); + } + + #[tokio::test] + async fn pipeline_failed_upload_deletes_the_other_job() { + let f = fixture(); + let fake = FakeTranscriber { + reject_upload: Some("job-out"), + ..FakeTranscriber::returning(vec![], vec![]) + }; + + let err = f.process(&fake).await.unwrap_err(); + + assert_eq!( + err.user_message(), + "DeepScript hat die Datei abgelehnt (unsupported-format)." + ); + assert_eq!(f.status(), CallStatus::Failed); + assert_eq!(fake.deleted(), vec!["job-mic"]); + assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]); + } + + #[tokio::test] + async fn leftover_remote_jobs_are_deleted_and_forgotten() { + let f = fixture(); + f.store.record_remote_job(f.id, "mic", "old-mic").unwrap(); + f.store.record_remote_job(f.id, "out", "old-out").unwrap(); + let fake = FakeTranscriber::default(); + + delete_remote_jobs(&f.store, &fake, f.id).await; + + assert_eq!(fake.deleted(), vec!["old-mic", "old-out"]); + assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]); + } + + #[tokio::test] + async fn pipeline_unreadable_audio_sets_failed() { + let f = fixture(); + fs::remove_file(&f.out).unwrap(); + let fake = FakeTranscriber::returning(vec![], vec![]); + + let err = f.process(&fake).await.unwrap_err(); + + assert!( + err.user_message().starts_with("Audiodatei nicht lesbar: "), + "{}", + err.user_message() + ); + assert_eq!(f.status(), CallStatus::Failed); + assert_eq!(fake.submitted(), vec![]); + assert_eq!(f.upload_files(), Vec::::new()); + } + + #[tokio::test] + async fn pipeline_tolerates_empty_track() { + let f = fixture(); + let fake = FakeTranscriber::returning(vec![word("Hallo", 0.0, 0.5, None)], vec![]); + + f.process(&fake).await.unwrap(); + + assert_eq!(f.status(), CallStatus::Done); + assert_eq!(transcript(&f), vec!["Ich: Hallo".to_string()]); + } + + #[tokio::test] + async fn pipeline_skips_transcription_when_segments_exist() { + let f = fixture(); + let segments = vec![Segment { + speaker: Speaker::Me, + start: 0.0, + end: 1.0, + text: "Schon da".to_string(), + }]; + f.store.save_segments(f.id, &segments).unwrap(); + f.store + .set_status(f.id, CallStatus::Failed, Some("vorher")) + .unwrap(); + let fake = FakeTranscriber::default(); + + f.process(&fake).await.unwrap(); + + let call = f.store.get_call(f.id).unwrap().unwrap(); + assert_eq!(call.status, CallStatus::Done); + assert_eq!(call.error, None); + assert_eq!(fake.submitted(), vec![]); + assert_eq!(f.store.segments(f.id).unwrap(), segments); + } + + /// Zeroes the RIFF and data size fields, as left by a recording killed + /// before its first header refresh. + fn zero_header_sizes(path: &Path) { + let mut file = OpenOptions::new().write(true).open(path).unwrap(); + for offset in [4, 40] { + file.seek(SeekFrom::Start(offset)).unwrap(); + file.write_all(&0u32.to_le_bytes()).unwrap(); + } + } + + fn wav_duration(path: &Path) -> f64 { + let reader = hound::WavReader::open(path).unwrap(); + reader.duration() as f64 / reader.spec().sample_rate as f64 + } + + #[test] + fn startup_repairs_interrupted_recordings() { + let store = Store::open_in_memory().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mic = dir.path().join("1-mic.wav"); + let out = dir.path().join("1-out.wav"); + write_wav(&mic, 3.0); + write_wav(&out, 2.5); + zero_header_sizes(&mic); + zero_header_sizes(&out); + let started = Local.with_ymd_and_hms(2026, 11, 5, 10, 14, 0).unwrap(); + let id = store + .create_call("Microsoft Teams", started, &mic, &out) + .unwrap(); + + let report = recover_on_startup(&store).unwrap(); + + assert_eq!(report.interrupted, vec![id]); + let call = store.get_call(id).unwrap().unwrap(); + assert_eq!(call.status, CallStatus::AwaitingDecision); + assert!(call.interrupted); + assert_eq!(wav_duration(&mic), 3.0); + assert_eq!(wav_duration(&out), 2.5); + assert_eq!(call.ended_at, Some(started + Duration::milliseconds(2500))); + } + + #[test] + fn startup_treats_a_missing_track_as_empty() { + let store = Store::open_in_memory().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mic = dir.path().join("1-mic.wav"); + write_wav(&mic, 3.0); + let started = Local.with_ymd_and_hms(2026, 11, 5, 10, 14, 0).unwrap(); + let id = store + .create_call("Manuell", started, &mic, &dir.path().join("1-out.wav")) + .unwrap(); + + recover_on_startup(&store).unwrap(); + + let call = store.get_call(id).unwrap().unwrap(); + assert_eq!(call.status, CallStatus::AwaitingDecision); + assert_eq!(call.ended_at, Some(started)); + } +} diff --git a/crates/core/src/transcribe/deepscript.rs b/crates/core/src/transcribe/deepscript.rs index 3bb02d5..d591692 100644 --- a/crates/core/src/transcribe/deepscript.rs +++ b/crates/core/src/transcribe/deepscript.rs @@ -216,7 +216,7 @@ impl Transcriber for DeepScript { "queued" | "processing" => {} other => { if !self.warned_unknown_status.swap(true, Ordering::Relaxed) { - eprintln!( + log::warn!( "DeepScript: unbekannter Transkriptionsstatus '{other}', warte weiter" ); } diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 63462cb..047093b 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -21,6 +21,7 @@ tauri = { version = "2.12.5", features = ["tray-icon"] } tauri-plugin-log = "2" tauri-plugin-autostart = "2" tauri-plugin-single-instance = "2" +tauri-plugin-notification = "2" chrono = { version = "0.4", features = ["serde"] } tempfile = "3" recorder-core = { path = "../crates/core" } diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index 718e39f..ca59881 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -8,6 +8,7 @@ "prompt" ], "permissions": [ - "core:default" + "core:default", + "notification:default" ] } diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 5459a5a..db91f29 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -16,6 +16,7 @@ use serde::Serialize; use tauri::{AppHandle, Emitter, Manager, State}; use tauri_plugin_autostart::ManagerExt; +use crate::processing; use crate::state::{AppState, CallBrief, DeviceLevels, Ended, PendingPrompt, RecordingState}; const LEVELS_INTERVAL: Duration = Duration::from_millis(100); @@ -120,19 +121,35 @@ pub fn get_call_brief(state: State<'_, AppState>, id: CallId) -> CmdResult CmdResult<()> { blocking(app, move |app| { - if app.state::().discard_call(id)? == 0 { - set_window_visible(app, "prompt", false); - } + app.state::().discard_call(id)?; + prompt_queue_changed(app); Ok(()) }) .await } -/// Placeholder until transcription exists; the prompt shows the error. +/// Transkribieren in the end-of-call prompt: starts processing in the +/// background and returns at once; the call leaves the prompt. #[tauri::command] -pub fn transcribe_call(id: CallId) -> CmdResult<()> { - let _ = id; - Err("Transkription ist noch nicht verfügbar".to_string()) +pub async fn transcribe_call(app: AppHandle, id: CallId) -> CmdResult<()> { + blocking(app, move |app| processing::start(app, id, false)).await +} + +/// Processes a failed (or still undecided) call again. +#[tauri::command] +pub async fn retry_call(app: AppHandle, id: CallId) -> CmdResult<()> { + blocking(app, move |app| processing::start(app, id, true)).await +} + +/// After a call left the prompt queue: hides the prompt if no call waits +/// any more, and tells it to reload (`prompt-changed`) otherwise. +pub fn prompt_queue_changed(app: &AppHandle) { + if app.state::().prompts_waiting() == 0 { + set_window_visible(app, "prompt", false); + } + if let Err(e) = app.emit("prompt-changed", ()) { + log::error!("could not emit prompt-changed: {e}"); + } } /// Starts a manual recording, shows the floating window and streams levels. diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7f91e10..01ccf13 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,8 +1,10 @@ mod commands; mod detection; +mod processing; mod state; mod tray; +use recorder_core::pipeline; use recorder_core::secrets::KeyringSecrets; use recorder_core::store::Store; use tauri::{Manager, WindowEvent}; @@ -28,6 +30,7 @@ pub fn run() { MacosLauncher::LaunchAgent, Some(vec![AUTOSTART_ARG]), )) + .plugin(tauri_plugin_notification::init()) .setup(|app| { // Release builds log to a file in the app log directory (on // Windows %LOCALAPPDATA%\eu.euvino.callrecorder\logs), debug @@ -50,7 +53,7 @@ pub fn run() { let config_dir = app.path().app_config_dir()?; std::fs::create_dir_all(&data_dir)?; let store = Store::open(&data_dir.join("calls.sqlite"))?; - let recovery = store.startup_recovery()?; + let recovery = pipeline::recover_on_startup(&store)?; log::info!("startup recovery: {recovery:?}"); let state = AppState::new( @@ -62,6 +65,7 @@ pub fn run() { let autostart = state.settings().autostart; app.manage(state); commands::apply_autostart(app.handle(), autostart); + processing::delete_leftover_jobs(app.handle(), recovery.failed); tray::create(app.handle())?; detection::spawn(app.handle().clone()); @@ -93,6 +97,7 @@ pub fn run() { commands::get_call_brief, commands::discard_call, commands::transcribe_call, + commands::retry_call, ]) .run(tauri::generate_context!()) .expect("error while building tauri application"); diff --git a/src-tauri/src/processing.rs b/src-tauri/src/processing.rs new file mode 100644 index 0000000..71ab368 --- /dev/null +++ b/src-tauri/src/processing.rs @@ -0,0 +1,167 @@ +//! Runs the processing pipeline for calls in the background and tells the +//! user about it: a `call-updated` event on every status change and a +//! system notification when a run ends. Notifications never contain +//! transcript text, and the API key is never logged. + +use recorder_core::model::{CallId, CallStatus}; +use recorder_core::pipeline::{self, PipelineOptions}; +use recorder_core::secrets::Service; +use recorder_core::transcribe::deepscript::DeepScript; +use recorder_core::transcribe::TranscribeError; +use serde::Serialize; +use tauri::{AppHandle, Emitter, Manager}; +use tauri_plugin_notification::NotificationExt; + +use crate::commands; +use crate::state::AppState; + +/// Payload of the `call-updated` event: the call's status changed. +#[derive(Serialize, Clone)] +struct CallUpdated { + id: CallId, +} + +/// A call reserved with [`AppState::claim_processing`]; released when +/// dropped, also if the run panics. +struct Claim { + app: AppHandle, + id: CallId, +} + +impl Claim { + fn take(app: &AppHandle, id: CallId, retry: bool) -> Result { + app.state::().claim_processing(id, retry)?; + Ok(Self { + app: app.clone(), + id, + }) + } +} + +impl Drop for Claim { + fn drop(&mut self) { + self.app.state::().release_processing(self.id); + } +} + +/// Starts processing call `id` in the background and returns at once; the +/// call leaves the end-of-call prompt. `retry` also allows failed calls. +/// Without a DeepScript key nothing starts: the call is marked failed and +/// the error returned. Reads the keyring, so call it off the UI thread. +pub fn start(app: &AppHandle, id: CallId, retry: bool) -> Result<(), String> { + let claim = Claim::take(app, id, retry)?; + let state = app.state::(); + + let Some(key) = deepscript_key(&state) else { + let msg = TranscribeError::MissingKey.user_message(); + if let Err(e) = state.store.set_status(id, CallStatus::Failed, Some(&msg)) { + log::error!("call {id}: could not mark as failed: {e}"); + } + emit_updated(app, id); + return Err(msg); + }; + let work_dir = app + .path() + .app_cache_dir() + .map_err(|e| format!("Cache-Ordner nicht gefunden: {e}"))? + .join("upload"); + let settings = state.settings(); + let opts = PipelineOptions { + language: settings.language, + deepscript_model: settings.deepscript_model, + }; + // A fresh client per run, so a key changed in the settings applies to + // the next run. + let transcriber = DeepScript::new(state.transport.clone(), key); + + // Shown as processing right away, before the run gets going. + state + .store + .set_status(id, CallStatus::Processing, None) + .map_err(|e| format!("Interner Fehler: {e}"))?; + emit_updated(app, id); + if state.remove_queued_prompt(id).is_some() { + commands::prompt_queue_changed(app); + } + + log::info!("call {id}: processing started"); + tauri::async_runtime::spawn(async move { + let app = &claim.app; + let state = app.state::(); + let result = pipeline::process_call(&state.store, &transcriber, id, &opts, &work_dir).await; + emit_updated(app, id); + notify(app, id, result.err().map(|e| e.user_message())); + }); + Ok(()) +} + +/// Best-effort deletes the DeepScript jobs of calls whose processing the +/// last run of the app did not finish ([`RecoveryReport::failed`]), in the +/// background. Skipped without a key. +/// +/// [`RecoveryReport::failed`]: recorder_core::store::RecoveryReport::failed +pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec) { + if calls.is_empty() { + return; + } + let app = app.clone(); + tauri::async_runtime::spawn(async move { + let key = { + let app = app.clone(); + tauri::async_runtime::spawn_blocking(move || deepscript_key(&app.state())).await + }; + let Ok(Some(key)) = key else { + log::info!("no DeepScript key: remote jobs of calls {calls:?} stay for now"); + return; + }; + let state = app.state::(); + let transcriber = DeepScript::new(state.transport.clone(), key); + for id in calls { + // A retry the user started meanwhile cleans up itself. + let Ok(_claim) = Claim::take(&app, id, true) else { + continue; + }; + pipeline::delete_remote_jobs(&state.store, &transcriber, id).await; + } + }); +} + +fn deepscript_key(state: &AppState) -> Option { + state + .secrets + .get(Service::DeepScript) + .filter(|key| !key.trim().is_empty()) +} + +fn emit_updated(app: &AppHandle, id: CallId) { + if let Err(e) = app.emit("call-updated", CallUpdated { id }) { + log::error!("could not emit call-updated: {e}"); + } +} + +/// "Transkript bereit" with " · min", or the failure's message. +fn notify(app: &AppHandle, id: CallId, error: Option) { + let (title, body) = match error { + None => ("Transkript bereit", call_summary_line(app, id)), + Some(msg) => ("Transkription fehlgeschlagen", msg), + }; + let result = app.notification().builder().title(title).body(body).show(); + if let Err(e) = result { + log::warn!("call {id}: could not show the notification: {e}"); + } +} + +/// "Microsoft Teams · 32 min" (whole minutes, at least 1, like the UI). +fn call_summary_line(app: &AppHandle, id: CallId) -> String { + match app.state::().call_brief(id) { + Ok(brief) => format!( + "{} · {} min", + brief.app, + ((brief.duration_secs as f64 / 60.0).round() as i64).max(1) + ), + Err(e) => { + log::error!("call {id}: could not load it for the notification: {e}"); + String::new() + } + } +} diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index 8a8e5fd..ca6f477 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -1,22 +1,24 @@ //! Application state shared by commands, the tray and background threads. //! -//! Holds the store, the settings, the secret store, the recording that is -//! currently running, the call detector and the call awaiting a decision in -//! the end-of-call prompt. Everything here is independent of windows and +//! Holds the store, the settings, the secret store, the HTTP transport, the +//! recording that is currently running, the call detector, the calls +//! awaiting a decision in the end-of-call prompt and the calls being +//! processed. Everything here is independent of windows and //! events; the command layer turns results into UI updates. -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use std::fs; use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, MutexGuard}; use std::thread; use std::time::{Duration, Instant}; use chrono::{DateTime, Local}; use recorder_core::capture::{CaptureSource, OutputTarget}; use recorder_core::detect::{CallDetector, DetectorEvent, MicSession}; +use recorder_core::http::{HttpTransport, ReqwestTransport}; use recorder_core::model::{Call, CallId, CallStatus}; use recorder_core::recorder::Recorder; use recorder_core::secrets::SecretStore; @@ -31,6 +33,7 @@ const MANUAL_APP: &str = "Manuell"; /// How long [`AppState::test_devices`] records. const TEST_DURATION: Duration = Duration::from_secs(5); const TEST_POLL: Duration = Duration::from_millis(50); +const ALREADY_PROCESSING: &str = "Wird bereits verarbeitet"; /// Payload of the `recording-state` event. #[derive(Serialize, Clone, Debug)] @@ -111,6 +114,8 @@ struct CurrentRecording { pub struct AppState { pub store: Store, pub secrets: Box, + /// The one HTTP client all API calls share. + pub transport: Arc, settings: Mutex, settings_path: PathBuf, recordings_dir: PathBuf, @@ -120,6 +125,8 @@ pub struct AppState { /// first. A call leaves the queue when it is decided (discarded or sent /// to transcription). pending_prompts: Mutex>, + /// Calls a pipeline run is under way for; at most one run per call. + processing: Mutex>, /// Set while a recording starts or the devices are tested, so neither /// runs twice and `current` is never locked for longer than a moment. devices_busy: AtomicBool, @@ -146,12 +153,14 @@ impl AppState { Self { store, secrets, + transport: Arc::new(ReqwestTransport::new()), settings: Mutex::new(settings), settings_path, recordings_dir, current: Mutex::new(None), detector: Mutex::new(detector), pending_prompts: Mutex::new(VecDeque::new()), + processing: Mutex::new(HashSet::new()), devices_busy: AtomicBool::new(false), } } @@ -432,6 +441,40 @@ impl AppState { Ok(self.prompts_waiting()) } + /// Reserves `id` for one pipeline run; release it with + /// [`AppState::release_processing`] when the run ends. A call that waits + /// for the decision can be processed; a failed one only on `retry`, or + /// while the prompt still shows it (its first attempt failed before + /// anything started, e.g. for lack of an API key). + pub fn claim_processing(&self, id: CallId, retry: bool) -> Result<(), String> { + let mut processing = lock(&self.processing); + if processing.contains(&id) { + return Err(ALREADY_PROCESSING.to_string()); + } + let call = self + .store + .get_call(id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "Gespräch nicht gefunden".to_string())?; + match call.status { + CallStatus::AwaitingDecision => {} + CallStatus::Failed if retry || lock(&self.pending_prompts).contains(&id) => {} + CallStatus::Processing => return Err(ALREADY_PROCESSING.to_string()), + CallStatus::Recording => return Err("Das Gespräch wird gerade aufgenommen".to_string()), + CallStatus::Done => return Err("Das Gespräch ist bereits transkribiert".to_string()), + CallStatus::Failed => { + return Err("Das Gespräch wartet nicht auf eine Entscheidung".to_string()) + } + } + processing.insert(id); + Ok(()) + } + + /// Ends the pipeline run claimed by [`AppState::claim_processing`]. + pub fn release_processing(&self, id: CallId) { + lock(&self.processing).remove(&id); + } + /// Records both configured devices into a temporary directory for five /// seconds and returns the peak level of each track. pub fn test_devices(&self) -> Result { @@ -664,6 +707,67 @@ mod tests { assert_eq!(f.state.prompts_waiting(), 0); } + #[test] + fn a_call_is_processed_once_at_a_time() { + let f = fixture(); + let (id, _, _) = finished_call(&f, "a"); + assert_eq!(f.state.claim_processing(id, false), Ok(())); + assert_eq!( + f.state.claim_processing(id, true), + Err("Wird bereits verarbeitet".to_string()) + ); + f.state.release_processing(id); + assert_eq!(f.state.claim_processing(id, false), Ok(())); + } + + #[test] + fn a_processing_call_is_not_claimed_again() { + let f = fixture(); + let (id, _, _) = finished_call(&f, "a"); + f.state + .store + .set_status(id, CallStatus::Processing, None) + .unwrap(); + assert_eq!( + f.state.claim_processing(id, true), + Err("Wird bereits verarbeitet".to_string()) + ); + } + + #[test] + fn a_failed_call_is_processed_again_on_retry_or_from_the_prompt() { + let f = fixture(); + let (id, _, _) = finished_call(&f, "a"); + f.state + .store + .set_status(id, CallStatus::Failed, Some("Fehler")) + .unwrap(); + assert!(f.state.claim_processing(id, false).is_err()); + assert_eq!(f.state.claim_processing(id, true), Ok(())); + f.state.release_processing(id); + + f.state.queue_prompt(id); + assert_eq!(f.state.claim_processing(id, false), Ok(())); + } + + #[test] + fn done_and_recording_calls_are_not_processed() { + let f = fixture(); + let (done, _, _) = finished_call(&f, "a"); + f.state + .store + .set_status(done, CallStatus::Done, None) + .unwrap(); + assert!(f.state.claim_processing(done, true).is_err()); + let recording = f + .state + .store + .create_call("Manuell", at(9, 0, 0), Path::new(""), Path::new("")) + .unwrap(); + assert!(f.state.claim_processing(recording, true).is_err()); + assert!(f.state.claim_processing(recording + 1, true).is_err()); + } + #[test] fn invalid_detection_delays_are_not_saved() { let f = fixture(); diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index 4946f21..17d5637 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -74,12 +74,19 @@ export const discardCurrentRecording = () => invoke('discard_current_recor export const getPendingPrompt = () => invoke('get_pending_prompt') export const getCallBrief = (id: CallId) => invoke('get_call_brief', { id }) export const transcribeCall = (id: CallId) => invoke('transcribe_call', { id }) +export const retryCall = (id: CallId) => invoke('retry_call', { id }) export const discardCall = (id: CallId) => invoke('discard_call', { id }) export const onRecordingState = (handler: (state: RecordingState) => void): Promise => listen('recording-state', (event) => handler(event.payload)) export const onCallEnded = (handler: (id: CallId) => void): Promise => listen<{ id: CallId }>('call-ended', (event) => handler(event.payload.id)) +/** The prompt queue changed (a call was decided elsewhere). */ +export const onPromptChanged = (handler: () => void): Promise => + listen('prompt-changed', () => handler()) +/** A call's status changed (processing started, done or failed). */ +export const onCallUpdated = (handler: (id: CallId) => void): Promise => + listen<{ id: CallId }>('call-updated', (event) => handler(event.payload.id)) export const onLevels = (handler: (levels: Levels) => void): Promise => listen('levels', (event) => handler(event.payload)) diff --git a/ui/src/routes/Prompt.svelte b/ui/src/routes/Prompt.svelte index b4f3258..83ec83c 100644 --- a/ui/src/routes/Prompt.svelte +++ b/ui/src/routes/Prompt.svelte @@ -5,6 +5,7 @@ errorText, getPendingPrompt, onCallEnded, + onPromptChanged, transcribeCall, type CallId, type PendingPrompt, @@ -29,12 +30,12 @@ .catch((err) => (error = errorText(err))) onMount(() => { - const unlisten = onCallEnded(refresh) + const unlisten = [onCallEnded(refresh), onPromptChanged(refresh)] refresh() window.addEventListener('focus', refresh) return () => { window.removeEventListener('focus', refresh) - unlisten.then((off) => off()) + unlisten.forEach((u) => u.then((off) => off())) } })