feat #14740: transcription pipeline
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@ use serde::Serialize;
|
||||
use tauri::{AppHandle, Emitter, Manager, State};
|
||||
use tauri_plugin_autostart::ManagerExt;
|
||||
|
||||
use crate::processing;
|
||||
use crate::state::{AppState, CallBrief, DeviceLevels, Ended, PendingPrompt, RecordingState};
|
||||
|
||||
const LEVELS_INTERVAL: Duration = Duration::from_millis(100);
|
||||
@@ -120,19 +121,35 @@ pub fn get_call_brief(state: State<'_, AppState>, id: CallId) -> CmdResult<CallB
|
||||
#[tauri::command]
|
||||
pub async fn discard_call(app: AppHandle, id: CallId) -> CmdResult<()> {
|
||||
blocking(app, move |app| {
|
||||
if app.state::<AppState>().discard_call(id)? == 0 {
|
||||
set_window_visible(app, "prompt", false);
|
||||
}
|
||||
app.state::<AppState>().discard_call(id)?;
|
||||
prompt_queue_changed(app);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Placeholder until transcription exists; the prompt shows the error.
|
||||
/// Transkribieren in the end-of-call prompt: starts processing in the
|
||||
/// background and returns at once; the call leaves the prompt.
|
||||
#[tauri::command]
|
||||
pub fn transcribe_call(id: CallId) -> CmdResult<()> {
|
||||
let _ = id;
|
||||
Err("Transkription ist noch nicht verfügbar".to_string())
|
||||
pub async fn transcribe_call(app: AppHandle, id: CallId) -> CmdResult<()> {
|
||||
blocking(app, move |app| processing::start(app, id, false)).await
|
||||
}
|
||||
|
||||
/// Processes a failed (or still undecided) call again.
|
||||
#[tauri::command]
|
||||
pub async fn retry_call(app: AppHandle, id: CallId) -> CmdResult<()> {
|
||||
blocking(app, move |app| processing::start(app, id, true)).await
|
||||
}
|
||||
|
||||
/// After a call left the prompt queue: hides the prompt if no call waits
|
||||
/// any more, and tells it to reload (`prompt-changed`) otherwise.
|
||||
pub fn prompt_queue_changed(app: &AppHandle) {
|
||||
if app.state::<AppState>().prompts_waiting() == 0 {
|
||||
set_window_visible(app, "prompt", false);
|
||||
}
|
||||
if let Err(e) = app.emit("prompt-changed", ()) {
|
||||
log::error!("could not emit prompt-changed: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts a manual recording, shows the floating window and streams levels.
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
mod commands;
|
||||
mod detection;
|
||||
mod processing;
|
||||
mod state;
|
||||
mod tray;
|
||||
|
||||
use recorder_core::pipeline;
|
||||
use recorder_core::secrets::KeyringSecrets;
|
||||
use recorder_core::store::Store;
|
||||
use tauri::{Manager, WindowEvent};
|
||||
@@ -28,6 +30,7 @@ pub fn run() {
|
||||
MacosLauncher::LaunchAgent,
|
||||
Some(vec![AUTOSTART_ARG]),
|
||||
))
|
||||
.plugin(tauri_plugin_notification::init())
|
||||
.setup(|app| {
|
||||
// Release builds log to a file in the app log directory (on
|
||||
// Windows %LOCALAPPDATA%\eu.euvino.callrecorder\logs), debug
|
||||
@@ -50,7 +53,7 @@ pub fn run() {
|
||||
let config_dir = app.path().app_config_dir()?;
|
||||
std::fs::create_dir_all(&data_dir)?;
|
||||
let store = Store::open(&data_dir.join("calls.sqlite"))?;
|
||||
let recovery = store.startup_recovery()?;
|
||||
let recovery = pipeline::recover_on_startup(&store)?;
|
||||
log::info!("startup recovery: {recovery:?}");
|
||||
|
||||
let state = AppState::new(
|
||||
@@ -62,6 +65,7 @@ pub fn run() {
|
||||
let autostart = state.settings().autostart;
|
||||
app.manage(state);
|
||||
commands::apply_autostart(app.handle(), autostart);
|
||||
processing::delete_leftover_jobs(app.handle(), recovery.failed);
|
||||
|
||||
tray::create(app.handle())?;
|
||||
detection::spawn(app.handle().clone());
|
||||
@@ -93,6 +97,7 @@ pub fn run() {
|
||||
commands::get_call_brief,
|
||||
commands::discard_call,
|
||||
commands::transcribe_call,
|
||||
commands::retry_call,
|
||||
])
|
||||
.run(tauri::generate_context!())
|
||||
.expect("error while building tauri application");
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
//! Runs the processing pipeline for calls in the background and tells the
|
||||
//! user about it: a `call-updated` event on every status change and a
|
||||
//! system notification when a run ends. Notifications never contain
|
||||
//! transcript text, and the API key is never logged.
|
||||
|
||||
use recorder_core::model::{CallId, CallStatus};
|
||||
use recorder_core::pipeline::{self, PipelineOptions};
|
||||
use recorder_core::secrets::Service;
|
||||
use recorder_core::transcribe::deepscript::DeepScript;
|
||||
use recorder_core::transcribe::TranscribeError;
|
||||
use serde::Serialize;
|
||||
use tauri::{AppHandle, Emitter, Manager};
|
||||
use tauri_plugin_notification::NotificationExt;
|
||||
|
||||
use crate::commands;
|
||||
use crate::state::AppState;
|
||||
|
||||
/// Payload of the `call-updated` event: the call's status changed.
|
||||
#[derive(Serialize, Clone)]
|
||||
struct CallUpdated {
|
||||
id: CallId,
|
||||
}
|
||||
|
||||
/// A call reserved with [`AppState::claim_processing`]; released when
|
||||
/// dropped, also if the run panics.
|
||||
struct Claim {
|
||||
app: AppHandle,
|
||||
id: CallId,
|
||||
}
|
||||
|
||||
impl Claim {
|
||||
fn take(app: &AppHandle, id: CallId, retry: bool) -> Result<Self, String> {
|
||||
app.state::<AppState>().claim_processing(id, retry)?;
|
||||
Ok(Self {
|
||||
app: app.clone(),
|
||||
id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Claim {
|
||||
fn drop(&mut self) {
|
||||
self.app.state::<AppState>().release_processing(self.id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts processing call `id` in the background and returns at once; the
|
||||
/// call leaves the end-of-call prompt. `retry` also allows failed calls.
|
||||
/// Without a DeepScript key nothing starts: the call is marked failed and
|
||||
/// the error returned. Reads the keyring, so call it off the UI thread.
|
||||
pub fn start(app: &AppHandle, id: CallId, retry: bool) -> Result<(), String> {
|
||||
let claim = Claim::take(app, id, retry)?;
|
||||
let state = app.state::<AppState>();
|
||||
|
||||
let Some(key) = deepscript_key(&state) else {
|
||||
let msg = TranscribeError::MissingKey.user_message();
|
||||
if let Err(e) = state.store.set_status(id, CallStatus::Failed, Some(&msg)) {
|
||||
log::error!("call {id}: could not mark as failed: {e}");
|
||||
}
|
||||
emit_updated(app, id);
|
||||
return Err(msg);
|
||||
};
|
||||
let work_dir = app
|
||||
.path()
|
||||
.app_cache_dir()
|
||||
.map_err(|e| format!("Cache-Ordner nicht gefunden: {e}"))?
|
||||
.join("upload");
|
||||
let settings = state.settings();
|
||||
let opts = PipelineOptions {
|
||||
language: settings.language,
|
||||
deepscript_model: settings.deepscript_model,
|
||||
};
|
||||
// A fresh client per run, so a key changed in the settings applies to
|
||||
// the next run.
|
||||
let transcriber = DeepScript::new(state.transport.clone(), key);
|
||||
|
||||
// Shown as processing right away, before the run gets going.
|
||||
state
|
||||
.store
|
||||
.set_status(id, CallStatus::Processing, None)
|
||||
.map_err(|e| format!("Interner Fehler: {e}"))?;
|
||||
emit_updated(app, id);
|
||||
if state.remove_queued_prompt(id).is_some() {
|
||||
commands::prompt_queue_changed(app);
|
||||
}
|
||||
|
||||
log::info!("call {id}: processing started");
|
||||
tauri::async_runtime::spawn(async move {
|
||||
let app = &claim.app;
|
||||
let state = app.state::<AppState>();
|
||||
let result = pipeline::process_call(&state.store, &transcriber, id, &opts, &work_dir).await;
|
||||
emit_updated(app, id);
|
||||
notify(app, id, result.err().map(|e| e.user_message()));
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Best-effort deletes the DeepScript jobs of calls whose processing the
|
||||
/// last run of the app did not finish ([`RecoveryReport::failed`]), in the
|
||||
/// background. Skipped without a key.
|
||||
///
|
||||
/// [`RecoveryReport::failed`]: recorder_core::store::RecoveryReport::failed
|
||||
pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec<CallId>) {
|
||||
if calls.is_empty() {
|
||||
return;
|
||||
}
|
||||
let app = app.clone();
|
||||
tauri::async_runtime::spawn(async move {
|
||||
let key = {
|
||||
let app = app.clone();
|
||||
tauri::async_runtime::spawn_blocking(move || deepscript_key(&app.state())).await
|
||||
};
|
||||
let Ok(Some(key)) = key else {
|
||||
log::info!("no DeepScript key: remote jobs of calls {calls:?} stay for now");
|
||||
return;
|
||||
};
|
||||
let state = app.state::<AppState>();
|
||||
let transcriber = DeepScript::new(state.transport.clone(), key);
|
||||
for id in calls {
|
||||
// A retry the user started meanwhile cleans up itself.
|
||||
let Ok(_claim) = Claim::take(&app, id, true) else {
|
||||
continue;
|
||||
};
|
||||
pipeline::delete_remote_jobs(&state.store, &transcriber, id).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn deepscript_key(state: &AppState) -> Option<String> {
|
||||
state
|
||||
.secrets
|
||||
.get(Service::DeepScript)
|
||||
.filter(|key| !key.trim().is_empty())
|
||||
}
|
||||
|
||||
fn emit_updated(app: &AppHandle, id: CallId) {
|
||||
if let Err(e) = app.emit("call-updated", CallUpdated { id }) {
|
||||
log::error!("could not emit call-updated: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// "Transkript bereit" with "<app> · <N> min", or the failure's message.
|
||||
fn notify(app: &AppHandle, id: CallId, error: Option<String>) {
|
||||
let (title, body) = match error {
|
||||
None => ("Transkript bereit", call_summary_line(app, id)),
|
||||
Some(msg) => ("Transkription fehlgeschlagen", msg),
|
||||
};
|
||||
let result = app.notification().builder().title(title).body(body).show();
|
||||
if let Err(e) = result {
|
||||
log::warn!("call {id}: could not show the notification: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// "Microsoft Teams · 32 min" (whole minutes, at least 1, like the UI).
|
||||
fn call_summary_line(app: &AppHandle, id: CallId) -> String {
|
||||
match app.state::<AppState>().call_brief(id) {
|
||||
Ok(brief) => format!(
|
||||
"{} · {} min",
|
||||
brief.app,
|
||||
((brief.duration_secs as f64 / 60.0).round() as i64).max(1)
|
||||
),
|
||||
Err(e) => {
|
||||
log::error!("call {id}: could not load it for the notification: {e}");
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
+109
-5
@@ -1,22 +1,24 @@
|
||||
//! Application state shared by commands, the tray and background threads.
|
||||
//!
|
||||
//! Holds the store, the settings, the secret store, the recording that is
|
||||
//! currently running, the call detector and the call awaiting a decision in
|
||||
//! the end-of-call prompt. Everything here is independent of windows and
|
||||
//! Holds the store, the settings, the secret store, the HTTP transport, the
|
||||
//! recording that is currently running, the call detector, the calls
|
||||
//! awaiting a decision in the end-of-call prompt and the calls being
|
||||
//! processed. Everything here is independent of windows and
|
||||
//! events; the command layer turns results into UI updates.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::fs;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use chrono::{DateTime, Local};
|
||||
use recorder_core::capture::{CaptureSource, OutputTarget};
|
||||
use recorder_core::detect::{CallDetector, DetectorEvent, MicSession};
|
||||
use recorder_core::http::{HttpTransport, ReqwestTransport};
|
||||
use recorder_core::model::{Call, CallId, CallStatus};
|
||||
use recorder_core::recorder::Recorder;
|
||||
use recorder_core::secrets::SecretStore;
|
||||
@@ -31,6 +33,7 @@ const MANUAL_APP: &str = "Manuell";
|
||||
/// How long [`AppState::test_devices`] records.
|
||||
const TEST_DURATION: Duration = Duration::from_secs(5);
|
||||
const TEST_POLL: Duration = Duration::from_millis(50);
|
||||
const ALREADY_PROCESSING: &str = "Wird bereits verarbeitet";
|
||||
|
||||
/// Payload of the `recording-state` event.
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
@@ -111,6 +114,8 @@ struct CurrentRecording {
|
||||
pub struct AppState {
|
||||
pub store: Store,
|
||||
pub secrets: Box<dyn SecretStore>,
|
||||
/// The one HTTP client all API calls share.
|
||||
pub transport: Arc<dyn HttpTransport>,
|
||||
settings: Mutex<Settings>,
|
||||
settings_path: PathBuf,
|
||||
recordings_dir: PathBuf,
|
||||
@@ -120,6 +125,8 @@ pub struct AppState {
|
||||
/// first. A call leaves the queue when it is decided (discarded or sent
|
||||
/// to transcription).
|
||||
pending_prompts: Mutex<VecDeque<CallId>>,
|
||||
/// Calls a pipeline run is under way for; at most one run per call.
|
||||
processing: Mutex<HashSet<CallId>>,
|
||||
/// Set while a recording starts or the devices are tested, so neither
|
||||
/// runs twice and `current` is never locked for longer than a moment.
|
||||
devices_busy: AtomicBool,
|
||||
@@ -146,12 +153,14 @@ impl AppState {
|
||||
Self {
|
||||
store,
|
||||
secrets,
|
||||
transport: Arc::new(ReqwestTransport::new()),
|
||||
settings: Mutex::new(settings),
|
||||
settings_path,
|
||||
recordings_dir,
|
||||
current: Mutex::new(None),
|
||||
detector: Mutex::new(detector),
|
||||
pending_prompts: Mutex::new(VecDeque::new()),
|
||||
processing: Mutex::new(HashSet::new()),
|
||||
devices_busy: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
@@ -432,6 +441,40 @@ impl AppState {
|
||||
Ok(self.prompts_waiting())
|
||||
}
|
||||
|
||||
/// Reserves `id` for one pipeline run; release it with
|
||||
/// [`AppState::release_processing`] when the run ends. A call that waits
|
||||
/// for the decision can be processed; a failed one only on `retry`, or
|
||||
/// while the prompt still shows it (its first attempt failed before
|
||||
/// anything started, e.g. for lack of an API key).
|
||||
pub fn claim_processing(&self, id: CallId, retry: bool) -> Result<(), String> {
|
||||
let mut processing = lock(&self.processing);
|
||||
if processing.contains(&id) {
|
||||
return Err(ALREADY_PROCESSING.to_string());
|
||||
}
|
||||
let call = self
|
||||
.store
|
||||
.get_call(id)
|
||||
.map_err(|e| e.to_string())?
|
||||
.ok_or_else(|| "Gespräch nicht gefunden".to_string())?;
|
||||
match call.status {
|
||||
CallStatus::AwaitingDecision => {}
|
||||
CallStatus::Failed if retry || lock(&self.pending_prompts).contains(&id) => {}
|
||||
CallStatus::Processing => return Err(ALREADY_PROCESSING.to_string()),
|
||||
CallStatus::Recording => return Err("Das Gespräch wird gerade aufgenommen".to_string()),
|
||||
CallStatus::Done => return Err("Das Gespräch ist bereits transkribiert".to_string()),
|
||||
CallStatus::Failed => {
|
||||
return Err("Das Gespräch wartet nicht auf eine Entscheidung".to_string())
|
||||
}
|
||||
}
|
||||
processing.insert(id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ends the pipeline run claimed by [`AppState::claim_processing`].
|
||||
pub fn release_processing(&self, id: CallId) {
|
||||
lock(&self.processing).remove(&id);
|
||||
}
|
||||
|
||||
/// Records both configured devices into a temporary directory for five
|
||||
/// seconds and returns the peak level of each track.
|
||||
pub fn test_devices(&self) -> Result<DeviceLevels, String> {
|
||||
@@ -664,6 +707,67 @@ mod tests {
|
||||
assert_eq!(f.state.prompts_waiting(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_call_is_processed_once_at_a_time() {
|
||||
let f = fixture();
|
||||
let (id, _, _) = finished_call(&f, "a");
|
||||
assert_eq!(f.state.claim_processing(id, false), Ok(()));
|
||||
assert_eq!(
|
||||
f.state.claim_processing(id, true),
|
||||
Err("Wird bereits verarbeitet".to_string())
|
||||
);
|
||||
f.state.release_processing(id);
|
||||
assert_eq!(f.state.claim_processing(id, false), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_processing_call_is_not_claimed_again() {
|
||||
let f = fixture();
|
||||
let (id, _, _) = finished_call(&f, "a");
|
||||
f.state
|
||||
.store
|
||||
.set_status(id, CallStatus::Processing, None)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
f.state.claim_processing(id, true),
|
||||
Err("Wird bereits verarbeitet".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_call_is_processed_again_on_retry_or_from_the_prompt() {
|
||||
let f = fixture();
|
||||
let (id, _, _) = finished_call(&f, "a");
|
||||
f.state
|
||||
.store
|
||||
.set_status(id, CallStatus::Failed, Some("Fehler"))
|
||||
.unwrap();
|
||||
assert!(f.state.claim_processing(id, false).is_err());
|
||||
assert_eq!(f.state.claim_processing(id, true), Ok(()));
|
||||
f.state.release_processing(id);
|
||||
|
||||
f.state.queue_prompt(id);
|
||||
assert_eq!(f.state.claim_processing(id, false), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn done_and_recording_calls_are_not_processed() {
|
||||
let f = fixture();
|
||||
let (done, _, _) = finished_call(&f, "a");
|
||||
f.state
|
||||
.store
|
||||
.set_status(done, CallStatus::Done, None)
|
||||
.unwrap();
|
||||
assert!(f.state.claim_processing(done, true).is_err());
|
||||
let recording = f
|
||||
.state
|
||||
.store
|
||||
.create_call("Manuell", at(9, 0, 0), Path::new(""), Path::new(""))
|
||||
.unwrap();
|
||||
assert!(f.state.claim_processing(recording, true).is_err());
|
||||
assert!(f.state.claim_processing(recording + 1, true).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_detection_delays_are_not_saved() {
|
||||
let f = fixture();
|
||||
|
||||
Reference in New Issue
Block a user