fix #14740: always clean up deepscript jobs and upload copies
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+143
-14
@@ -68,7 +68,8 @@ const OUT_TRACK: &str = "out";
|
|||||||
|
|
||||||
/// Transcribes call `id` and stores the merged transcript, ending in
|
/// Transcribes call `id` and stores the merged transcript, ending in
|
||||||
/// `Done`. A call that already has a transcript (a retry after a later
|
/// `Done`. A call that already has a transcript (a retry after a later
|
||||||
/// step failed) is not transcribed again.
|
/// step failed) is not transcribed again; only remote jobs still on record
|
||||||
|
/// for it are deleted.
|
||||||
///
|
///
|
||||||
/// Both tracks are encoded to FLAC in `work_dir` (removed again afterwards)
|
/// Both tracks are encoded to FLAC in `work_dir` (removed again afterwards)
|
||||||
/// and uploaded; the remote jobs are deleted once their results are in, or
|
/// and uploaded; the remote jobs are deleted once their results are in, or
|
||||||
@@ -103,6 +104,9 @@ async fn run(
|
|||||||
if call.transcribed_at.is_none() && store.segments(id)?.is_empty() {
|
if call.transcribed_at.is_none() && store.segments(id)?.is_empty() {
|
||||||
store.set_status(id, CallStatus::Processing, None)?;
|
store.set_status(id, CallStatus::Processing, None)?;
|
||||||
transcribe(store, tr, &call, opts, work_dir).await?;
|
transcribe(store, tr, &call, opts, work_dir).await?;
|
||||||
|
} else {
|
||||||
|
// Jobs whose delete failed in an earlier run.
|
||||||
|
delete_remote_jobs(store, tr, id).await;
|
||||||
}
|
}
|
||||||
// SUMMARY SEAM (Task 14b): summarize here.
|
// SUMMARY SEAM (Task 14b): summarize here.
|
||||||
store.set_status(id, CallStatus::Done, None)?;
|
store.set_status(id, CallStatus::Done, None)?;
|
||||||
@@ -128,10 +132,9 @@ async fn transcribe(
|
|||||||
let out_flac = work_dir.join(format!("{id}-out.flac"));
|
let out_flac = work_dir.join(format!("{id}-out.flac"));
|
||||||
let _flac_files = RemoveOnDrop(vec![mic_flac.clone(), out_flac.clone()]);
|
let _flac_files = RemoveOnDrop(vec![mic_flac.clone(), out_flac.clone()]);
|
||||||
|
|
||||||
tokio::try_join!(
|
// One after the other: each encode holds a whole track in memory.
|
||||||
encode(&call.mic_path, &mic_flac),
|
encode(&call.mic_path, &mic_flac).await?;
|
||||||
encode(&call.out_path, &out_flac)
|
encode(&call.out_path, &out_flac).await?;
|
||||||
)?;
|
|
||||||
log::info!("call {id}: uploading both tracks");
|
log::info!("call {id}: uploading both tracks");
|
||||||
|
|
||||||
let track_opts = |single_speaker| TranscribeOptions {
|
let track_opts = |single_speaker| TranscribeOptions {
|
||||||
@@ -160,7 +163,8 @@ async fn transcribe(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Waits for both jobs at once, once both were submitted.
|
/// Waits for both jobs at once, once both were submitted. Returns as soon
|
||||||
|
/// as one of them fails, without waiting for the other.
|
||||||
async fn wait_both(
|
async fn wait_both(
|
||||||
tr: &dyn Transcriber,
|
tr: &dyn Transcriber,
|
||||||
id: CallId,
|
id: CallId,
|
||||||
@@ -169,8 +173,7 @@ async fn wait_both(
|
|||||||
) -> Result<(Vec<Word>, Vec<Word>), PipelineError> {
|
) -> Result<(Vec<Word>, Vec<Word>), PipelineError> {
|
||||||
let (mic_job, out_job) = (mic_job?, out_job?);
|
let (mic_job, out_job) = (mic_job?, out_job?);
|
||||||
log::info!("call {id}: waiting for remote jobs {mic_job} and {out_job}");
|
log::info!("call {id}: waiting for remote jobs {mic_job} and {out_job}");
|
||||||
let (mic, out) = tokio::join!(tr.wait(&mic_job), tr.wait(&out_job));
|
Ok(tokio::try_join!(tr.wait(&mic_job), tr.wait(&out_job))?)
|
||||||
Ok((mic?, out?))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Encodes `wav` to FLAC at `flac` on the blocking pool.
|
/// Encodes `wav` to FLAC at `flac` on the blocking pool.
|
||||||
@@ -186,7 +189,8 @@ async fn encode(wav: &Path, flac: &Path) -> Result<(), PipelineError> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Uploads one track and records its remote job right away, so a crash
|
/// Uploads one track and records its remote job right away, so a crash
|
||||||
/// mid-wait still leaves a record of what to delete remotely.
|
/// mid-wait still leaves a record of what to delete remotely. If recording
|
||||||
|
/// fails, the job id is still returned: the run deletes it at the end.
|
||||||
async fn submit(
|
async fn submit(
|
||||||
store: &Store,
|
store: &Store,
|
||||||
tr: &dyn Transcriber,
|
tr: &dyn Transcriber,
|
||||||
@@ -196,12 +200,15 @@ async fn submit(
|
|||||||
opts: &TranscribeOptions,
|
opts: &TranscribeOptions,
|
||||||
) -> Result<String, PipelineError> {
|
) -> Result<String, PipelineError> {
|
||||||
let remote_id = tr.submit(flac, opts).await?;
|
let remote_id = tr.submit(flac, opts).await?;
|
||||||
store.record_remote_job(id, track, &remote_id)?;
|
if let Err(e) = store.record_remote_job(id, track, &remote_id) {
|
||||||
|
log::error!("call {id}: could not record remote job {remote_id} ({track}): {e}");
|
||||||
|
}
|
||||||
Ok(remote_id)
|
Ok(remote_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Best-effort deletes the remote jobs recorded for call `id` (left over
|
/// Best-effort deletes the remote jobs recorded for call `id` (left over
|
||||||
/// from a run the app did not finish), then forgets them.
|
/// from a run the app did not finish, or whose delete failed), then forgets
|
||||||
|
/// them. Does nothing if none are recorded.
|
||||||
pub async fn delete_remote_jobs(store: &Store, tr: &dyn Transcriber, id: CallId) {
|
pub async fn delete_remote_jobs(store: &Store, tr: &dyn Transcriber, id: CallId) {
|
||||||
delete_jobs(store, tr, id, &[]).await;
|
delete_jobs(store, tr, id, &[]).await;
|
||||||
}
|
}
|
||||||
@@ -222,6 +229,9 @@ async fn delete_jobs(store: &Store, tr: &dyn Transcriber, id: CallId, known: &[S
|
|||||||
jobs.push(job.clone());
|
jobs.push(job.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if jobs.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let mut all_deleted = true;
|
let mut all_deleted = true;
|
||||||
for job in &jobs {
|
for job in &jobs {
|
||||||
@@ -261,8 +271,8 @@ impl Drop for RemoveOnDrop {
|
|||||||
/// plus the shorter track's length (a missing or unrepairable track counts
|
/// plus the shorter track's length (a missing or unrepairable track counts
|
||||||
/// as empty).
|
/// as empty).
|
||||||
///
|
///
|
||||||
/// The caller is responsible for deleting the remote jobs of the calls
|
/// The caller is responsible for deleting the remote jobs still on record
|
||||||
/// reported as `failed` (see [`delete_remote_jobs`]).
|
/// (see [`Store::calls_with_remote_jobs`] and [`delete_remote_jobs`]).
|
||||||
pub fn recover_on_startup(store: &Store) -> Result<RecoveryReport, PipelineError> {
|
pub fn recover_on_startup(store: &Store) -> Result<RecoveryReport, PipelineError> {
|
||||||
let report = store.startup_recovery()?;
|
let report = store.startup_recovery()?;
|
||||||
for &id in &report.interrupted {
|
for &id in &report.interrupted {
|
||||||
@@ -312,9 +322,16 @@ mod tests {
|
|||||||
struct FakeTranscriber {
|
struct FakeTranscriber {
|
||||||
results: Mutex<HashMap<String, Result<Vec<Word>, TranscribeError>>>,
|
results: Mutex<HashMap<String, Result<Vec<Word>, TranscribeError>>>,
|
||||||
submitted: Mutex<Vec<(String, TranscribeOptions)>>,
|
submitted: Mutex<Vec<(String, TranscribeOptions)>>,
|
||||||
|
/// Every delete attempt, also the failed ones.
|
||||||
deleted: Mutex<Vec<String>>,
|
deleted: Mutex<Vec<String>>,
|
||||||
/// Job whose upload fails with `Rejected`.
|
/// Job whose upload fails with `Rejected`.
|
||||||
reject_upload: Option<&'static str>,
|
reject_upload: Option<&'static str>,
|
||||||
|
/// Job whose `wait` takes an hour before it returns.
|
||||||
|
slow_wait: Option<&'static str>,
|
||||||
|
/// How many of the next delete attempts fail.
|
||||||
|
failing_deletes: Mutex<usize>,
|
||||||
|
/// Runs at the start of every `submit`.
|
||||||
|
on_submit: Option<Box<dyn Fn() + Send + Sync>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FakeTranscriber {
|
impl FakeTranscriber {
|
||||||
@@ -349,6 +366,9 @@ mod tests {
|
|||||||
audio: &Path,
|
audio: &Path,
|
||||||
opts: &TranscribeOptions,
|
opts: &TranscribeOptions,
|
||||||
) -> Result<String, TranscribeError> {
|
) -> Result<String, TranscribeError> {
|
||||||
|
if let Some(hook) = &self.on_submit {
|
||||||
|
hook();
|
||||||
|
}
|
||||||
let bytes = fs::read(audio).expect("submitted file exists");
|
let bytes = fs::read(audio).expect("submitted file exists");
|
||||||
assert_eq!(&bytes[0..4], b"fLaC", "uploads are FLAC");
|
assert_eq!(&bytes[0..4], b"fLaC", "uploads are FLAC");
|
||||||
let name = audio.file_name().unwrap().to_string_lossy();
|
let name = audio.file_name().unwrap().to_string_lossy();
|
||||||
@@ -368,6 +388,9 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn wait(&self, remote_id: &str) -> Result<Vec<Word>, TranscribeError> {
|
async fn wait(&self, remote_id: &str) -> Result<Vec<Word>, TranscribeError> {
|
||||||
|
if self.slow_wait == Some(remote_id) {
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
|
||||||
|
}
|
||||||
self.results
|
self.results
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@@ -378,6 +401,11 @@ mod tests {
|
|||||||
|
|
||||||
async fn delete(&self, remote_id: &str) -> Result<(), TranscribeError> {
|
async fn delete(&self, remote_id: &str) -> Result<(), TranscribeError> {
|
||||||
self.deleted.lock().unwrap().push(remote_id.to_string());
|
self.deleted.lock().unwrap().push(remote_id.to_string());
|
||||||
|
let mut failing = self.failing_deletes.lock().unwrap();
|
||||||
|
if *failing > 0 {
|
||||||
|
*failing -= 1;
|
||||||
|
return Err(TranscribeError::Transport("connection reset".to_string()));
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -408,6 +436,10 @@ mod tests {
|
|||||||
writer.finalize().unwrap();
|
writer.finalize().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// File name of the fixture's database, so a test can open a second
|
||||||
|
/// connection to it.
|
||||||
|
const DB_FILE: &str = "calls.sqlite";
|
||||||
|
|
||||||
struct Fixture {
|
struct Fixture {
|
||||||
store: Store,
|
store: Store,
|
||||||
dir: tempfile::TempDir,
|
dir: tempfile::TempDir,
|
||||||
@@ -438,9 +470,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// A call awaiting the decision, with one second of audio per track.
|
/// A call awaiting the decision, with one second of audio per track.
|
||||||
|
/// The database is a file in the temp dir (see [`DB_FILE`]).
|
||||||
fn fixture() -> Fixture {
|
fn fixture() -> Fixture {
|
||||||
let store = Store::open_in_memory().unwrap();
|
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let store = Store::open(&dir.path().join(DB_FILE)).unwrap();
|
||||||
let mic = dir.path().join("1-mic.wav");
|
let mic = dir.path().join("1-mic.wav");
|
||||||
let out = dir.path().join("1-out.wav");
|
let out = dir.path().join("1-out.wav");
|
||||||
write_wav(&mic, 1.0);
|
write_wav(&mic, 1.0);
|
||||||
@@ -556,6 +589,102 @@ mod tests {
|
|||||||
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
|
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pipeline_deletes_a_job_it_could_not_record() {
|
||||||
|
let f = fixture();
|
||||||
|
// The call row vanishes during the upload, so recording the remote
|
||||||
|
// jobs fails (foreign key).
|
||||||
|
let (db, id) = (f.dir.path().join(DB_FILE), f.id);
|
||||||
|
let fake = FakeTranscriber {
|
||||||
|
on_submit: Some(Box::new(move || {
|
||||||
|
Store::open(&db).unwrap().delete_call(id).unwrap();
|
||||||
|
})),
|
||||||
|
..FakeTranscriber::returning(vec![], vec![])
|
||||||
|
};
|
||||||
|
|
||||||
|
// How the run ends for a vanished call does not matter here.
|
||||||
|
let _ = f.process(&fake).await;
|
||||||
|
|
||||||
|
assert_eq!(fake.submitted().len(), 2);
|
||||||
|
assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]);
|
||||||
|
assert_eq!(f.upload_files(), Vec::<PathBuf>::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn pipeline_fails_fast_when_one_wait_fails() {
|
||||||
|
let f = fixture();
|
||||||
|
let fake = FakeTranscriber {
|
||||||
|
slow_wait: Some("job-mic"),
|
||||||
|
..FakeTranscriber::returning(vec![], vec![])
|
||||||
|
};
|
||||||
|
fake.script(
|
||||||
|
"job-out",
|
||||||
|
Err(TranscribeError::Failed("kaputt".to_string())),
|
||||||
|
);
|
||||||
|
let started = tokio::time::Instant::now();
|
||||||
|
|
||||||
|
let err = f.process(&fake).await.unwrap_err();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
started.elapsed() < std::time::Duration::from_secs(60),
|
||||||
|
"waited {:?} for the other job",
|
||||||
|
started.elapsed()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
err.user_message(),
|
||||||
|
TranscribeError::Failed("kaputt".to_string()).user_message()
|
||||||
|
);
|
||||||
|
assert_eq!(f.status(), CallStatus::Failed);
|
||||||
|
// The job still in flight is deleted (cancelled) as well.
|
||||||
|
assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]);
|
||||||
|
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_delete_is_retried_by_the_next_run() {
|
||||||
|
let f = fixture();
|
||||||
|
let fake = FakeTranscriber {
|
||||||
|
failing_deletes: Mutex::new(1),
|
||||||
|
..FakeTranscriber::returning(vec![word("Hallo", 0.0, 0.5, None)], vec![])
|
||||||
|
};
|
||||||
|
|
||||||
|
f.process(&fake).await.unwrap();
|
||||||
|
|
||||||
|
// The transcript is in, but a job could not be deleted: both stay
|
||||||
|
// on record, also for the cleanup at the next start.
|
||||||
|
assert_eq!(f.status(), CallStatus::Done);
|
||||||
|
assert_eq!(f.store.remote_jobs(f.id).unwrap().len(), 2);
|
||||||
|
assert_eq!(f.store.calls_with_remote_jobs().unwrap(), vec![f.id]);
|
||||||
|
|
||||||
|
// Processing the call again only cleans up.
|
||||||
|
f.process(&fake).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(f.status(), CallStatus::Done);
|
||||||
|
assert_eq!(fake.submitted().len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
fake.deleted(),
|
||||||
|
vec!["job-mic", "job-mic", "job-out", "job-out"]
|
||||||
|
);
|
||||||
|
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
|
||||||
|
assert_eq!(
|
||||||
|
f.store.calls_with_remote_jobs().unwrap(),
|
||||||
|
Vec::<CallId>::new()
|
||||||
|
);
|
||||||
|
assert_eq!(transcript(&f), vec!["Ich: Hallo".to_string()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn skipping_transcription_deletes_nothing_without_leftovers() {
|
||||||
|
let f = fixture();
|
||||||
|
f.store.mark_transcribed(f.id, Local::now()).unwrap();
|
||||||
|
let fake = FakeTranscriber::default();
|
||||||
|
|
||||||
|
f.process(&fake).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(f.status(), CallStatus::Done);
|
||||||
|
assert_eq!(fake.deleted(), Vec::<String>::new());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn leftover_remote_jobs_are_deleted_and_forgotten() {
|
async fn leftover_remote_jobs_are_deleted_and_forgotten() {
|
||||||
let f = fixture();
|
let f = fixture();
|
||||||
|
|||||||
@@ -360,6 +360,20 @@ impl Store {
|
|||||||
Ok(jobs)
|
Ok(jobs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Calls that still have remote jobs on record (their delete failed or
|
||||||
|
/// never ran), ordered by id.
|
||||||
|
pub fn calls_with_remote_jobs(&self) -> Result<Vec<CallId>> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt =
|
||||||
|
conn.prepare("SELECT DISTINCT call_id FROM remote_jobs ORDER BY call_id ASC")?;
|
||||||
|
let mut rows = stmt.query([])?;
|
||||||
|
let mut ids = Vec::new();
|
||||||
|
while let Some(row) = rows.next()? {
|
||||||
|
ids.push(row.get(0)?);
|
||||||
|
}
|
||||||
|
Ok(ids)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn clear_remote_jobs(&self, id: CallId) -> Result<()> {
|
pub fn clear_remote_jobs(&self, id: CallId) -> Result<()> {
|
||||||
let conn = self.conn.lock().unwrap();
|
let conn = self.conn.lock().unwrap();
|
||||||
conn.execute("DELETE FROM remote_jobs WHERE call_id = ?1", params![id])?;
|
conn.execute("DELETE FROM remote_jobs WHERE call_id = ?1", params![id])?;
|
||||||
@@ -822,4 +836,27 @@ mod tests {
|
|||||||
assert_eq!(store.speaker_names(id).unwrap(), BTreeMap::new());
|
assert_eq!(store.speaker_names(id).unwrap(), BTreeMap::new());
|
||||||
assert_eq!(store.remote_jobs(id).unwrap(), Vec::new());
|
assert_eq!(store.remote_jobs(id).unwrap(), Vec::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn calls_with_remote_jobs_lists_each_call_once() {
|
||||||
|
let store = Store::open_in_memory().unwrap();
|
||||||
|
let call = |app| {
|
||||||
|
store
|
||||||
|
.create_call(app, Local::now(), Path::new(""), Path::new(""))
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
let (a, _without_jobs, c) = (call("a"), call("b"), call("c"));
|
||||||
|
assert_eq!(
|
||||||
|
store.calls_with_remote_jobs().unwrap(),
|
||||||
|
Vec::<CallId>::new()
|
||||||
|
);
|
||||||
|
|
||||||
|
store.record_remote_job(c, "mic", "job-c").unwrap();
|
||||||
|
store.record_remote_job(a, "mic", "job-a1").unwrap();
|
||||||
|
store.record_remote_job(a, "out", "job-a2").unwrap();
|
||||||
|
assert_eq!(store.calls_with_remote_jobs().unwrap(), vec![a, c]);
|
||||||
|
|
||||||
|
store.clear_remote_jobs(a).unwrap();
|
||||||
|
assert_eq!(store.calls_with_remote_jobs().unwrap(), vec![c]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ use super::{TranscribeError, TranscribeOptions, Transcriber};
|
|||||||
const DEFAULT_BASE_URL: &str = "https://api.deepscript.com/v1";
|
const DEFAULT_BASE_URL: &str = "https://api.deepscript.com/v1";
|
||||||
const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(5);
|
const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(5);
|
||||||
const DEFAULT_MAX_WAIT: Duration = Duration::from_secs(60 * 60);
|
const DEFAULT_MAX_WAIT: Duration = Duration::from_secs(60 * 60);
|
||||||
|
/// How many status polls in a row may fail transiently (network, timeout,
|
||||||
|
/// HTTP 429/5xx) before `wait` gives up.
|
||||||
|
const MAX_POLL_RETRIES: u32 = 3;
|
||||||
|
|
||||||
/// DeepScript client. Build with [`DeepScript::new`]; the `with_*` builders
|
/// DeepScript client. Build with [`DeepScript::new`]; the `with_*` builders
|
||||||
/// exist so tests can shrink `poll_interval`/`max_wait` and point
|
/// exist so tests can shrink `poll_interval`/`max_wait` and point
|
||||||
@@ -73,12 +76,37 @@ impl DeepScript {
|
|||||||
/// A non-2xx response is *not* an error here; callers inspect
|
/// A non-2xx response is *not* an error here; callers inspect
|
||||||
/// `resp.status` themselves.
|
/// `resp.status` themselves.
|
||||||
async fn send(&self, req: HttpRequest) -> Result<HttpResponse, TranscribeError> {
|
async fn send(&self, req: HttpRequest) -> Result<HttpResponse, TranscribeError> {
|
||||||
self.transport.send(req).await.map_err(|e| match e {
|
self.transport.send(req).await.map_err(transport_error)
|
||||||
TransportError::Timeout => TranscribeError::Timeout,
|
}
|
||||||
TransportError::Io(msg) | TransportError::Network(msg) => {
|
|
||||||
TranscribeError::Transport(msg)
|
/// Asks for the status of `remote_id` once.
|
||||||
|
async fn poll_status(&self, remote_id: &str) -> Result<String, PollError> {
|
||||||
|
let req = HttpRequest {
|
||||||
|
method: Method::Get,
|
||||||
|
url: format!("{}/transcriptions/{remote_id}/status", self.base_url),
|
||||||
|
headers: vec![self.auth_header()],
|
||||||
|
body: HttpBody::Empty,
|
||||||
|
};
|
||||||
|
let resp = match self.transport.send(req).await {
|
||||||
|
Ok(resp) => resp,
|
||||||
|
Err(e @ (TransportError::Network(_) | TransportError::Timeout)) => {
|
||||||
|
return Err(PollError::Transient(transport_error(e)))
|
||||||
}
|
}
|
||||||
})
|
Err(e) => return Err(PollError::Fatal(transport_error(e))),
|
||||||
|
};
|
||||||
|
match resp.status {
|
||||||
|
200 => resp
|
||||||
|
.json::<DataEnvelope<StatusData>>()
|
||||||
|
.map(|envelope| envelope.data.status)
|
||||||
|
.map_err(|e| {
|
||||||
|
PollError::Fatal(TranscribeError::Failed(format!("ungültige Antwort: {e}")))
|
||||||
|
}),
|
||||||
|
429 | 500..=599 => Err(PollError::Transient(error_for_status(
|
||||||
|
resp.status,
|
||||||
|
&resp.body,
|
||||||
|
))),
|
||||||
|
other => Err(PollError::Fatal(error_for_status(other, &resp.body))),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_detail(&self, remote_id: &str) -> Result<DetailData, TranscribeError> {
|
async fn fetch_detail(&self, remote_id: &str) -> Result<DetailData, TranscribeError> {
|
||||||
@@ -191,26 +219,20 @@ impl Transcriber for DeepScript {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Polls until the job is done. A poll that fails transiently is retried
|
||||||
|
/// at the normal interval, up to [`MAX_POLL_RETRIES`] times in a row.
|
||||||
async fn wait(&self, remote_id: &str) -> Result<Vec<Word>, TranscribeError> {
|
async fn wait(&self, remote_id: &str) -> Result<Vec<Word>, TranscribeError> {
|
||||||
let deadline = Instant::now() + self.max_wait;
|
let deadline = Instant::now() + self.max_wait;
|
||||||
|
let mut failed_polls = 0;
|
||||||
loop {
|
loop {
|
||||||
if Instant::now() >= deadline {
|
if Instant::now() >= deadline {
|
||||||
return Err(TranscribeError::Timeout);
|
return Err(TranscribeError::Timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
let req = HttpRequest {
|
match self.poll_status(remote_id).await {
|
||||||
method: Method::Get,
|
Ok(status) => {
|
||||||
url: format!("{}/transcriptions/{remote_id}/status", self.base_url),
|
failed_polls = 0;
|
||||||
headers: vec![self.auth_header()],
|
match status.as_str() {
|
||||||
body: HttpBody::Empty,
|
|
||||||
};
|
|
||||||
let resp = self.send(req).await?;
|
|
||||||
match resp.status {
|
|
||||||
200 => {
|
|
||||||
let envelope: DataEnvelope<StatusData> = resp
|
|
||||||
.json()
|
|
||||||
.map_err(|e| TranscribeError::Failed(format!("ungültige Antwort: {e}")))?;
|
|
||||||
match envelope.data.status.as_str() {
|
|
||||||
"completed" => return self.collect_words(remote_id).await,
|
"completed" => return self.collect_words(remote_id).await,
|
||||||
"failed" => return self.collect_failure(remote_id).await,
|
"failed" => return self.collect_failure(remote_id).await,
|
||||||
"queued" | "processing" => {}
|
"queued" | "processing" => {}
|
||||||
@@ -223,7 +245,14 @@ impl Transcriber for DeepScript {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
other => return Err(error_for_status(other, &resp.body)),
|
Err(PollError::Transient(e)) if failed_polls < MAX_POLL_RETRIES => {
|
||||||
|
failed_polls += 1;
|
||||||
|
log::warn!(
|
||||||
|
"DeepScript: Statusabfrage für {remote_id} fehlgeschlagen ({}), Versuch {failed_polls} von {MAX_POLL_RETRIES}",
|
||||||
|
e.user_message()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(PollError::Transient(e) | PollError::Fatal(e)) => return Err(e),
|
||||||
}
|
}
|
||||||
|
|
||||||
tokio::time::sleep(self.poll_interval).await;
|
tokio::time::sleep(self.poll_interval).await;
|
||||||
@@ -245,6 +274,20 @@ impl Transcriber for DeepScript {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A failed status poll: `Transient` ones (network, timeout, HTTP 429/5xx)
|
||||||
|
/// are worth retrying.
|
||||||
|
enum PollError {
|
||||||
|
Transient(TranscribeError),
|
||||||
|
Fatal(TranscribeError),
|
||||||
|
}
|
||||||
|
|
||||||
|
fn transport_error(e: TransportError) -> TranscribeError {
|
||||||
|
match e {
|
||||||
|
TransportError::Timeout => TranscribeError::Timeout,
|
||||||
|
TransportError::Io(msg) | TransportError::Network(msg) => TranscribeError::Transport(msg),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Maps a non-2xx status that isn't handled specially by the caller (401,
|
/// Maps a non-2xx status that isn't handled specially by the caller (401,
|
||||||
/// 429, and a generic fallback carrying the problem body's `detail`/`title`
|
/// 429, and a generic fallback carrying the problem body's `detail`/`title`
|
||||||
/// — the set shared by every DeepScript endpoint).
|
/// — the set shared by every DeepScript endpoint).
|
||||||
@@ -600,6 +643,72 @@ mod tests {
|
|||||||
transport.assert_done();
|
transport.assert_done();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn wait_retries_a_transient_status_error() {
|
||||||
|
let transport = Arc::new(FakeTransport::new());
|
||||||
|
let status_url = format!("{BASE}/transcriptions/job-8/status");
|
||||||
|
transport.expect(Method::Get, &status_url, 503, "");
|
||||||
|
transport.expect(
|
||||||
|
Method::Get,
|
||||||
|
&status_url,
|
||||||
|
200,
|
||||||
|
r#"{"data":{"id":"job-8","status":"completed"}}"#,
|
||||||
|
);
|
||||||
|
transport.expect(
|
||||||
|
Method::Get,
|
||||||
|
&format!("{BASE}/transcriptions/job-8"),
|
||||||
|
200,
|
||||||
|
r#"{"data":{"id":"job-8","status":"completed","resultJson":{"words":[]}}}"#,
|
||||||
|
);
|
||||||
|
let ds = deep_script(&transport);
|
||||||
|
|
||||||
|
let words = ds.wait("job-8").await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(words, Vec::<Word>::new());
|
||||||
|
transport.assert_done();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn wait_gives_up_after_four_transient_errors_in_a_row() {
|
||||||
|
let transport = Arc::new(FakeTransport::new());
|
||||||
|
let status_url = format!("{BASE}/transcriptions/job-9/status");
|
||||||
|
// A successful poll in between resets the count.
|
||||||
|
for status in [503, 429, 200, 502, 503, 500, 503] {
|
||||||
|
let body = if status == 200 {
|
||||||
|
r#"{"data":{"id":"job-9","status":"processing"}}"#
|
||||||
|
} else {
|
||||||
|
r#"{"title":"Service Unavailable"}"#
|
||||||
|
};
|
||||||
|
transport.expect(Method::Get, &status_url, status, body);
|
||||||
|
}
|
||||||
|
let ds = deep_script(&transport);
|
||||||
|
|
||||||
|
let err = ds.wait("job-9").await.unwrap_err();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
err,
|
||||||
|
TranscribeError::Failed("HTTP 503: Service Unavailable".to_string())
|
||||||
|
);
|
||||||
|
transport.assert_done();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn wait_does_not_retry_a_client_error() {
|
||||||
|
let transport = Arc::new(FakeTransport::new());
|
||||||
|
transport.expect(
|
||||||
|
Method::Get,
|
||||||
|
&format!("{BASE}/transcriptions/job-10/status"),
|
||||||
|
401,
|
||||||
|
"",
|
||||||
|
);
|
||||||
|
let ds = deep_script(&transport);
|
||||||
|
|
||||||
|
let err = ds.wait("job-10").await.unwrap_err();
|
||||||
|
|
||||||
|
assert_eq!(err, TranscribeError::Unauthorized);
|
||||||
|
transport.assert_done();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn wait_times_out_after_max_wait() {
|
async fn wait_times_out_after_max_wait() {
|
||||||
let transport = Arc::new(FakeTransport::new());
|
let transport = Arc::new(FakeTransport::new());
|
||||||
|
|||||||
@@ -62,10 +62,16 @@ pub fn run() {
|
|||||||
config_dir.join("settings.json"),
|
config_dir.join("settings.json"),
|
||||||
data_dir.join("recordings"),
|
data_dir.join("recordings"),
|
||||||
);
|
);
|
||||||
|
let leftover_jobs = state.store.calls_with_remote_jobs().unwrap_or_else(|e| {
|
||||||
|
log::error!("could not read leftover remote jobs: {e}");
|
||||||
|
Vec::new()
|
||||||
|
});
|
||||||
let autostart = state.settings().autostart;
|
let autostart = state.settings().autostart;
|
||||||
|
// Before anything can start a run.
|
||||||
|
processing::clear_upload_dir(app.handle());
|
||||||
app.manage(state);
|
app.manage(state);
|
||||||
commands::apply_autostart(app.handle(), autostart);
|
commands::apply_autostart(app.handle(), autostart);
|
||||||
processing::delete_leftover_jobs(app.handle(), recovery.failed);
|
processing::delete_leftover_jobs(app.handle(), leftover_jobs);
|
||||||
|
|
||||||
tray::create(app.handle())?;
|
tray::create(app.handle())?;
|
||||||
detection::spawn(app.handle().clone());
|
detection::spawn(app.handle().clone());
|
||||||
|
|||||||
+43
-11
@@ -3,6 +3,9 @@
|
|||||||
//! system notification when a run ends. Notifications never contain
|
//! system notification when a run ends. Notifications never contain
|
||||||
//! transcript text, and the API key is never logged.
|
//! transcript text, and the API key is never logged.
|
||||||
|
|
||||||
|
use std::io::ErrorKind;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
use recorder_core::model::{CallId, CallStatus};
|
use recorder_core::model::{CallId, CallStatus};
|
||||||
use recorder_core::pipeline::{self, PipelineOptions};
|
use recorder_core::pipeline::{self, PipelineOptions};
|
||||||
use recorder_core::secrets::Service;
|
use recorder_core::secrets::Service;
|
||||||
@@ -36,6 +39,16 @@ impl Claim {
|
|||||||
id,
|
id,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reserves the call for a cleanup ([`AppState::reserve_processing`]).
|
||||||
|
fn for_cleanup(app: &AppHandle, id: CallId) -> Option<Self> {
|
||||||
|
app.state::<AppState>()
|
||||||
|
.reserve_processing(id)
|
||||||
|
.then(|| Self {
|
||||||
|
app: app.clone(),
|
||||||
|
id,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for Claim {
|
impl Drop for Claim {
|
||||||
@@ -60,11 +73,7 @@ pub fn start(app: &AppHandle, id: CallId, retry: bool) -> Result<(), String> {
|
|||||||
emit_updated(app, id);
|
emit_updated(app, id);
|
||||||
return Err(msg);
|
return Err(msg);
|
||||||
};
|
};
|
||||||
let work_dir = app
|
let work_dir = upload_dir(app)?;
|
||||||
.path()
|
|
||||||
.app_cache_dir()
|
|
||||||
.map_err(|e| format!("Cache-Ordner nicht gefunden: {e}"))?
|
|
||||||
.join("upload");
|
|
||||||
let settings = state.settings();
|
let settings = state.settings();
|
||||||
let opts = PipelineOptions {
|
let opts = PipelineOptions {
|
||||||
language: settings.language,
|
language: settings.language,
|
||||||
@@ -95,11 +104,34 @@ pub fn start(app: &AppHandle, id: CallId, retry: bool) -> Result<(), String> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Best-effort deletes the DeepScript jobs of calls whose processing the
|
/// Where the pipeline puts the FLAC copies it uploads.
|
||||||
/// last run of the app did not finish ([`RecoveryReport::failed`]), in the
|
fn upload_dir(app: &AppHandle) -> Result<PathBuf, String> {
|
||||||
|
app.path()
|
||||||
|
.app_cache_dir()
|
||||||
|
.map(|dir| dir.join("upload"))
|
||||||
|
.map_err(|e| format!("Cache-Ordner nicht gefunden: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Removes FLAC copies left in the upload directory by a run the app did
|
||||||
|
/// not finish. Call before any run can start.
|
||||||
|
pub fn clear_upload_dir(app: &AppHandle) {
|
||||||
|
let dir = match upload_dir(app) {
|
||||||
|
Ok(dir) => dir,
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!("{e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match std::fs::remove_dir_all(&dir) {
|
||||||
|
Ok(()) => log::info!("removed leftover uploads in {}", dir.display()),
|
||||||
|
Err(e) if e.kind() == ErrorKind::NotFound => {}
|
||||||
|
Err(e) => log::warn!("could not empty {}: {e}", dir.display()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Best-effort deletes the DeepScript jobs still on record for `calls`
|
||||||
|
/// (from runs the app did not finish, or whose delete failed), in the
|
||||||
/// background. Skipped without a key.
|
/// background. Skipped without a key.
|
||||||
///
|
|
||||||
/// [`RecoveryReport::failed`]: recorder_core::store::RecoveryReport::failed
|
|
||||||
pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec<CallId>) {
|
pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec<CallId>) {
|
||||||
if calls.is_empty() {
|
if calls.is_empty() {
|
||||||
return;
|
return;
|
||||||
@@ -117,8 +149,8 @@ pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec<CallId>) {
|
|||||||
let state = app.state::<AppState>();
|
let state = app.state::<AppState>();
|
||||||
let transcriber = DeepScript::new(state.transport.clone(), key);
|
let transcriber = DeepScript::new(state.transport.clone(), key);
|
||||||
for id in calls {
|
for id in calls {
|
||||||
// A retry the user started meanwhile cleans up itself.
|
// A run the user started meanwhile cleans up itself.
|
||||||
let Ok(_claim) = Claim::take(&app, id, true) else {
|
let Some(_claim) = Claim::for_cleanup(&app, id) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
pipeline::delete_remote_jobs(&state.store, &transcriber, id).await;
|
pipeline::delete_remote_jobs(&state.store, &transcriber, id).await;
|
||||||
|
|||||||
+69
-2
@@ -400,7 +400,7 @@ impl AppState {
|
|||||||
|
|
||||||
/// Deletes a finished call: both audio files and the row, and takes it
|
/// Deletes a finished call: both audio files and the row, and takes it
|
||||||
/// out of the prompt queue. Returns how many calls still wait for the
|
/// out of the prompt queue. Returns how many calls still wait for the
|
||||||
/// prompt.
|
/// prompt. A call that is being processed is not deleted.
|
||||||
pub fn discard_call(&self, id: CallId) -> Result<usize, String> {
|
pub fn discard_call(&self, id: CallId) -> Result<usize, String> {
|
||||||
if lock(&self.current)
|
if lock(&self.current)
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -408,11 +408,19 @@ impl AppState {
|
|||||||
{
|
{
|
||||||
return Err("Das Gespräch wird gerade aufgenommen".to_string());
|
return Err("Das Gespräch wird gerade aufgenommen".to_string());
|
||||||
}
|
}
|
||||||
|
// Held until the call is gone, so no run can claim it meanwhile.
|
||||||
|
let processing = lock(&self.processing);
|
||||||
|
if processing.contains(&id) {
|
||||||
|
return Err(ALREADY_PROCESSING.to_string());
|
||||||
|
}
|
||||||
let call = self
|
let call = self
|
||||||
.store
|
.store
|
||||||
.get_call(id)
|
.get_call(id)
|
||||||
.map_err(|e| e.to_string())?
|
.map_err(|e| e.to_string())?
|
||||||
.ok_or_else(|| "Gespräch nicht gefunden".to_string())?;
|
.ok_or_else(|| "Gespräch nicht gefunden".to_string())?;
|
||||||
|
if call.status == CallStatus::Processing {
|
||||||
|
return Err(ALREADY_PROCESSING.to_string());
|
||||||
|
}
|
||||||
// Files first: if one cannot be deleted, the row stays so the user
|
// Files first: if one cannot be deleted, the row stays so the user
|
||||||
// can try again instead of leaving orphaned audio behind.
|
// can try again instead of leaving orphaned audio behind.
|
||||||
for path in [&call.mic_path, &call.out_path] {
|
for path in [&call.mic_path, &call.out_path] {
|
||||||
@@ -438,6 +446,7 @@ impl AppState {
|
|||||||
}
|
}
|
||||||
return Err(format!("Gespräch konnte nicht gelöscht werden: {e}"));
|
return Err(format!("Gespräch konnte nicht gelöscht werden: {e}"));
|
||||||
}
|
}
|
||||||
|
drop(processing);
|
||||||
Ok(self.prompts_waiting())
|
Ok(self.prompts_waiting())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -470,7 +479,15 @@ impl AppState {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ends the pipeline run claimed by [`AppState::claim_processing`].
|
/// Reserves `id` for a cleanup of its remote jobs, whatever its status;
|
||||||
|
/// `false` if a run or cleanup already has it. Release it with
|
||||||
|
/// [`AppState::release_processing`].
|
||||||
|
pub fn reserve_processing(&self, id: CallId) -> bool {
|
||||||
|
lock(&self.processing).insert(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ends the pipeline run claimed by [`AppState::claim_processing`] (or
|
||||||
|
/// the cleanup reserved by [`AppState::reserve_processing`]).
|
||||||
pub fn release_processing(&self, id: CallId) {
|
pub fn release_processing(&self, id: CallId) {
|
||||||
lock(&self.processing).remove(&id);
|
lock(&self.processing).remove(&id);
|
||||||
}
|
}
|
||||||
@@ -707,6 +724,56 @@ mod tests {
|
|||||||
assert_eq!(f.state.prompts_waiting(), 0);
|
assert_eq!(f.state.prompts_waiting(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_call_being_processed_is_not_discarded() {
|
||||||
|
let f = fixture();
|
||||||
|
let (id, mic, out) = finished_call(&f, "a");
|
||||||
|
f.state.queue_prompt(id);
|
||||||
|
f.state.claim_processing(id, false).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
f.state.discard_call(id),
|
||||||
|
Err("Wird bereits verarbeitet".to_string())
|
||||||
|
);
|
||||||
|
f.state.release_processing(id);
|
||||||
|
f.state
|
||||||
|
.store
|
||||||
|
.set_status(id, CallStatus::Processing, None)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
f.state.discard_call(id),
|
||||||
|
Err("Wird bereits verarbeitet".to_string())
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(mic.exists());
|
||||||
|
assert!(out.exists());
|
||||||
|
assert!(f.state.store.get_call(id).unwrap().is_some());
|
||||||
|
assert_eq!(f.state.prompts_waiting(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_cleanup_reserves_a_call_of_any_status() {
|
||||||
|
let f = fixture();
|
||||||
|
let (id, _, _) = finished_call(&f, "a");
|
||||||
|
f.state
|
||||||
|
.store
|
||||||
|
.set_status(id, CallStatus::Done, None)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(f.state.reserve_processing(id));
|
||||||
|
assert!(!f.state.reserve_processing(id));
|
||||||
|
f.state
|
||||||
|
.store
|
||||||
|
.set_status(id, CallStatus::Failed, Some("Fehler"))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
f.state.claim_processing(id, true),
|
||||||
|
Err("Wird bereits verarbeitet".to_string())
|
||||||
|
);
|
||||||
|
f.state.release_processing(id);
|
||||||
|
assert!(f.state.reserve_processing(id));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_call_is_processed_once_at_a_time() {
|
fn a_call_is_processed_once_at_a_time() {
|
||||||
let f = fixture();
|
let f = fixture();
|
||||||
|
|||||||
Reference in New Issue
Block a user