diff --git a/crates/core/src/pipeline.rs b/crates/core/src/pipeline.rs index 29a7cec..0ba42b0 100644 --- a/crates/core/src/pipeline.rs +++ b/crates/core/src/pipeline.rs @@ -68,7 +68,8 @@ const OUT_TRACK: &str = "out"; /// Transcribes call `id` and stores the merged transcript, ending in /// `Done`. A call that already has a transcript (a retry after a later -/// step failed) is not transcribed again. +/// step failed) is not transcribed again; only remote jobs still on record +/// for it are deleted. /// /// Both tracks are encoded to FLAC in `work_dir` (removed again afterwards) /// and uploaded; the remote jobs are deleted once their results are in, or @@ -103,6 +104,9 @@ async fn run( if call.transcribed_at.is_none() && store.segments(id)?.is_empty() { store.set_status(id, CallStatus::Processing, None)?; transcribe(store, tr, &call, opts, work_dir).await?; + } else { + // Jobs whose delete failed in an earlier run. + delete_remote_jobs(store, tr, id).await; } // SUMMARY SEAM (Task 14b): summarize here. store.set_status(id, CallStatus::Done, None)?; @@ -128,10 +132,9 @@ async fn transcribe( let out_flac = work_dir.join(format!("{id}-out.flac")); let _flac_files = RemoveOnDrop(vec![mic_flac.clone(), out_flac.clone()]); - tokio::try_join!( - encode(&call.mic_path, &mic_flac), - encode(&call.out_path, &out_flac) - )?; + // One after the other: each encode holds a whole track in memory. + encode(&call.mic_path, &mic_flac).await?; + encode(&call.out_path, &out_flac).await?; log::info!("call {id}: uploading both tracks"); let track_opts = |single_speaker| TranscribeOptions { @@ -160,7 +163,8 @@ async fn transcribe( Ok(()) } -/// Waits for both jobs at once, once both were submitted. +/// Waits for both jobs at once, once both were submitted. Returns as soon +/// as one of them fails, without waiting for the other. async fn wait_both( tr: &dyn Transcriber, id: CallId, @@ -169,8 +173,7 @@ async fn wait_both( ) -> Result<(Vec, Vec), PipelineError> { let (mic_job, out_job) = (mic_job?, out_job?); log::info!("call {id}: waiting for remote jobs {mic_job} and {out_job}"); - let (mic, out) = tokio::join!(tr.wait(&mic_job), tr.wait(&out_job)); - Ok((mic?, out?)) + Ok(tokio::try_join!(tr.wait(&mic_job), tr.wait(&out_job))?) } /// Encodes `wav` to FLAC at `flac` on the blocking pool. @@ -186,7 +189,8 @@ async fn encode(wav: &Path, flac: &Path) -> Result<(), PipelineError> { } /// Uploads one track and records its remote job right away, so a crash -/// mid-wait still leaves a record of what to delete remotely. +/// mid-wait still leaves a record of what to delete remotely. If recording +/// fails, the job id is still returned: the run deletes it at the end. async fn submit( store: &Store, tr: &dyn Transcriber, @@ -196,12 +200,15 @@ async fn submit( opts: &TranscribeOptions, ) -> Result { let remote_id = tr.submit(flac, opts).await?; - store.record_remote_job(id, track, &remote_id)?; + if let Err(e) = store.record_remote_job(id, track, &remote_id) { + log::error!("call {id}: could not record remote job {remote_id} ({track}): {e}"); + } Ok(remote_id) } /// Best-effort deletes the remote jobs recorded for call `id` (left over -/// from a run the app did not finish), then forgets them. +/// from a run the app did not finish, or whose delete failed), then forgets +/// them. Does nothing if none are recorded. pub async fn delete_remote_jobs(store: &Store, tr: &dyn Transcriber, id: CallId) { delete_jobs(store, tr, id, &[]).await; } @@ -222,6 +229,9 @@ async fn delete_jobs(store: &Store, tr: &dyn Transcriber, id: CallId, known: &[S jobs.push(job.clone()); } } + if jobs.is_empty() { + return; + } let mut all_deleted = true; for job in &jobs { @@ -261,8 +271,8 @@ impl Drop for RemoveOnDrop { /// plus the shorter track's length (a missing or unrepairable track counts /// as empty). /// -/// The caller is responsible for deleting the remote jobs of the calls -/// reported as `failed` (see [`delete_remote_jobs`]). +/// The caller is responsible for deleting the remote jobs still on record +/// (see [`Store::calls_with_remote_jobs`] and [`delete_remote_jobs`]). pub fn recover_on_startup(store: &Store) -> Result { let report = store.startup_recovery()?; for &id in &report.interrupted { @@ -312,9 +322,16 @@ mod tests { struct FakeTranscriber { results: Mutex, TranscribeError>>>, submitted: Mutex>, + /// Every delete attempt, also the failed ones. deleted: Mutex>, /// Job whose upload fails with `Rejected`. reject_upload: Option<&'static str>, + /// Job whose `wait` takes an hour before it returns. + slow_wait: Option<&'static str>, + /// How many of the next delete attempts fail. + failing_deletes: Mutex, + /// Runs at the start of every `submit`. + on_submit: Option>, } impl FakeTranscriber { @@ -349,6 +366,9 @@ mod tests { audio: &Path, opts: &TranscribeOptions, ) -> Result { + if let Some(hook) = &self.on_submit { + hook(); + } let bytes = fs::read(audio).expect("submitted file exists"); assert_eq!(&bytes[0..4], b"fLaC", "uploads are FLAC"); let name = audio.file_name().unwrap().to_string_lossy(); @@ -368,6 +388,9 @@ mod tests { } async fn wait(&self, remote_id: &str) -> Result, TranscribeError> { + if self.slow_wait == Some(remote_id) { + tokio::time::sleep(std::time::Duration::from_secs(3600)).await; + } self.results .lock() .unwrap() @@ -378,6 +401,11 @@ mod tests { async fn delete(&self, remote_id: &str) -> Result<(), TranscribeError> { self.deleted.lock().unwrap().push(remote_id.to_string()); + let mut failing = self.failing_deletes.lock().unwrap(); + if *failing > 0 { + *failing -= 1; + return Err(TranscribeError::Transport("connection reset".to_string())); + } Ok(()) } } @@ -408,6 +436,10 @@ mod tests { writer.finalize().unwrap(); } + /// File name of the fixture's database, so a test can open a second + /// connection to it. + const DB_FILE: &str = "calls.sqlite"; + struct Fixture { store: Store, dir: tempfile::TempDir, @@ -438,9 +470,10 @@ mod tests { } /// A call awaiting the decision, with one second of audio per track. + /// The database is a file in the temp dir (see [`DB_FILE`]). fn fixture() -> Fixture { - let store = Store::open_in_memory().unwrap(); let dir = tempfile::tempdir().unwrap(); + let store = Store::open(&dir.path().join(DB_FILE)).unwrap(); let mic = dir.path().join("1-mic.wav"); let out = dir.path().join("1-out.wav"); write_wav(&mic, 1.0); @@ -556,6 +589,102 @@ mod tests { assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]); } + #[tokio::test] + async fn pipeline_deletes_a_job_it_could_not_record() { + let f = fixture(); + // The call row vanishes during the upload, so recording the remote + // jobs fails (foreign key). + let (db, id) = (f.dir.path().join(DB_FILE), f.id); + let fake = FakeTranscriber { + on_submit: Some(Box::new(move || { + Store::open(&db).unwrap().delete_call(id).unwrap(); + })), + ..FakeTranscriber::returning(vec![], vec![]) + }; + + // How the run ends for a vanished call does not matter here. + let _ = f.process(&fake).await; + + assert_eq!(fake.submitted().len(), 2); + assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]); + assert_eq!(f.upload_files(), Vec::::new()); + } + + #[tokio::test(start_paused = true)] + async fn pipeline_fails_fast_when_one_wait_fails() { + let f = fixture(); + let fake = FakeTranscriber { + slow_wait: Some("job-mic"), + ..FakeTranscriber::returning(vec![], vec![]) + }; + fake.script( + "job-out", + Err(TranscribeError::Failed("kaputt".to_string())), + ); + let started = tokio::time::Instant::now(); + + let err = f.process(&fake).await.unwrap_err(); + + assert!( + started.elapsed() < std::time::Duration::from_secs(60), + "waited {:?} for the other job", + started.elapsed() + ); + assert_eq!( + err.user_message(), + TranscribeError::Failed("kaputt".to_string()).user_message() + ); + assert_eq!(f.status(), CallStatus::Failed); + // The job still in flight is deleted (cancelled) as well. + assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]); + assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]); + } + + #[tokio::test] + async fn failed_delete_is_retried_by_the_next_run() { + let f = fixture(); + let fake = FakeTranscriber { + failing_deletes: Mutex::new(1), + ..FakeTranscriber::returning(vec![word("Hallo", 0.0, 0.5, None)], vec![]) + }; + + f.process(&fake).await.unwrap(); + + // The transcript is in, but a job could not be deleted: both stay + // on record, also for the cleanup at the next start. + assert_eq!(f.status(), CallStatus::Done); + assert_eq!(f.store.remote_jobs(f.id).unwrap().len(), 2); + assert_eq!(f.store.calls_with_remote_jobs().unwrap(), vec![f.id]); + + // Processing the call again only cleans up. + f.process(&fake).await.unwrap(); + + assert_eq!(f.status(), CallStatus::Done); + assert_eq!(fake.submitted().len(), 2); + assert_eq!( + fake.deleted(), + vec!["job-mic", "job-mic", "job-out", "job-out"] + ); + assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]); + assert_eq!( + f.store.calls_with_remote_jobs().unwrap(), + Vec::::new() + ); + assert_eq!(transcript(&f), vec!["Ich: Hallo".to_string()]); + } + + #[tokio::test] + async fn skipping_transcription_deletes_nothing_without_leftovers() { + let f = fixture(); + f.store.mark_transcribed(f.id, Local::now()).unwrap(); + let fake = FakeTranscriber::default(); + + f.process(&fake).await.unwrap(); + + assert_eq!(f.status(), CallStatus::Done); + assert_eq!(fake.deleted(), Vec::::new()); + } + #[tokio::test] async fn leftover_remote_jobs_are_deleted_and_forgotten() { let f = fixture(); diff --git a/crates/core/src/store.rs b/crates/core/src/store.rs index 6fd5ff0..60d4268 100644 --- a/crates/core/src/store.rs +++ b/crates/core/src/store.rs @@ -360,6 +360,20 @@ impl Store { Ok(jobs) } + /// Calls that still have remote jobs on record (their delete failed or + /// never ran), ordered by id. + pub fn calls_with_remote_jobs(&self) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = + conn.prepare("SELECT DISTINCT call_id FROM remote_jobs ORDER BY call_id ASC")?; + let mut rows = stmt.query([])?; + let mut ids = Vec::new(); + while let Some(row) = rows.next()? { + ids.push(row.get(0)?); + } + Ok(ids) + } + pub fn clear_remote_jobs(&self, id: CallId) -> Result<()> { let conn = self.conn.lock().unwrap(); conn.execute("DELETE FROM remote_jobs WHERE call_id = ?1", params![id])?; @@ -822,4 +836,27 @@ mod tests { assert_eq!(store.speaker_names(id).unwrap(), BTreeMap::new()); assert_eq!(store.remote_jobs(id).unwrap(), Vec::new()); } + + #[test] + fn calls_with_remote_jobs_lists_each_call_once() { + let store = Store::open_in_memory().unwrap(); + let call = |app| { + store + .create_call(app, Local::now(), Path::new(""), Path::new("")) + .unwrap() + }; + let (a, _without_jobs, c) = (call("a"), call("b"), call("c")); + assert_eq!( + store.calls_with_remote_jobs().unwrap(), + Vec::::new() + ); + + store.record_remote_job(c, "mic", "job-c").unwrap(); + store.record_remote_job(a, "mic", "job-a1").unwrap(); + store.record_remote_job(a, "out", "job-a2").unwrap(); + assert_eq!(store.calls_with_remote_jobs().unwrap(), vec![a, c]); + + store.clear_remote_jobs(a).unwrap(); + assert_eq!(store.calls_with_remote_jobs().unwrap(), vec![c]); + } } diff --git a/crates/core/src/transcribe/deepscript.rs b/crates/core/src/transcribe/deepscript.rs index d591692..66d10f6 100644 --- a/crates/core/src/transcribe/deepscript.rs +++ b/crates/core/src/transcribe/deepscript.rs @@ -20,6 +20,9 @@ use super::{TranscribeError, TranscribeOptions, Transcriber}; const DEFAULT_BASE_URL: &str = "https://api.deepscript.com/v1"; const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(5); const DEFAULT_MAX_WAIT: Duration = Duration::from_secs(60 * 60); +/// How many status polls in a row may fail transiently (network, timeout, +/// HTTP 429/5xx) before `wait` gives up. +const MAX_POLL_RETRIES: u32 = 3; /// DeepScript client. Build with [`DeepScript::new`]; the `with_*` builders /// exist so tests can shrink `poll_interval`/`max_wait` and point @@ -73,12 +76,37 @@ impl DeepScript { /// A non-2xx response is *not* an error here; callers inspect /// `resp.status` themselves. async fn send(&self, req: HttpRequest) -> Result { - self.transport.send(req).await.map_err(|e| match e { - TransportError::Timeout => TranscribeError::Timeout, - TransportError::Io(msg) | TransportError::Network(msg) => { - TranscribeError::Transport(msg) + self.transport.send(req).await.map_err(transport_error) + } + + /// Asks for the status of `remote_id` once. + async fn poll_status(&self, remote_id: &str) -> Result { + let req = HttpRequest { + method: Method::Get, + url: format!("{}/transcriptions/{remote_id}/status", self.base_url), + headers: vec![self.auth_header()], + body: HttpBody::Empty, + }; + let resp = match self.transport.send(req).await { + Ok(resp) => resp, + Err(e @ (TransportError::Network(_) | TransportError::Timeout)) => { + return Err(PollError::Transient(transport_error(e))) } - }) + Err(e) => return Err(PollError::Fatal(transport_error(e))), + }; + match resp.status { + 200 => resp + .json::>() + .map(|envelope| envelope.data.status) + .map_err(|e| { + PollError::Fatal(TranscribeError::Failed(format!("ungültige Antwort: {e}"))) + }), + 429 | 500..=599 => Err(PollError::Transient(error_for_status( + resp.status, + &resp.body, + ))), + other => Err(PollError::Fatal(error_for_status(other, &resp.body))), + } } async fn fetch_detail(&self, remote_id: &str) -> Result { @@ -191,26 +219,20 @@ impl Transcriber for DeepScript { } } + /// Polls until the job is done. A poll that fails transiently is retried + /// at the normal interval, up to [`MAX_POLL_RETRIES`] times in a row. async fn wait(&self, remote_id: &str) -> Result, TranscribeError> { let deadline = Instant::now() + self.max_wait; + let mut failed_polls = 0; loop { if Instant::now() >= deadline { return Err(TranscribeError::Timeout); } - let req = HttpRequest { - method: Method::Get, - url: format!("{}/transcriptions/{remote_id}/status", self.base_url), - headers: vec![self.auth_header()], - body: HttpBody::Empty, - }; - let resp = self.send(req).await?; - match resp.status { - 200 => { - let envelope: DataEnvelope = resp - .json() - .map_err(|e| TranscribeError::Failed(format!("ungültige Antwort: {e}")))?; - match envelope.data.status.as_str() { + match self.poll_status(remote_id).await { + Ok(status) => { + failed_polls = 0; + match status.as_str() { "completed" => return self.collect_words(remote_id).await, "failed" => return self.collect_failure(remote_id).await, "queued" | "processing" => {} @@ -223,7 +245,14 @@ impl Transcriber for DeepScript { } } } - other => return Err(error_for_status(other, &resp.body)), + Err(PollError::Transient(e)) if failed_polls < MAX_POLL_RETRIES => { + failed_polls += 1; + log::warn!( + "DeepScript: Statusabfrage für {remote_id} fehlgeschlagen ({}), Versuch {failed_polls} von {MAX_POLL_RETRIES}", + e.user_message() + ); + } + Err(PollError::Transient(e) | PollError::Fatal(e)) => return Err(e), } tokio::time::sleep(self.poll_interval).await; @@ -245,6 +274,20 @@ impl Transcriber for DeepScript { } } +/// A failed status poll: `Transient` ones (network, timeout, HTTP 429/5xx) +/// are worth retrying. +enum PollError { + Transient(TranscribeError), + Fatal(TranscribeError), +} + +fn transport_error(e: TransportError) -> TranscribeError { + match e { + TransportError::Timeout => TranscribeError::Timeout, + TransportError::Io(msg) | TransportError::Network(msg) => TranscribeError::Transport(msg), + } +} + /// Maps a non-2xx status that isn't handled specially by the caller (401, /// 429, and a generic fallback carrying the problem body's `detail`/`title` /// — the set shared by every DeepScript endpoint). @@ -600,6 +643,72 @@ mod tests { transport.assert_done(); } + #[tokio::test] + async fn wait_retries_a_transient_status_error() { + let transport = Arc::new(FakeTransport::new()); + let status_url = format!("{BASE}/transcriptions/job-8/status"); + transport.expect(Method::Get, &status_url, 503, ""); + transport.expect( + Method::Get, + &status_url, + 200, + r#"{"data":{"id":"job-8","status":"completed"}}"#, + ); + transport.expect( + Method::Get, + &format!("{BASE}/transcriptions/job-8"), + 200, + r#"{"data":{"id":"job-8","status":"completed","resultJson":{"words":[]}}}"#, + ); + let ds = deep_script(&transport); + + let words = ds.wait("job-8").await.unwrap(); + + assert_eq!(words, Vec::::new()); + transport.assert_done(); + } + + #[tokio::test] + async fn wait_gives_up_after_four_transient_errors_in_a_row() { + let transport = Arc::new(FakeTransport::new()); + let status_url = format!("{BASE}/transcriptions/job-9/status"); + // A successful poll in between resets the count. + for status in [503, 429, 200, 502, 503, 500, 503] { + let body = if status == 200 { + r#"{"data":{"id":"job-9","status":"processing"}}"# + } else { + r#"{"title":"Service Unavailable"}"# + }; + transport.expect(Method::Get, &status_url, status, body); + } + let ds = deep_script(&transport); + + let err = ds.wait("job-9").await.unwrap_err(); + + assert_eq!( + err, + TranscribeError::Failed("HTTP 503: Service Unavailable".to_string()) + ); + transport.assert_done(); + } + + #[tokio::test] + async fn wait_does_not_retry_a_client_error() { + let transport = Arc::new(FakeTransport::new()); + transport.expect( + Method::Get, + &format!("{BASE}/transcriptions/job-10/status"), + 401, + "", + ); + let ds = deep_script(&transport); + + let err = ds.wait("job-10").await.unwrap_err(); + + assert_eq!(err, TranscribeError::Unauthorized); + transport.assert_done(); + } + #[tokio::test(start_paused = true)] async fn wait_times_out_after_max_wait() { let transport = Arc::new(FakeTransport::new()); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 01ccf13..bc39bdc 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -62,10 +62,16 @@ pub fn run() { config_dir.join("settings.json"), data_dir.join("recordings"), ); + let leftover_jobs = state.store.calls_with_remote_jobs().unwrap_or_else(|e| { + log::error!("could not read leftover remote jobs: {e}"); + Vec::new() + }); let autostart = state.settings().autostart; + // Before anything can start a run. + processing::clear_upload_dir(app.handle()); app.manage(state); commands::apply_autostart(app.handle(), autostart); - processing::delete_leftover_jobs(app.handle(), recovery.failed); + processing::delete_leftover_jobs(app.handle(), leftover_jobs); tray::create(app.handle())?; detection::spawn(app.handle().clone()); diff --git a/src-tauri/src/processing.rs b/src-tauri/src/processing.rs index 71ab368..2e507eb 100644 --- a/src-tauri/src/processing.rs +++ b/src-tauri/src/processing.rs @@ -3,6 +3,9 @@ //! system notification when a run ends. Notifications never contain //! transcript text, and the API key is never logged. +use std::io::ErrorKind; +use std::path::PathBuf; + use recorder_core::model::{CallId, CallStatus}; use recorder_core::pipeline::{self, PipelineOptions}; use recorder_core::secrets::Service; @@ -36,6 +39,16 @@ impl Claim { id, }) } + + /// Reserves the call for a cleanup ([`AppState::reserve_processing`]). + fn for_cleanup(app: &AppHandle, id: CallId) -> Option { + app.state::() + .reserve_processing(id) + .then(|| Self { + app: app.clone(), + id, + }) + } } impl Drop for Claim { @@ -60,11 +73,7 @@ pub fn start(app: &AppHandle, id: CallId, retry: bool) -> Result<(), String> { emit_updated(app, id); return Err(msg); }; - let work_dir = app - .path() - .app_cache_dir() - .map_err(|e| format!("Cache-Ordner nicht gefunden: {e}"))? - .join("upload"); + let work_dir = upload_dir(app)?; let settings = state.settings(); let opts = PipelineOptions { language: settings.language, @@ -95,11 +104,34 @@ pub fn start(app: &AppHandle, id: CallId, retry: bool) -> Result<(), String> { Ok(()) } -/// Best-effort deletes the DeepScript jobs of calls whose processing the -/// last run of the app did not finish ([`RecoveryReport::failed`]), in the +/// Where the pipeline puts the FLAC copies it uploads. +fn upload_dir(app: &AppHandle) -> Result { + app.path() + .app_cache_dir() + .map(|dir| dir.join("upload")) + .map_err(|e| format!("Cache-Ordner nicht gefunden: {e}")) +} + +/// Removes FLAC copies left in the upload directory by a run the app did +/// not finish. Call before any run can start. +pub fn clear_upload_dir(app: &AppHandle) { + let dir = match upload_dir(app) { + Ok(dir) => dir, + Err(e) => { + log::warn!("{e}"); + return; + } + }; + match std::fs::remove_dir_all(&dir) { + Ok(()) => log::info!("removed leftover uploads in {}", dir.display()), + Err(e) if e.kind() == ErrorKind::NotFound => {} + Err(e) => log::warn!("could not empty {}: {e}", dir.display()), + } +} + +/// Best-effort deletes the DeepScript jobs still on record for `calls` +/// (from runs the app did not finish, or whose delete failed), in the /// background. Skipped without a key. -/// -/// [`RecoveryReport::failed`]: recorder_core::store::RecoveryReport::failed pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec) { if calls.is_empty() { return; @@ -117,8 +149,8 @@ pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec) { let state = app.state::(); let transcriber = DeepScript::new(state.transport.clone(), key); for id in calls { - // A retry the user started meanwhile cleans up itself. - let Ok(_claim) = Claim::take(&app, id, true) else { + // A run the user started meanwhile cleans up itself. + let Some(_claim) = Claim::for_cleanup(&app, id) else { continue; }; pipeline::delete_remote_jobs(&state.store, &transcriber, id).await; diff --git a/src-tauri/src/state.rs b/src-tauri/src/state.rs index ca6f477..5a33195 100644 --- a/src-tauri/src/state.rs +++ b/src-tauri/src/state.rs @@ -400,7 +400,7 @@ impl AppState { /// Deletes a finished call: both audio files and the row, and takes it /// out of the prompt queue. Returns how many calls still wait for the - /// prompt. + /// prompt. A call that is being processed is not deleted. pub fn discard_call(&self, id: CallId) -> Result { if lock(&self.current) .as_ref() @@ -408,11 +408,19 @@ impl AppState { { return Err("Das Gespräch wird gerade aufgenommen".to_string()); } + // Held until the call is gone, so no run can claim it meanwhile. + let processing = lock(&self.processing); + if processing.contains(&id) { + return Err(ALREADY_PROCESSING.to_string()); + } let call = self .store .get_call(id) .map_err(|e| e.to_string())? .ok_or_else(|| "Gespräch nicht gefunden".to_string())?; + if call.status == CallStatus::Processing { + return Err(ALREADY_PROCESSING.to_string()); + } // Files first: if one cannot be deleted, the row stays so the user // can try again instead of leaving orphaned audio behind. for path in [&call.mic_path, &call.out_path] { @@ -438,6 +446,7 @@ impl AppState { } return Err(format!("Gespräch konnte nicht gelöscht werden: {e}")); } + drop(processing); Ok(self.prompts_waiting()) } @@ -470,7 +479,15 @@ impl AppState { Ok(()) } - /// Ends the pipeline run claimed by [`AppState::claim_processing`]. + /// Reserves `id` for a cleanup of its remote jobs, whatever its status; + /// `false` if a run or cleanup already has it. Release it with + /// [`AppState::release_processing`]. + pub fn reserve_processing(&self, id: CallId) -> bool { + lock(&self.processing).insert(id) + } + + /// Ends the pipeline run claimed by [`AppState::claim_processing`] (or + /// the cleanup reserved by [`AppState::reserve_processing`]). pub fn release_processing(&self, id: CallId) { lock(&self.processing).remove(&id); } @@ -707,6 +724,56 @@ mod tests { assert_eq!(f.state.prompts_waiting(), 0); } + #[test] + fn a_call_being_processed_is_not_discarded() { + let f = fixture(); + let (id, mic, out) = finished_call(&f, "a"); + f.state.queue_prompt(id); + f.state.claim_processing(id, false).unwrap(); + + assert_eq!( + f.state.discard_call(id), + Err("Wird bereits verarbeitet".to_string()) + ); + f.state.release_processing(id); + f.state + .store + .set_status(id, CallStatus::Processing, None) + .unwrap(); + assert_eq!( + f.state.discard_call(id), + Err("Wird bereits verarbeitet".to_string()) + ); + + assert!(mic.exists()); + assert!(out.exists()); + assert!(f.state.store.get_call(id).unwrap().is_some()); + assert_eq!(f.state.prompts_waiting(), 1); + } + + #[test] + fn a_cleanup_reserves_a_call_of_any_status() { + let f = fixture(); + let (id, _, _) = finished_call(&f, "a"); + f.state + .store + .set_status(id, CallStatus::Done, None) + .unwrap(); + + assert!(f.state.reserve_processing(id)); + assert!(!f.state.reserve_processing(id)); + f.state + .store + .set_status(id, CallStatus::Failed, Some("Fehler")) + .unwrap(); + assert_eq!( + f.state.claim_processing(id, true), + Err("Wird bereits verarbeitet".to_string()) + ); + f.state.release_processing(id); + assert!(f.state.reserve_processing(id)); + } + #[test] fn a_call_is_processed_once_at_a_time() { let f = fixture();