fix #14740: always clean up deepscript jobs and upload copies
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+143
-14
@@ -68,7 +68,8 @@ const OUT_TRACK: &str = "out";
|
||||
|
||||
/// Transcribes call `id` and stores the merged transcript, ending in
|
||||
/// `Done`. A call that already has a transcript (a retry after a later
|
||||
/// step failed) is not transcribed again.
|
||||
/// step failed) is not transcribed again; only remote jobs still on record
|
||||
/// for it are deleted.
|
||||
///
|
||||
/// Both tracks are encoded to FLAC in `work_dir` (removed again afterwards)
|
||||
/// and uploaded; the remote jobs are deleted once their results are in, or
|
||||
@@ -103,6 +104,9 @@ async fn run(
|
||||
if call.transcribed_at.is_none() && store.segments(id)?.is_empty() {
|
||||
store.set_status(id, CallStatus::Processing, None)?;
|
||||
transcribe(store, tr, &call, opts, work_dir).await?;
|
||||
} else {
|
||||
// Jobs whose delete failed in an earlier run.
|
||||
delete_remote_jobs(store, tr, id).await;
|
||||
}
|
||||
// SUMMARY SEAM (Task 14b): summarize here.
|
||||
store.set_status(id, CallStatus::Done, None)?;
|
||||
@@ -128,10 +132,9 @@ async fn transcribe(
|
||||
let out_flac = work_dir.join(format!("{id}-out.flac"));
|
||||
let _flac_files = RemoveOnDrop(vec![mic_flac.clone(), out_flac.clone()]);
|
||||
|
||||
tokio::try_join!(
|
||||
encode(&call.mic_path, &mic_flac),
|
||||
encode(&call.out_path, &out_flac)
|
||||
)?;
|
||||
// One after the other: each encode holds a whole track in memory.
|
||||
encode(&call.mic_path, &mic_flac).await?;
|
||||
encode(&call.out_path, &out_flac).await?;
|
||||
log::info!("call {id}: uploading both tracks");
|
||||
|
||||
let track_opts = |single_speaker| TranscribeOptions {
|
||||
@@ -160,7 +163,8 @@ async fn transcribe(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Waits for both jobs at once, once both were submitted.
|
||||
/// Waits for both jobs at once, once both were submitted. Returns as soon
|
||||
/// as one of them fails, without waiting for the other.
|
||||
async fn wait_both(
|
||||
tr: &dyn Transcriber,
|
||||
id: CallId,
|
||||
@@ -169,8 +173,7 @@ async fn wait_both(
|
||||
) -> Result<(Vec<Word>, Vec<Word>), PipelineError> {
|
||||
let (mic_job, out_job) = (mic_job?, out_job?);
|
||||
log::info!("call {id}: waiting for remote jobs {mic_job} and {out_job}");
|
||||
let (mic, out) = tokio::join!(tr.wait(&mic_job), tr.wait(&out_job));
|
||||
Ok((mic?, out?))
|
||||
Ok(tokio::try_join!(tr.wait(&mic_job), tr.wait(&out_job))?)
|
||||
}
|
||||
|
||||
/// Encodes `wav` to FLAC at `flac` on the blocking pool.
|
||||
@@ -186,7 +189,8 @@ async fn encode(wav: &Path, flac: &Path) -> Result<(), PipelineError> {
|
||||
}
|
||||
|
||||
/// Uploads one track and records its remote job right away, so a crash
|
||||
/// mid-wait still leaves a record of what to delete remotely.
|
||||
/// mid-wait still leaves a record of what to delete remotely. If recording
|
||||
/// fails, the job id is still returned: the run deletes it at the end.
|
||||
async fn submit(
|
||||
store: &Store,
|
||||
tr: &dyn Transcriber,
|
||||
@@ -196,12 +200,15 @@ async fn submit(
|
||||
opts: &TranscribeOptions,
|
||||
) -> Result<String, PipelineError> {
|
||||
let remote_id = tr.submit(flac, opts).await?;
|
||||
store.record_remote_job(id, track, &remote_id)?;
|
||||
if let Err(e) = store.record_remote_job(id, track, &remote_id) {
|
||||
log::error!("call {id}: could not record remote job {remote_id} ({track}): {e}");
|
||||
}
|
||||
Ok(remote_id)
|
||||
}
|
||||
|
||||
/// Best-effort deletes the remote jobs recorded for call `id` (left over
|
||||
/// from a run the app did not finish), then forgets them.
|
||||
/// from a run the app did not finish, or whose delete failed), then forgets
|
||||
/// them. Does nothing if none are recorded.
|
||||
pub async fn delete_remote_jobs(store: &Store, tr: &dyn Transcriber, id: CallId) {
|
||||
delete_jobs(store, tr, id, &[]).await;
|
||||
}
|
||||
@@ -222,6 +229,9 @@ async fn delete_jobs(store: &Store, tr: &dyn Transcriber, id: CallId, known: &[S
|
||||
jobs.push(job.clone());
|
||||
}
|
||||
}
|
||||
if jobs.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut all_deleted = true;
|
||||
for job in &jobs {
|
||||
@@ -261,8 +271,8 @@ impl Drop for RemoveOnDrop {
|
||||
/// plus the shorter track's length (a missing or unrepairable track counts
|
||||
/// as empty).
|
||||
///
|
||||
/// The caller is responsible for deleting the remote jobs of the calls
|
||||
/// reported as `failed` (see [`delete_remote_jobs`]).
|
||||
/// The caller is responsible for deleting the remote jobs still on record
|
||||
/// (see [`Store::calls_with_remote_jobs`] and [`delete_remote_jobs`]).
|
||||
pub fn recover_on_startup(store: &Store) -> Result<RecoveryReport, PipelineError> {
|
||||
let report = store.startup_recovery()?;
|
||||
for &id in &report.interrupted {
|
||||
@@ -312,9 +322,16 @@ mod tests {
|
||||
struct FakeTranscriber {
|
||||
results: Mutex<HashMap<String, Result<Vec<Word>, TranscribeError>>>,
|
||||
submitted: Mutex<Vec<(String, TranscribeOptions)>>,
|
||||
/// Every delete attempt, also the failed ones.
|
||||
deleted: Mutex<Vec<String>>,
|
||||
/// Job whose upload fails with `Rejected`.
|
||||
reject_upload: Option<&'static str>,
|
||||
/// Job whose `wait` takes an hour before it returns.
|
||||
slow_wait: Option<&'static str>,
|
||||
/// How many of the next delete attempts fail.
|
||||
failing_deletes: Mutex<usize>,
|
||||
/// Runs at the start of every `submit`.
|
||||
on_submit: Option<Box<dyn Fn() + Send + Sync>>,
|
||||
}
|
||||
|
||||
impl FakeTranscriber {
|
||||
@@ -349,6 +366,9 @@ mod tests {
|
||||
audio: &Path,
|
||||
opts: &TranscribeOptions,
|
||||
) -> Result<String, TranscribeError> {
|
||||
if let Some(hook) = &self.on_submit {
|
||||
hook();
|
||||
}
|
||||
let bytes = fs::read(audio).expect("submitted file exists");
|
||||
assert_eq!(&bytes[0..4], b"fLaC", "uploads are FLAC");
|
||||
let name = audio.file_name().unwrap().to_string_lossy();
|
||||
@@ -368,6 +388,9 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn wait(&self, remote_id: &str) -> Result<Vec<Word>, TranscribeError> {
|
||||
if self.slow_wait == Some(remote_id) {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
|
||||
}
|
||||
self.results
|
||||
.lock()
|
||||
.unwrap()
|
||||
@@ -378,6 +401,11 @@ mod tests {
|
||||
|
||||
async fn delete(&self, remote_id: &str) -> Result<(), TranscribeError> {
|
||||
self.deleted.lock().unwrap().push(remote_id.to_string());
|
||||
let mut failing = self.failing_deletes.lock().unwrap();
|
||||
if *failing > 0 {
|
||||
*failing -= 1;
|
||||
return Err(TranscribeError::Transport("connection reset".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -408,6 +436,10 @@ mod tests {
|
||||
writer.finalize().unwrap();
|
||||
}
|
||||
|
||||
/// File name of the fixture's database, so a test can open a second
|
||||
/// connection to it.
|
||||
const DB_FILE: &str = "calls.sqlite";
|
||||
|
||||
struct Fixture {
|
||||
store: Store,
|
||||
dir: tempfile::TempDir,
|
||||
@@ -438,9 +470,10 @@ mod tests {
|
||||
}
|
||||
|
||||
/// A call awaiting the decision, with one second of audio per track.
|
||||
/// The database is a file in the temp dir (see [`DB_FILE`]).
|
||||
fn fixture() -> Fixture {
|
||||
let store = Store::open_in_memory().unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let store = Store::open(&dir.path().join(DB_FILE)).unwrap();
|
||||
let mic = dir.path().join("1-mic.wav");
|
||||
let out = dir.path().join("1-out.wav");
|
||||
write_wav(&mic, 1.0);
|
||||
@@ -556,6 +589,102 @@ mod tests {
|
||||
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pipeline_deletes_a_job_it_could_not_record() {
|
||||
let f = fixture();
|
||||
// The call row vanishes during the upload, so recording the remote
|
||||
// jobs fails (foreign key).
|
||||
let (db, id) = (f.dir.path().join(DB_FILE), f.id);
|
||||
let fake = FakeTranscriber {
|
||||
on_submit: Some(Box::new(move || {
|
||||
Store::open(&db).unwrap().delete_call(id).unwrap();
|
||||
})),
|
||||
..FakeTranscriber::returning(vec![], vec![])
|
||||
};
|
||||
|
||||
// How the run ends for a vanished call does not matter here.
|
||||
let _ = f.process(&fake).await;
|
||||
|
||||
assert_eq!(fake.submitted().len(), 2);
|
||||
assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]);
|
||||
assert_eq!(f.upload_files(), Vec::<PathBuf>::new());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn pipeline_fails_fast_when_one_wait_fails() {
|
||||
let f = fixture();
|
||||
let fake = FakeTranscriber {
|
||||
slow_wait: Some("job-mic"),
|
||||
..FakeTranscriber::returning(vec![], vec![])
|
||||
};
|
||||
fake.script(
|
||||
"job-out",
|
||||
Err(TranscribeError::Failed("kaputt".to_string())),
|
||||
);
|
||||
let started = tokio::time::Instant::now();
|
||||
|
||||
let err = f.process(&fake).await.unwrap_err();
|
||||
|
||||
assert!(
|
||||
started.elapsed() < std::time::Duration::from_secs(60),
|
||||
"waited {:?} for the other job",
|
||||
started.elapsed()
|
||||
);
|
||||
assert_eq!(
|
||||
err.user_message(),
|
||||
TranscribeError::Failed("kaputt".to_string()).user_message()
|
||||
);
|
||||
assert_eq!(f.status(), CallStatus::Failed);
|
||||
// The job still in flight is deleted (cancelled) as well.
|
||||
assert_eq!(fake.deleted(), vec!["job-mic", "job-out"]);
|
||||
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_delete_is_retried_by_the_next_run() {
|
||||
let f = fixture();
|
||||
let fake = FakeTranscriber {
|
||||
failing_deletes: Mutex::new(1),
|
||||
..FakeTranscriber::returning(vec![word("Hallo", 0.0, 0.5, None)], vec![])
|
||||
};
|
||||
|
||||
f.process(&fake).await.unwrap();
|
||||
|
||||
// The transcript is in, but a job could not be deleted: both stay
|
||||
// on record, also for the cleanup at the next start.
|
||||
assert_eq!(f.status(), CallStatus::Done);
|
||||
assert_eq!(f.store.remote_jobs(f.id).unwrap().len(), 2);
|
||||
assert_eq!(f.store.calls_with_remote_jobs().unwrap(), vec![f.id]);
|
||||
|
||||
// Processing the call again only cleans up.
|
||||
f.process(&fake).await.unwrap();
|
||||
|
||||
assert_eq!(f.status(), CallStatus::Done);
|
||||
assert_eq!(fake.submitted().len(), 2);
|
||||
assert_eq!(
|
||||
fake.deleted(),
|
||||
vec!["job-mic", "job-mic", "job-out", "job-out"]
|
||||
);
|
||||
assert_eq!(f.store.remote_jobs(f.id).unwrap(), vec![]);
|
||||
assert_eq!(
|
||||
f.store.calls_with_remote_jobs().unwrap(),
|
||||
Vec::<CallId>::new()
|
||||
);
|
||||
assert_eq!(transcript(&f), vec!["Ich: Hallo".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn skipping_transcription_deletes_nothing_without_leftovers() {
|
||||
let f = fixture();
|
||||
f.store.mark_transcribed(f.id, Local::now()).unwrap();
|
||||
let fake = FakeTranscriber::default();
|
||||
|
||||
f.process(&fake).await.unwrap();
|
||||
|
||||
assert_eq!(f.status(), CallStatus::Done);
|
||||
assert_eq!(fake.deleted(), Vec::<String>::new());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn leftover_remote_jobs_are_deleted_and_forgotten() {
|
||||
let f = fixture();
|
||||
|
||||
Reference in New Issue
Block a user