feat #14740: transcription pipeline

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
hauke
2026-11-05 16:41:53 +01:00
co-authored by Claude Opus 5.5
parent 1420e56e77
commit e2828791da
13 changed files with 1066 additions and 17 deletions
+1
View File
@@ -21,6 +21,7 @@ tauri = { version = "2.12.5", features = ["tray-icon"] }
tauri-plugin-log = "2"
tauri-plugin-autostart = "2"
tauri-plugin-single-instance = "2"
tauri-plugin-notification = "2"
chrono = { version = "0.4", features = ["serde"] }
tempfile = "3"
recorder-core = { path = "../crates/core" }
+2 -1
View File
@@ -8,6 +8,7 @@
"prompt"
],
"permissions": [
"core:default"
"core:default",
"notification:default"
]
}
+24 -7
View File
@@ -16,6 +16,7 @@ use serde::Serialize;
use tauri::{AppHandle, Emitter, Manager, State};
use tauri_plugin_autostart::ManagerExt;
use crate::processing;
use crate::state::{AppState, CallBrief, DeviceLevels, Ended, PendingPrompt, RecordingState};
const LEVELS_INTERVAL: Duration = Duration::from_millis(100);
@@ -120,19 +121,35 @@ pub fn get_call_brief(state: State<'_, AppState>, id: CallId) -> CmdResult<CallB
#[tauri::command]
pub async fn discard_call(app: AppHandle, id: CallId) -> CmdResult<()> {
blocking(app, move |app| {
if app.state::<AppState>().discard_call(id)? == 0 {
set_window_visible(app, "prompt", false);
}
app.state::<AppState>().discard_call(id)?;
prompt_queue_changed(app);
Ok(())
})
.await
}
/// Placeholder until transcription exists; the prompt shows the error.
/// Transkribieren in the end-of-call prompt: starts processing in the
/// background and returns at once; the call leaves the prompt.
#[tauri::command]
pub fn transcribe_call(id: CallId) -> CmdResult<()> {
let _ = id;
Err("Transkription ist noch nicht verfügbar".to_string())
pub async fn transcribe_call(app: AppHandle, id: CallId) -> CmdResult<()> {
blocking(app, move |app| processing::start(app, id, false)).await
}
/// Processes a failed (or still undecided) call again.
#[tauri::command]
pub async fn retry_call(app: AppHandle, id: CallId) -> CmdResult<()> {
blocking(app, move |app| processing::start(app, id, true)).await
}
/// After a call left the prompt queue: hides the prompt if no call waits
/// any more, and tells it to reload (`prompt-changed`) otherwise.
pub fn prompt_queue_changed(app: &AppHandle) {
if app.state::<AppState>().prompts_waiting() == 0 {
set_window_visible(app, "prompt", false);
}
if let Err(e) = app.emit("prompt-changed", ()) {
log::error!("could not emit prompt-changed: {e}");
}
}
/// Starts a manual recording, shows the floating window and streams levels.
+6 -1
View File
@@ -1,8 +1,10 @@
mod commands;
mod detection;
mod processing;
mod state;
mod tray;
use recorder_core::pipeline;
use recorder_core::secrets::KeyringSecrets;
use recorder_core::store::Store;
use tauri::{Manager, WindowEvent};
@@ -28,6 +30,7 @@ pub fn run() {
MacosLauncher::LaunchAgent,
Some(vec![AUTOSTART_ARG]),
))
.plugin(tauri_plugin_notification::init())
.setup(|app| {
// Release builds log to a file in the app log directory (on
// Windows %LOCALAPPDATA%\eu.euvino.callrecorder\logs), debug
@@ -50,7 +53,7 @@ pub fn run() {
let config_dir = app.path().app_config_dir()?;
std::fs::create_dir_all(&data_dir)?;
let store = Store::open(&data_dir.join("calls.sqlite"))?;
let recovery = store.startup_recovery()?;
let recovery = pipeline::recover_on_startup(&store)?;
log::info!("startup recovery: {recovery:?}");
let state = AppState::new(
@@ -62,6 +65,7 @@ pub fn run() {
let autostart = state.settings().autostart;
app.manage(state);
commands::apply_autostart(app.handle(), autostart);
processing::delete_leftover_jobs(app.handle(), recovery.failed);
tray::create(app.handle())?;
detection::spawn(app.handle().clone());
@@ -93,6 +97,7 @@ pub fn run() {
commands::get_call_brief,
commands::discard_call,
commands::transcribe_call,
commands::retry_call,
])
.run(tauri::generate_context!())
.expect("error while building tauri application");
+167
View File
@@ -0,0 +1,167 @@
//! Runs the processing pipeline for calls in the background and tells the
//! user about it: a `call-updated` event on every status change and a
//! system notification when a run ends. Notifications never contain
//! transcript text, and the API key is never logged.
use recorder_core::model::{CallId, CallStatus};
use recorder_core::pipeline::{self, PipelineOptions};
use recorder_core::secrets::Service;
use recorder_core::transcribe::deepscript::DeepScript;
use recorder_core::transcribe::TranscribeError;
use serde::Serialize;
use tauri::{AppHandle, Emitter, Manager};
use tauri_plugin_notification::NotificationExt;
use crate::commands;
use crate::state::AppState;
/// Payload of the `call-updated` event: the call's status changed.
#[derive(Serialize, Clone)]
struct CallUpdated {
id: CallId,
}
/// A call reserved with [`AppState::claim_processing`]; released when
/// dropped, also if the run panics.
struct Claim {
app: AppHandle,
id: CallId,
}
impl Claim {
fn take(app: &AppHandle, id: CallId, retry: bool) -> Result<Self, String> {
app.state::<AppState>().claim_processing(id, retry)?;
Ok(Self {
app: app.clone(),
id,
})
}
}
impl Drop for Claim {
fn drop(&mut self) {
self.app.state::<AppState>().release_processing(self.id);
}
}
/// Starts processing call `id` in the background and returns at once; the
/// call leaves the end-of-call prompt. `retry` also allows failed calls.
/// Without a DeepScript key nothing starts: the call is marked failed and
/// the error returned. Reads the keyring, so call it off the UI thread.
pub fn start(app: &AppHandle, id: CallId, retry: bool) -> Result<(), String> {
let claim = Claim::take(app, id, retry)?;
let state = app.state::<AppState>();
let Some(key) = deepscript_key(&state) else {
let msg = TranscribeError::MissingKey.user_message();
if let Err(e) = state.store.set_status(id, CallStatus::Failed, Some(&msg)) {
log::error!("call {id}: could not mark as failed: {e}");
}
emit_updated(app, id);
return Err(msg);
};
let work_dir = app
.path()
.app_cache_dir()
.map_err(|e| format!("Cache-Ordner nicht gefunden: {e}"))?
.join("upload");
let settings = state.settings();
let opts = PipelineOptions {
language: settings.language,
deepscript_model: settings.deepscript_model,
};
// A fresh client per run, so a key changed in the settings applies to
// the next run.
let transcriber = DeepScript::new(state.transport.clone(), key);
// Shown as processing right away, before the run gets going.
state
.store
.set_status(id, CallStatus::Processing, None)
.map_err(|e| format!("Interner Fehler: {e}"))?;
emit_updated(app, id);
if state.remove_queued_prompt(id).is_some() {
commands::prompt_queue_changed(app);
}
log::info!("call {id}: processing started");
tauri::async_runtime::spawn(async move {
let app = &claim.app;
let state = app.state::<AppState>();
let result = pipeline::process_call(&state.store, &transcriber, id, &opts, &work_dir).await;
emit_updated(app, id);
notify(app, id, result.err().map(|e| e.user_message()));
});
Ok(())
}
/// Best-effort deletes the DeepScript jobs of calls whose processing the
/// last run of the app did not finish ([`RecoveryReport::failed`]), in the
/// background. Skipped without a key.
///
/// [`RecoveryReport::failed`]: recorder_core::store::RecoveryReport::failed
pub fn delete_leftover_jobs(app: &AppHandle, calls: Vec<CallId>) {
if calls.is_empty() {
return;
}
let app = app.clone();
tauri::async_runtime::spawn(async move {
let key = {
let app = app.clone();
tauri::async_runtime::spawn_blocking(move || deepscript_key(&app.state())).await
};
let Ok(Some(key)) = key else {
log::info!("no DeepScript key: remote jobs of calls {calls:?} stay for now");
return;
};
let state = app.state::<AppState>();
let transcriber = DeepScript::new(state.transport.clone(), key);
for id in calls {
// A retry the user started meanwhile cleans up itself.
let Ok(_claim) = Claim::take(&app, id, true) else {
continue;
};
pipeline::delete_remote_jobs(&state.store, &transcriber, id).await;
}
});
}
fn deepscript_key(state: &AppState) -> Option<String> {
state
.secrets
.get(Service::DeepScript)
.filter(|key| !key.trim().is_empty())
}
fn emit_updated(app: &AppHandle, id: CallId) {
if let Err(e) = app.emit("call-updated", CallUpdated { id }) {
log::error!("could not emit call-updated: {e}");
}
}
/// "Transkript bereit" with "<app> · <N> min", or the failure's message.
fn notify(app: &AppHandle, id: CallId, error: Option<String>) {
let (title, body) = match error {
None => ("Transkript bereit", call_summary_line(app, id)),
Some(msg) => ("Transkription fehlgeschlagen", msg),
};
let result = app.notification().builder().title(title).body(body).show();
if let Err(e) = result {
log::warn!("call {id}: could not show the notification: {e}");
}
}
/// "Microsoft Teams · 32 min" (whole minutes, at least 1, like the UI).
fn call_summary_line(app: &AppHandle, id: CallId) -> String {
match app.state::<AppState>().call_brief(id) {
Ok(brief) => format!(
"{} · {} min",
brief.app,
((brief.duration_secs as f64 / 60.0).round() as i64).max(1)
),
Err(e) => {
log::error!("call {id}: could not load it for the notification: {e}");
String::new()
}
}
}
+109 -5
View File
@@ -1,22 +1,24 @@
//! Application state shared by commands, the tray and background threads.
//!
//! Holds the store, the settings, the secret store, the recording that is
//! currently running, the call detector and the call awaiting a decision in
//! the end-of-call prompt. Everything here is independent of windows and
//! Holds the store, the settings, the secret store, the HTTP transport, the
//! recording that is currently running, the call detector, the calls
//! awaiting a decision in the end-of-call prompt and the calls being
//! processed. Everything here is independent of windows and
//! events; the command layer turns results into UI updates.
use std::collections::VecDeque;
use std::collections::{HashSet, VecDeque};
use std::fs;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Mutex, MutexGuard};
use std::sync::{Arc, Mutex, MutexGuard};
use std::thread;
use std::time::{Duration, Instant};
use chrono::{DateTime, Local};
use recorder_core::capture::{CaptureSource, OutputTarget};
use recorder_core::detect::{CallDetector, DetectorEvent, MicSession};
use recorder_core::http::{HttpTransport, ReqwestTransport};
use recorder_core::model::{Call, CallId, CallStatus};
use recorder_core::recorder::Recorder;
use recorder_core::secrets::SecretStore;
@@ -31,6 +33,7 @@ const MANUAL_APP: &str = "Manuell";
/// How long [`AppState::test_devices`] records.
const TEST_DURATION: Duration = Duration::from_secs(5);
const TEST_POLL: Duration = Duration::from_millis(50);
const ALREADY_PROCESSING: &str = "Wird bereits verarbeitet";
/// Payload of the `recording-state` event.
#[derive(Serialize, Clone, Debug)]
@@ -111,6 +114,8 @@ struct CurrentRecording {
pub struct AppState {
pub store: Store,
pub secrets: Box<dyn SecretStore>,
/// The one HTTP client all API calls share.
pub transport: Arc<dyn HttpTransport>,
settings: Mutex<Settings>,
settings_path: PathBuf,
recordings_dir: PathBuf,
@@ -120,6 +125,8 @@ pub struct AppState {
/// first. A call leaves the queue when it is decided (discarded or sent
/// to transcription).
pending_prompts: Mutex<VecDeque<CallId>>,
/// Calls a pipeline run is under way for; at most one run per call.
processing: Mutex<HashSet<CallId>>,
/// Set while a recording starts or the devices are tested, so neither
/// runs twice and `current` is never locked for longer than a moment.
devices_busy: AtomicBool,
@@ -146,12 +153,14 @@ impl AppState {
Self {
store,
secrets,
transport: Arc::new(ReqwestTransport::new()),
settings: Mutex::new(settings),
settings_path,
recordings_dir,
current: Mutex::new(None),
detector: Mutex::new(detector),
pending_prompts: Mutex::new(VecDeque::new()),
processing: Mutex::new(HashSet::new()),
devices_busy: AtomicBool::new(false),
}
}
@@ -432,6 +441,40 @@ impl AppState {
Ok(self.prompts_waiting())
}
/// Reserves `id` for one pipeline run; release it with
/// [`AppState::release_processing`] when the run ends. A call that waits
/// for the decision can be processed; a failed one only on `retry`, or
/// while the prompt still shows it (its first attempt failed before
/// anything started, e.g. for lack of an API key).
pub fn claim_processing(&self, id: CallId, retry: bool) -> Result<(), String> {
let mut processing = lock(&self.processing);
if processing.contains(&id) {
return Err(ALREADY_PROCESSING.to_string());
}
let call = self
.store
.get_call(id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "Gespräch nicht gefunden".to_string())?;
match call.status {
CallStatus::AwaitingDecision => {}
CallStatus::Failed if retry || lock(&self.pending_prompts).contains(&id) => {}
CallStatus::Processing => return Err(ALREADY_PROCESSING.to_string()),
CallStatus::Recording => return Err("Das Gespräch wird gerade aufgenommen".to_string()),
CallStatus::Done => return Err("Das Gespräch ist bereits transkribiert".to_string()),
CallStatus::Failed => {
return Err("Das Gespräch wartet nicht auf eine Entscheidung".to_string())
}
}
processing.insert(id);
Ok(())
}
/// Ends the pipeline run claimed by [`AppState::claim_processing`].
pub fn release_processing(&self, id: CallId) {
lock(&self.processing).remove(&id);
}
/// Records both configured devices into a temporary directory for five
/// seconds and returns the peak level of each track.
pub fn test_devices(&self) -> Result<DeviceLevels, String> {
@@ -664,6 +707,67 @@ mod tests {
assert_eq!(f.state.prompts_waiting(), 0);
}
#[test]
fn a_call_is_processed_once_at_a_time() {
let f = fixture();
let (id, _, _) = finished_call(&f, "a");
assert_eq!(f.state.claim_processing(id, false), Ok(()));
assert_eq!(
f.state.claim_processing(id, true),
Err("Wird bereits verarbeitet".to_string())
);
f.state.release_processing(id);
assert_eq!(f.state.claim_processing(id, false), Ok(()));
}
#[test]
fn a_processing_call_is_not_claimed_again() {
let f = fixture();
let (id, _, _) = finished_call(&f, "a");
f.state
.store
.set_status(id, CallStatus::Processing, None)
.unwrap();
assert_eq!(
f.state.claim_processing(id, true),
Err("Wird bereits verarbeitet".to_string())
);
}
#[test]
fn a_failed_call_is_processed_again_on_retry_or_from_the_prompt() {
let f = fixture();
let (id, _, _) = finished_call(&f, "a");
f.state
.store
.set_status(id, CallStatus::Failed, Some("Fehler"))
.unwrap();
assert!(f.state.claim_processing(id, false).is_err());
assert_eq!(f.state.claim_processing(id, true), Ok(()));
f.state.release_processing(id);
f.state.queue_prompt(id);
assert_eq!(f.state.claim_processing(id, false), Ok(()));
}
#[test]
fn done_and_recording_calls_are_not_processed() {
let f = fixture();
let (done, _, _) = finished_call(&f, "a");
f.state
.store
.set_status(done, CallStatus::Done, None)
.unwrap();
assert!(f.state.claim_processing(done, true).is_err());
let recording = f
.state
.store
.create_call("Manuell", at(9, 0, 0), Path::new(""), Path::new(""))
.unwrap();
assert!(f.state.claim_processing(recording, true).is_err());
assert!(f.state.claim_processing(recording + 1, true).is_err());
}
#[test]
fn invalid_detection_delays_are_not_saved() {
let f = fixture();