From 8e8a3c3c581ecb95f46a6c39ddcf344eecdcf28e Mon Sep 17 00:00:00 2001 From: hauke Date: Thu, 5 Nov 2026 14:31:11 +0100 Subject: [PATCH] feat #14740: two-track capture and recorder Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 76 ++++ crates/audio/Cargo.toml | 8 + crates/audio/src/capture_thread.rs | 84 ++++ crates/audio/src/lib.rs | 24 +- crates/audio/src/linux/capture.rs | 154 +++++++ crates/audio/src/windows/capture.rs | 327 +++++++++++++++ crates/core/src/capture.rs | 72 ++++ crates/core/src/lib.rs | 2 + crates/core/src/recorder.rs | 610 ++++++++++++++++++++++++++++ 9 files changed, 1354 insertions(+), 3 deletions(-) create mode 100644 crates/audio/src/capture_thread.rs create mode 100644 crates/audio/src/linux/capture.rs create mode 100644 crates/audio/src/windows/capture.rs create mode 100644 crates/core/src/capture.rs create mode 100644 crates/core/src/recorder.rs diff --git a/Cargo.lock b/Cargo.lock index aa3545a..8ef957f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1856,6 +1856,54 @@ dependencies = [ "winapi", ] +[[package]] +name = "libpulse-binding" +version = "2.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909eb3049e16e373680fe65afe6e2a722ace06b671250cc4849557bc57d6a397" +dependencies = [ + "bitflags 2.13.2", + "libc", + "libpulse-sys", + "num-derive", + "num-traits", + "winapi", +] + +[[package]] +name = "libpulse-simple-binding" +version = "2.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7bebef0381c8e3e4b23cc24aaf36fab37472bece128de96f6a111efa464cfef" +dependencies = [ + "libpulse-binding", + "libpulse-simple-sys", + "libpulse-sys", +] + +[[package]] +name = "libpulse-simple-sys" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd96888fe37ad270d16abf5e82cccca1424871cf6afa2861824d2a52758eebc" +dependencies = [ + "libpulse-sys", + "pkg-config", +] + +[[package]] +name = "libpulse-sys" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d74371848b22e989f829cc1621d2ebd74960711557d8b45cfe740f60d0a05e61" +dependencies = [ + "libc", + "num-derive", + "num-traits", + "pkg-config", + "winapi", +] + [[package]] name = "libredox" version = "0.1.25" @@ -2048,6 +2096,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-integer" version = "0.1.47" @@ -2606,7 +2665,11 @@ dependencies = [ name = "recorder-audio" version = "0.1.0" dependencies = [ + "libpulse-binding", + "libpulse-simple-binding", + "log", "recorder-core", + "wasapi", ] [[package]] @@ -4059,6 +4122,19 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasapi" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5f8804a710cf7a18b07610225abb127e562f3249a1f3ed88a6f045c5444910" +dependencies = [ + "log", + "num-integer", + "thiserror 2.0.21", + "windows", + "windows-core", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index dd5d521..0b346ec 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -9,3 +9,11 @@ path = "src/lib.rs" [dependencies] recorder-core = { path = "../core" } +log = "0.4" + +[target.'cfg(target_os = "linux")'.dependencies] +libpulse-binding = "2.30" +libpulse-simple-binding = "2.29" + +[target.'cfg(windows)'.dependencies] +wasapi = "0.26" diff --git a/crates/audio/src/capture_thread.rs b/crates/audio/src/capture_thread.rs new file mode 100644 index 0000000..757e513 --- /dev/null +++ b/crates/audio/src/capture_thread.rs @@ -0,0 +1,84 @@ +//! The dedicated thread every platform capture source runs on. +//! +//! Platform stream handles (COM interfaces, PulseAudio connections) are not +//! `Send`, so they are created, used and dropped on this one thread. The +//! thread reports whether the stream opened through a channel, which lets +//! `CaptureSource::start` return that result synchronously. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, SyncSender}; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use recorder_core::capture::{CaptureError, StreamFormat}; + +/// How long `stop` waits for the thread before detaching it. Only reached if +/// a platform call blocks indefinitely; stopping must never hang the app. +const STOP_TIMEOUT: Duration = Duration::from_secs(2); + +/// Sends the outcome of opening the stream back to `start`. +pub(crate) type Ready = SyncSender>; + +pub(crate) struct CaptureThread { + stop: Arc, + /// Disconnects when the thread ends (its sender is dropped with it). + done: Receiver<()>, + handle: JoinHandle<()>, +} + +impl CaptureThread { + /// Spawns `body` on a new thread and waits until it has reported the + /// stream format (or an error) through `ready`. `body` must return soon + /// after the stop flag it is given is set. + pub(crate) fn spawn(name: &str, body: F) -> Result<(Self, StreamFormat), CaptureError> + where + F: FnOnce(&AtomicBool, Ready) + Send + 'static, + { + let stop = Arc::new(AtomicBool::new(false)); + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + let (done_tx, done_rx) = mpsc::channel::<()>(); + let thread_stop = stop.clone(); + let handle = thread::Builder::new() + .name(name.to_string()) + .spawn(move || { + let _done = done_tx; + body(&thread_stop, ready_tx); + }) + .map_err(|err| CaptureError::Backend(format!("cannot spawn capture thread: {err}")))?; + + match ready_rx.recv() { + Ok(Ok(format)) => Ok(( + Self { + stop, + done: done_rx, + handle, + }, + format, + )), + Ok(Err(err)) => { + let _ = handle.join(); + Err(err) + } + Err(_) => { + let _ = handle.join(); + Err(CaptureError::Backend( + "capture thread ended before the stream opened".into(), + )) + } + } + } + + /// Signals the thread to stop and waits for it to end. + pub(crate) fn stop(self) { + self.stop.store(true, Ordering::SeqCst); + match self.done.recv_timeout(STOP_TIMEOUT) { + Err(RecvTimeoutError::Timeout) => { + log::warn!("capture thread did not stop in time; detaching it"); + } + _ => { + let _ = self.handle.join(); + } + } + } +} diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index 5c10b11..aaf48a2 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -1,4 +1,22 @@ -//! Audio capture and encoding for the Gesprächsrekorder call recorder. +//! Platform audio capture for the Gesprächsrekorder call recorder. //! -//! Intentionally near-empty for now; audio device enumeration, capture and -//! WAV/FLAC encoding land in a later task. +//! Implements [`recorder_core::capture::CaptureSource`] for the microphone and +//! for the call app's audio output: WASAPI on Windows, PulseAudio on Linux. +//! The platform-neutral trait, types and the two-track recorder live in +//! `recorder-core`. + +mod capture_thread; + +#[cfg(target_os = "linux")] +mod linux { + pub mod capture; +} +#[cfg(target_os = "linux")] +pub use linux::capture::{list_devices, mic_source, output_source}; + +#[cfg(windows)] +mod windows { + pub mod capture; +} +#[cfg(windows)] +pub use windows::capture::{list_devices, mic_source, output_source}; diff --git a/crates/audio/src/linux/capture.rs b/crates/audio/src/linux/capture.rs new file mode 100644 index 0000000..a95e770 --- /dev/null +++ b/crates/audio/src/linux/capture.rs @@ -0,0 +1,154 @@ +//! PulseAudio (and PipeWire via pipewire-pulse) capture with libpulse-simple. +//! +//! Streams are opened directly as 16 kHz mono `f32`, so the server does the +//! resampling and downmix and the recorder's conditioner passes them through. +//! The call output is recorded from the default sink's monitor source; +//! per-process capture does not exist here, so `OutputTarget::Process` falls +//! back to that monitor as well. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use libpulse_binding::def::BufferAttr; +use libpulse_binding::error::{Code, PAErr}; +use libpulse_binding::sample::{Format, Spec}; +use libpulse_binding::stream::Direction; +use libpulse_simple_binding::Simple; +use recorder_core::capture::{ + CaptureError, CaptureSource, DataCallback, Devices, OutputTarget, StreamFormat, +}; + +use crate::capture_thread::{CaptureThread, Ready}; + +const APP_NAME: &str = "Gesprächsrekorder"; +const DEFAULT_MONITOR: &str = "@DEFAULT_MONITOR@"; +const FORMAT: StreamFormat = StreamFormat { + rate: 16_000, + channels: 1, +}; +/// Samples per blocking read (20 ms). Also bounds how long `stop` waits. +const CHUNK_SAMPLES: usize = 320; +const CHUNK_BYTES: usize = CHUNK_SAMPLES * std::mem::size_of::(); + +/// A PulseAudio record stream on one source (microphone or monitor). +pub struct PulseSource { + /// Source name; `None` for the server's default source. + device: Option, + stream_name: &'static str, + thread: Option, +} + +pub fn mic_source(device: Option<&str>) -> Box { + Box::new(PulseSource { + device: device.map(str::to_owned), + stream_name: "Mikrofon", + thread: None, + }) +} + +pub fn output_source(target: OutputTarget) -> Box { + let device = match target { + OutputTarget::Process { .. } | OutputTarget::DefaultRender => DEFAULT_MONITOR.to_owned(), + OutputTarget::Device(name) => name, + }; + Box::new(PulseSource { + device: Some(device), + stream_name: "Gesprächsaudio", + thread: None, + }) +} + +/// Device enumeration is not offered on Linux; recordings use the defaults. +pub fn list_devices() -> Devices { + Devices::default() +} + +impl CaptureSource for PulseSource { + fn start(&mut self, on_data: DataCallback) -> Result { + self.stop(); + let device = self.device.clone(); + let stream_name = self.stream_name; + let (thread, format) = + CaptureThread::spawn(&format!("pulse {stream_name}"), move |stop, ready| { + run(device.as_deref(), stream_name, stop, ready, on_data) + })?; + self.thread = Some(thread); + Ok(format) + } + + fn stop(&mut self) { + if let Some(thread) = self.thread.take() { + thread.stop(); + } + } +} + +impl Drop for PulseSource { + fn drop(&mut self) { + self.stop(); + } +} + +fn open(device: Option<&str>, stream_name: &str) -> Result { + let spec = Spec { + format: Format::FLOAT32NE, + rate: FORMAT.rate, + channels: FORMAT.channels as u8, + }; + let attr = BufferAttr { + maxlength: u32::MAX, + tlength: u32::MAX, + prebuf: u32::MAX, + minreq: u32::MAX, + fragsize: CHUNK_BYTES as u32, + }; + Simple::new( + None, + APP_NAME, + Direction::Record, + device, + stream_name, + &spec, + None, + Some(&attr), + ) + .map_err(|err| to_capture_error(err, device)) +} + +fn to_capture_error(err: PAErr, device: Option<&str>) -> CaptureError { + match (Code::try_from(err), device) { + (Ok(Code::NoEntity), Some(device)) => CaptureError::DeviceNotFound(device.to_owned()), + _ => CaptureError::Backend(format!("PulseAudio: {err}")), + } +} + +fn run( + device: Option<&str>, + stream_name: &str, + stop: &AtomicBool, + ready: Ready, + mut on_data: DataCallback, +) { + let simple = match open(device, stream_name) { + Ok(simple) => simple, + Err(err) => { + let _ = ready.send(Err(err)); + return; + } + }; + let _ = ready.send(Ok(FORMAT)); + + let mut bytes = vec![0u8; CHUNK_BYTES]; + let mut samples = vec![0f32; CHUNK_SAMPLES]; + while !stop.load(Ordering::SeqCst) { + if let Err(err) = simple.read(&mut bytes) { + // Source gone (unplugged, server restart): stop delivering; the + // recorder keeps the track aligned with silence. + log::warn!("PulseAudio capture '{stream_name}' ended: {err}"); + return; + } + for (sample, raw) in samples.iter_mut().zip(bytes.as_chunks::<4>().0) { + *sample = f32::from_ne_bytes(*raw); + } + on_data(&samples); + } +} diff --git a/crates/audio/src/windows/capture.rs b/crates/audio/src/windows/capture.rs new file mode 100644 index 0000000..8ffd336 --- /dev/null +++ b/crates/audio/src/windows/capture.rs @@ -0,0 +1,327 @@ +//! WASAPI capture (shared mode, event driven) via the `wasapi` crate. +//! +//! - Microphone: the default *communications* capture endpoint, or the +//! endpoint with the given device id. +//! - Output, `Process`: application loopback of the call app's process tree +//! (Windows 10 2004+), so only the call's audio is recorded. +//! - Output, `DefaultRender` / `Device`: loopback of the default +//! communications render endpoint / of the given render endpoint. +//! +//! All streams are delivered as interleaved `f32`. When the endpoint goes +//! away mid-recording (`AUDCLNT_E_DEVICE_INVALIDATED`, e.g. a headset was +//! unplugged), the source reopens the default endpoint once, in the original +//! format so the recorder's conditioner stays valid. If that fails, it stops +//! delivering data and the recorder pads the track with silence. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; + +use recorder_core::capture::{ + CaptureError, CaptureSource, DataCallback, DeviceInfo, Devices, OutputTarget, StreamFormat, +}; +use wasapi::{ + deinitialize, initialize_mta, AudioCaptureClient, AudioClient, Device, DeviceEnumerator, + Direction, Handle, Role, SampleType, StreamMode, WasapiError, WaveFormat, +}; + +use crate::capture_thread::{CaptureThread, Ready}; + +/// `AUDCLNT_E_DEVICE_INVALIDATED`: the endpoint was removed or disabled. +const AUDCLNT_E_DEVICE_INVALIDATED: i32 = 0x8889_0004_u32 as i32; +/// Format requested for application loopback, which has no mix format. +const PROCESS_LOOPBACK_FORMAT: StreamFormat = StreamFormat { + rate: 48_000, + channels: 2, +}; +/// Shared-mode buffer size (100 ns units): 200 ms of headroom so a slow +/// disk write in the recorder does not overflow the capture buffer. +const BUFFER_DURATION_HNS: i64 = 2_000_000; +/// Event wait timeout (ms). Loopback endpoints signal no events while +/// nothing plays, so the loop wakes up regularly to check the stop flag and +/// to notice an invalidated device. +const WAIT_MS: u32 = 100; + +#[derive(Debug, Clone)] +enum Endpoint { + /// Capture endpoint id; `None` for the default communications endpoint. + Mic(Option), + Output(OutputTarget), +} + +impl Endpoint { + /// What to reopen after the endpoint was invalidated. + fn fallback(&self) -> Endpoint { + match self { + Endpoint::Mic(_) => Endpoint::Mic(None), + Endpoint::Output(OutputTarget::Process { .. }) => self.clone(), + Endpoint::Output(_) => Endpoint::Output(OutputTarget::DefaultRender), + } + } + + fn label(&self) -> &'static str { + match self { + Endpoint::Mic(_) => "mic", + Endpoint::Output(_) => "out", + } + } +} + +pub struct WasapiSource { + endpoint: Endpoint, + thread: Option, +} + +pub fn mic_source(device: Option<&str>) -> Box { + Box::new(WasapiSource { + endpoint: Endpoint::Mic(device.map(str::to_owned)), + thread: None, + }) +} + +pub fn output_source(target: OutputTarget) -> Box { + Box::new(WasapiSource { + endpoint: Endpoint::Output(target), + thread: None, + }) +} + +/// Active capture (inputs) and render (outputs) endpoints. Enumerated on a +/// fresh thread so COM is initialised the way this code expects, +/// independent of the caller's apartment. Errors yield empty lists. +pub fn list_devices() -> Devices { + thread::spawn(|| { + if initialize_mta().is_err() { + return Devices::default(); + } + let devices = Devices { + inputs: endpoints(&Direction::Capture), + outputs: endpoints(&Direction::Render), + }; + deinitialize(); + devices + }) + .join() + .unwrap_or_default() +} + +fn endpoints(direction: &Direction) -> Vec { + let collection = match DeviceEnumerator::new() + .and_then(|enumerator| enumerator.get_device_collection(direction)) + { + Ok(collection) => collection, + Err(err) => { + log::warn!("cannot enumerate {direction} devices: {err}"); + return Vec::new(); + } + }; + (&collection) + .into_iter() + .filter_map(|device| { + let device = device.ok()?; + Some(DeviceInfo { + id: device.get_id().ok()?, + name: device.get_friendlyname().ok()?, + }) + }) + .collect() +} + +impl CaptureSource for WasapiSource { + fn start(&mut self, on_data: DataCallback) -> Result { + self.stop(); + let endpoint = self.endpoint.clone(); + let (thread, format) = CaptureThread::spawn( + &format!("wasapi {}", endpoint.label()), + move |stop, ready| run(endpoint, stop, ready, on_data), + )?; + self.thread = Some(thread); + Ok(format) + } + + fn stop(&mut self) { + if let Some(thread) = self.thread.take() { + thread.stop(); + } + } +} + +impl Drop for WasapiSource { + fn drop(&mut self) { + self.stop(); + } +} + +/// An opened, started WASAPI capture stream. Lives on the capture thread. +struct Stream { + client: AudioClient, + capture: AudioCaptureClient, + event: Handle, + format: StreamFormat, +} + +impl Drop for Stream { + fn drop(&mut self) { + let _ = self.client.stop_stream(); + } +} + +fn backend(err: WasapiError) -> CaptureError { + CaptureError::Backend(format!("WASAPI: {err}")) +} + +fn is_device_invalidated(err: &WasapiError) -> bool { + matches!(err, WasapiError::Windows(e) if e.code().0 == AUDCLNT_E_DEVICE_INVALIDATED) +} + +fn device_by_id(enumerator: &DeviceEnumerator, id: &str) -> Result { + enumerator + .get_device(id) + .map_err(|_| CaptureError::DeviceNotFound(id.to_owned())) +} + +/// Opens `endpoint` and starts streaming. `format` forces a stream format +/// (used when reopening); otherwise the endpoint's mix format is used, with +/// the samples converted to `f32` by the audio engine. +fn open(endpoint: &Endpoint, format: Option) -> Result { + let mut client = match endpoint { + Endpoint::Output(OutputTarget::Process { root_pid }) => { + AudioClient::new_application_loopback_client(*root_pid, true).map_err(backend)? + } + _ => { + let enumerator = DeviceEnumerator::new().map_err(backend)?; + let device = match endpoint { + Endpoint::Mic(Some(id)) | Endpoint::Output(OutputTarget::Device(id)) => { + device_by_id(&enumerator, id)? + } + Endpoint::Mic(None) => enumerator + .get_default_device_for_role(&Direction::Capture, &Role::Communications) + .map_err(backend)?, + Endpoint::Output(_) => enumerator + .get_default_device_for_role(&Direction::Render, &Role::Communications) + .map_err(backend)?, + }; + device.get_iaudioclient().map_err(backend)? + } + }; + + let format = match (format, endpoint) { + (Some(format), _) => format, + (None, Endpoint::Output(OutputTarget::Process { .. })) => PROCESS_LOOPBACK_FORMAT, + (None, _) => { + let mix = client.get_mixformat().map_err(backend)?; + StreamFormat { + rate: mix.get_samplespersec(), + channels: mix.get_nchannels(), + } + } + }; + let wave_format = WaveFormat::new( + 32, + 32, + &SampleType::Float, + format.rate as usize, + format.channels as usize, + None, + ); + let mode = StreamMode::EventsShared { + autoconvert: true, + buffer_duration_hns: BUFFER_DURATION_HNS, + }; + // Capture on a capture endpoint records it; on a render endpoint (and on + // the application loopback client) it sets up loopback capture. + client + .initialize_client(&wave_format, &Direction::Capture, &mode) + .map_err(backend)?; + let event = client.set_get_eventhandle().map_err(backend)?; + let capture = client.get_audiocaptureclient().map_err(backend)?; + client.start_stream().map_err(backend)?; + Ok(Stream { + client, + capture, + event, + format, + }) +} + +fn run(endpoint: Endpoint, stop: &AtomicBool, ready: Ready, on_data: DataCallback) { + if let Err(err) = initialize_mta().ok() { + let _ = ready.send(Err(CaptureError::Backend(format!( + "COM initialisation failed: {err}" + )))); + return; + } + capture(&endpoint, stop, ready, on_data); + // Every COM object is dropped by now. + deinitialize(); +} + +fn capture(endpoint: &Endpoint, stop: &AtomicBool, ready: Ready, mut on_data: DataCallback) { + let mut stream = match open(endpoint, None) { + Ok(stream) => stream, + Err(err) => { + let _ = ready.send(Err(err)); + return; + } + }; + let format = stream.format; + let _ = ready.send(Ok(format)); + + let label = endpoint.label(); + let mut reopened = false; + let mut bytes = Vec::new(); + let mut samples = Vec::new(); + while !stop.load(Ordering::SeqCst) { + // A timeout is normal (silent loopback); drain regardless, which + // also surfaces an invalidated device. + let _ = stream.event.wait_for_event(WAIT_MS); + let Err(err) = drain(&stream, &mut bytes, &mut samples, &mut on_data) else { + continue; + }; + if !is_device_invalidated(&err) || reopened { + log::warn!("WASAPI capture '{label}' ended: {err}"); + return; + } + reopened = true; + log::warn!("WASAPI capture '{label}': device invalidated, reopening the default device"); + drop(stream); + stream = match open(&endpoint.fallback(), Some(format)) { + Ok(stream) => stream, + Err(err) => { + log::warn!("WASAPI capture '{label}': reopening failed: {err}"); + return; + } + }; + } +} + +/// Delivers every packet currently queued in the capture buffer. +fn drain( + stream: &Stream, + bytes: &mut Vec, + samples: &mut Vec, + on_data: &mut DataCallback, +) -> Result<(), WasapiError> { + let block_align = stream.format.channels as usize * std::mem::size_of::(); + loop { + let frames = stream.capture.get_next_packet_size()?.unwrap_or(0) as usize; + if frames == 0 { + return Ok(()); + } + bytes.resize(frames * block_align, 0); + let (read, info) = stream.capture.read_from_device(bytes)?; + let len = read as usize * stream.format.channels as usize; + samples.clear(); + if info.flags.silent { + samples.resize(len, 0.0); + } else { + samples.extend( + bytes.as_chunks::<4>().0[..len] + .iter() + .map(|raw| f32::from_le_bytes(*raw)), + ); + } + if !samples.is_empty() { + on_data(samples); + } + } +} diff --git a/crates/core/src/capture.rs b/crates/core/src/capture.rs new file mode 100644 index 0000000..d47f280 --- /dev/null +++ b/crates/core/src/capture.rs @@ -0,0 +1,72 @@ +//! Platform-neutral capture interface. +//! +//! The platform crate (`recorder-audio`) implements [`CaptureSource`] for the +//! microphone and for the call app's audio output (WASAPI on Windows, +//! PulseAudio on Linux). The [`Recorder`](crate::recorder::Recorder) only sees +//! this trait, which keeps it testable with fake sources. + +use thiserror::Error; + +/// Format of the interleaved `f32` samples a [`CaptureSource`] delivers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StreamFormat { + pub rate: u32, + pub channels: u16, +} + +/// What the output ("out") track records. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OutputTarget { + /// Only the audio rendered by this process and its children (Windows + /// application loopback). Falls back to [`OutputTarget::DefaultRender`] + /// where per-process capture is not available. + Process { root_pid: u32 }, + /// Everything played on the default (communications) output device. + DefaultRender, + /// Everything played on the output device with this id. + Device(String), +} + +/// One audio endpoint as shown in the device picker. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeviceInfo { + pub id: String, + pub name: String, +} + +/// Available capture inputs (microphones) and outputs (render endpoints). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Devices { + pub inputs: Vec, + pub outputs: Vec, +} + +#[derive(Debug, Error)] +pub enum CaptureError { + #[error("audio device not found: {0}")] + DeviceNotFound(String), + #[error("unsupported: {0}")] + Unsupported(String), + #[error("audio backend error: {0}")] + Backend(String), +} + +/// Receives interleaved `f32` samples from a running [`CaptureSource`]. +pub type DataCallback = Box; + +/// A live audio stream (microphone or output loopback). +/// +/// Implementations run the platform stream on their own thread, so the +/// non-`Send` platform handles never leave it. [`CaptureSource::start`] +/// returns only once the stream is actually open (or failed to open); +/// afterwards `on_data` is called from the capture thread with interleaved +/// `f32` samples in the returned [`StreamFormat`]. [`CaptureSource::stop`] +/// blocks until the capture thread has ended and `on_data` will not be called +/// again (a thread stuck in a platform call may be detached after a timeout +/// instead, so callers must tolerate a stray late call); calling it more than +/// once is harmless. +pub trait CaptureSource: Send { + fn start(&mut self, on_data: DataCallback) -> Result; + + fn stop(&mut self); +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index c23a111..89542fc 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -5,7 +5,9 @@ //! application shell so it can be unit-tested without a GUI or network. pub mod audio; +pub mod capture; pub mod model; +pub mod recorder; pub mod retention; pub mod store; diff --git a/crates/core/src/recorder.rs b/crates/core/src/recorder.rs new file mode 100644 index 0000000..f31d818 --- /dev/null +++ b/crates/core/src/recorder.rs @@ -0,0 +1,610 @@ +//! Two-track call recorder: microphone ("mic") and call output ("out"). +//! +//! Each track runs its own pipeline inline in the capture callback: +//! [`Conditioner`] (downmix + resample to 16 kHz mono) -> [`GapFiller`] +//! (silence for missing packets) -> [`TrackWriter`] (crash-safe WAV). Both +//! tracks measure time against one shared start [`Instant`], and a timer +//! thread pads a track that stopped delivering data, so the two WAVs stay +//! aligned sample-for-sample with wall-clock time. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use thiserror::Error; + +use crate::audio::{rms_level, Conditioner, GapFiller, TrackWriter}; +use crate::capture::{CaptureError, CaptureSource, StreamFormat}; + +/// Track sample rate (matches the on-disk WAV format). +const RATE: u32 = 16_000; +/// How often silent tracks are padded up to wall-clock time. +const PAD_INTERVAL: Duration = Duration::from_millis(500); + +#[derive(Debug, Error)] +pub enum RecorderError { + #[error(transparent)] + Io(#[from] io::Error), + #[error("both audio sources failed to start (mic: {mic}; out: {out})")] + BothSourcesFailed { + mic: CaptureError, + out: CaptureError, + }, +} + +/// Pipeline state of one track, touched by the capture callback, the pad +/// timer and `stop`. +struct TrackState { + /// `None` until the source has reported its stream format. + conditioner: Option, + /// Raw samples that arrived before the format was known. + early: Vec, + gap: GapFiller, + /// `None` once finalized or discarded. + writer: Option, + /// First write error; later writes are skipped. + error: Option, +} + +impl TrackState { + fn write(&mut self, samples: &[f32]) { + if samples.is_empty() || self.error.is_some() { + return; + } + if let Some(writer) = self.writer.as_mut() { + if let Err(err) = writer.write(samples) { + self.error = Some(err); + } + } + } +} + +/// The part of a track shared with its capture callback and the pad timer. +struct TrackShared { + state: Mutex, + /// `f32` bits of the latest chunk's rms level, readable without locking. + level: AtomicU32, + write_failed: AtomicBool, +} + +impl TrackShared { + fn new(writer: TrackWriter) -> Self { + Self { + state: Mutex::new(TrackState { + conditioner: None, + early: Vec::new(), + gap: GapFiller::new(RATE), + writer: Some(writer), + error: None, + }), + level: AtomicU32::new(0f32.to_bits()), + write_failed: AtomicBool::new(false), + } + } + + /// Never fails on a poisoned lock: a panic elsewhere must not take the + /// recording down with it. + fn lock(&self) -> MutexGuard<'_, TrackState> { + self.state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn level(&self) -> f32 { + f32::from_bits(self.level.load(Ordering::Relaxed)) + } + + fn set_level(&self, level: f32) { + self.level.store(level.to_bits(), Ordering::Relaxed); + } + + /// Conditions and writes one capture chunk that arrived at `elapsed`. + fn on_data(&self, elapsed: Duration, raw: &[f32]) { + let mut state = self.lock(); + let Some(conditioner) = state.conditioner.as_mut() else { + state.early.extend_from_slice(raw); + return; + }; + let conditioned = conditioner.process(raw); + self.push(&mut state, elapsed, &conditioned); + } + + /// Installs the conditioner once the stream format is known and pushes + /// whatever arrived before. + fn set_format(&self, format: StreamFormat, elapsed: Duration) { + let mut state = self.lock(); + let mut conditioner = Conditioner::new(format.rate, format.channels); + let early = std::mem::take(&mut state.early); + let conditioned = conditioner.process(&early); + state.conditioner = Some(conditioner); + self.push(&mut state, elapsed, &conditioned); + } + + fn push(&self, state: &mut TrackState, elapsed: Duration, conditioned: &[f32]) { + if conditioned.is_empty() { + return; + } + let samples = state.gap.fill(elapsed, conditioned); + state.write(&samples); + self.set_level(rms_level(conditioned)); + self.note_error(state); + } + + /// Pads the track with silence up to `elapsed` if it is lagging. + fn pad(&self, elapsed: Duration) { + let mut state = self.lock(); + let zeros = state.gap.pad_to(elapsed); + if !zeros.is_empty() { + state.write(&zeros); + self.set_level(0.0); + self.note_error(&state); + } + } + + fn note_error(&self, state: &TrackState) { + if state.error.is_some() { + self.write_failed.store(true, Ordering::Relaxed); + } + } +} + +struct Track { + name: &'static str, + path: PathBuf, + source: Box, + shared: Arc, + start_failed: bool, +} + +impl Track { + fn create(name: &'static str, source: Box, path: &Path) -> io::Result { + let writer = TrackWriter::create(path)?; + Ok(Self { + name, + path: path.to_path_buf(), + source, + shared: Arc::new(TrackShared::new(writer)), + start_failed: false, + }) + } + + fn start(&mut self, started: Instant) -> Result<(), CaptureError> { + let shared = self.shared.clone(); + let on_data = Box::new(move |raw: &[f32]| shared.on_data(started.elapsed(), raw)); + let result = self.source.start(on_data).and_then(|format| { + if format.rate == 0 || format.channels == 0 { + self.source.stop(); + Err(CaptureError::Unsupported(format!( + "stream format {} Hz, {} channels", + format.rate, format.channels + ))) + } else { + Ok(format) + } + }); + match result { + Ok(format) => { + self.shared.set_format(format, started.elapsed()); + Ok(()) + } + Err(err) => { + self.start_failed = true; + Err(err) + } + } + } + + /// Flushes the conditioner and pads up to `elapsed`; returns the number + /// of samples the track now holds. + fn drain(&self, elapsed: Duration) -> u64 { + let mut state = self.shared.lock(); + if let Some(conditioner) = state.conditioner.as_mut() { + let tail = conditioner.flush(); + if !tail.is_empty() { + let samples = state.gap.fill(elapsed, &tail); + state.write(&samples); + } + } + let zeros = state.gap.pad_to(elapsed); + state.write(&zeros); + state.gap.written() + } + + /// Appends `extra` zeros and finalizes the WAV. + fn finish(&self, extra: u64) -> Result { + let mut state = self.shared.lock(); + state.write(&vec![0.0; extra as usize]); + let writer = state.writer.take(); + if let Some(err) = state.error.take() { + return Err(err.into()); + } + match writer { + Some(writer) => Ok(writer.finalize()?), + None => Ok(Duration::ZERO), + } + } + + fn remove_file(&self) { + // Close the file before deleting it (required on Windows). + drop(self.shared.lock().writer.take()); + let _ = fs::remove_file(&self.path); + } +} + +/// Pads both tracks every [`PAD_INTERVAL`] until told to stop. +struct PadTimer { + stop: Sender<()>, + thread: JoinHandle<()>, +} + +impl PadTimer { + fn spawn(tracks: [Arc; 2], started: Instant) -> Self { + let (stop, stop_rx) = mpsc::channel::<()>(); + let thread = thread::spawn(move || { + while let Err(RecvTimeoutError::Timeout) = stop_rx.recv_timeout(PAD_INTERVAL) { + let elapsed = started.elapsed(); + for track in &tracks { + track.pad(elapsed); + } + } + }); + Self { stop, thread } + } + + fn stop(self) { + let _ = self.stop.send(()); + let _ = self.thread.join(); + } +} + +/// A running two-track recording. Track 0 is "mic", track 1 is "out". +pub struct Recorder { + started: Instant, + tracks: [Track; 2], + timer: Option, +} + +impl Recorder { + /// Creates both WAV files and starts both sources. If one source fails to + /// start, the other keeps recording and the failed track is filled with + /// silence (see [`Recorder::failed_tracks`]); only if both fail is this an + /// error, in which case no files are left behind. + pub fn start( + mic: Box, + out: Box, + mic_path: &Path, + out_path: &Path, + ) -> Result { + let mic = Track::create("mic", mic, mic_path)?; + let out = match Track::create("out", out, out_path) { + Ok(out) => out, + Err(err) => { + mic.remove_file(); + return Err(err.into()); + } + }; + + let started = Instant::now(); + let mut recorder = Recorder { + started, + tracks: [mic, out], + timer: None, + }; + let mic_result = recorder.tracks[0].start(started); + let out_result = recorder.tracks[1].start(started); + if let (Err(mic), Err(out)) = (mic_result, out_result) { + recorder.discard(); + return Err(RecorderError::BothSourcesFailed { mic, out }); + } + + recorder.timer = Some(PadTimer::spawn( + [ + recorder.tracks[0].shared.clone(), + recorder.tracks[1].shared.clone(), + ], + started, + )); + Ok(recorder) + } + + /// Current (mic, out) rms levels in `0.0..=1.0`. + pub fn levels(&self) -> (f32, f32) { + (self.tracks[0].shared.level(), self.tracks[1].shared.level()) + } + + /// Names ("mic" / "out") of tracks whose source failed to start or whose + /// file could not be written. + pub fn failed_tracks(&self) -> Vec<&'static str> { + self.tracks + .iter() + .filter(|t| t.start_failed || t.shared.write_failed.load(Ordering::Relaxed)) + .map(|t| t.name) + .collect() + } + + /// Stops both sources and finalizes both WAVs to the same length. + /// Returns the recorded duration. + pub fn stop(mut self) -> Result { + self.halt(); + let elapsed = self.started.elapsed(); + let written = [self.tracks[0].drain(elapsed), self.tracks[1].drain(elapsed)]; + let longest = written[0].max(written[1]); + let mic = self.tracks[0].finish(longest - written[0]); + let out = self.tracks[1].finish(longest - written[1]); + Ok(mic?.max(out?)) + } + + /// Stops both sources and deletes both files. + pub fn discard(mut self) { + self.halt(); + for track in &self.tracks { + track.remove_file(); + } + } + + /// Stops the pad timer and both sources. Idempotent. + fn halt(&mut self) { + if let Some(timer) = self.timer.take() { + timer.stop(); + } + for track in &mut self.tracks { + track.source.stop(); + } + } +} + +impl Drop for Recorder { + fn drop(&mut self) { + self.halt(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capture::DataCallback; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + use std::thread::{self, JoinHandle}; + use std::time::Instant; + + const CHUNK: Duration = Duration::from_millis(20); + + /// Test source: a thread that pushes a constant-amplitude 20 ms chunk + /// every 20 ms, except inside the `silent` window (relative to start), + /// where it sends nothing at all. + struct FakeSource { + format: StreamFormat, + amplitude: f32, + silent: Option<(Duration, Duration)>, + fail: bool, + stop: Arc, + thread: Option>, + } + + impl FakeSource { + fn new(rate: u32, channels: u16, amplitude: f32) -> Self { + Self { + format: StreamFormat { rate, channels }, + amplitude, + silent: None, + fail: false, + stop: Arc::new(AtomicBool::new(false)), + thread: None, + } + } + + fn silent_between(mut self, from: Duration, to: Duration) -> Self { + self.silent = Some((from, to)); + self + } + + fn failing() -> Self { + let mut source = Self::new(16_000, 1, 0.0); + source.fail = true; + source + } + } + + impl CaptureSource for FakeSource { + fn start(&mut self, mut on_data: DataCallback) -> Result { + if self.fail { + return Err(CaptureError::DeviceNotFound("fake".into())); + } + let format = self.format; + let frames = (format.rate as u128 * CHUNK.as_millis() / 1000) as usize; + let chunk = vec![self.amplitude; frames * format.channels as usize]; + let silent = self.silent; + let stop = self.stop.clone(); + self.thread = Some(thread::spawn(move || { + let started = Instant::now(); + let mut sent = 0u32; + while !stop.load(Ordering::SeqCst) { + let due = CHUNK * (sent + 1); + if let Some(wait) = due.checked_sub(started.elapsed()) { + thread::sleep(wait); + } + sent += 1; + let muted = silent.is_some_and(|(from, to)| due > from && due <= to); + if !muted { + on_data(&chunk); + } + } + })); + Ok(format) + } + + fn stop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } + } + + fn read_wav(path: &Path) -> Vec { + hound::WavReader::open(path) + .unwrap() + .into_samples::() + .map(Result::unwrap) + .collect() + } + + fn secs(samples: usize) -> f64 { + samples as f64 / 16_000.0 + } + + #[test] + fn recorder_tracks_equal_length_when_output_silent_gap() { + let dir = tempfile::tempdir().unwrap(); + let mic_path = dir.path().join("mic.wav"); + let out_path = dir.path().join("out.wav"); + let mic = FakeSource::new(48_000, 2, 0.25); + let out = FakeSource::new(16_000, 1, 0.25) + .silent_between(Duration::from_millis(200), Duration::from_millis(1_200)); + + let started = Instant::now(); + let recorder = Recorder::start(Box::new(mic), Box::new(out), &mic_path, &out_path).unwrap(); + thread::sleep(Duration::from_millis(1_500)); + let duration = recorder.stop().unwrap(); + let wall = started.elapsed().as_secs_f64(); + + let mic_samples = read_wav(&mic_path); + let out_samples = read_wav(&out_path); + assert!( + (secs(mic_samples.len()) - secs(out_samples.len())).abs() <= 0.05, + "mic {} s vs out {} s", + secs(mic_samples.len()), + secs(out_samples.len()) + ); + assert!( + (duration.as_secs_f64() - wall).abs() < 0.15, + "duration {duration:?} vs wall clock {wall} s" + ); + + // The gap sits where it happened: silence well inside the window, + // signal again after it. + let window = &out_samples[(0.4 * 16_000.0) as usize..(1.0 * 16_000.0) as usize]; + assert!(window.iter().all(|&s| s == 0), "gap must be silence"); + let after = &out_samples[(1.35 * 16_000.0) as usize..(1.45 * 16_000.0) as usize]; + assert!( + after.iter().any(|&s| s != 0), + "signal must resume after the gap" + ); + } + + #[test] + fn recorder_discard_deletes_files() { + let dir = tempfile::tempdir().unwrap(); + let mic_path = dir.path().join("mic.wav"); + let out_path = dir.path().join("out.wav"); + let recorder = Recorder::start( + Box::new(FakeSource::new(16_000, 1, 0.1)), + Box::new(FakeSource::new(16_000, 1, 0.1)), + &mic_path, + &out_path, + ) + .unwrap(); + thread::sleep(Duration::from_millis(200)); + assert!(mic_path.exists() && out_path.exists()); + + recorder.discard(); + + assert!(!mic_path.exists()); + assert!(!out_path.exists()); + } + + #[test] + fn recorder_levels_reflect_input() { + let dir = tempfile::tempdir().unwrap(); + let recorder = Recorder::start( + Box::new(FakeSource::new(16_000, 1, 0.5)), + Box::new(FakeSource::new(48_000, 2, 0.0)), + &dir.path().join("mic.wav"), + &dir.path().join("out.wav"), + ) + .unwrap(); + thread::sleep(Duration::from_millis(300)); + + let (mic, out) = recorder.levels(); + recorder.stop().unwrap(); + + assert!((mic - 0.5).abs() < 0.05, "mic level {mic}"); + assert!(out < 0.01, "out level {out}"); + } + + #[test] + fn recorder_pads_a_silent_track_while_running() { + let dir = tempfile::tempdir().unwrap(); + let out = FakeSource::new(16_000, 1, 0.25) + .silent_between(Duration::from_millis(100), Duration::from_secs(60)); + let recorder = Recorder::start( + Box::new(FakeSource::new(16_000, 1, 0.25)), + Box::new(out), + &dir.path().join("mic.wav"), + &dir.path().join("out.wav"), + ) + .unwrap(); + thread::sleep(Duration::from_millis(50)); + let (_, out_before) = recorder.levels(); + thread::sleep(Duration::from_millis(700)); + let (_, out_after) = recorder.levels(); + recorder.stop().unwrap(); + + assert!(out_before > 0.2, "out level before the gap {out_before}"); + assert_eq!(out_after, 0.0, "timer padding resets the level"); + } + + #[test] + fn recorder_out_start_failure_keeps_mic_and_marks_failed() { + let dir = tempfile::tempdir().unwrap(); + let mic_path = dir.path().join("mic.wav"); + let out_path = dir.path().join("out.wav"); + let recorder = Recorder::start( + Box::new(FakeSource::new(16_000, 1, 0.25)), + Box::new(FakeSource::failing()), + &mic_path, + &out_path, + ) + .unwrap(); + thread::sleep(Duration::from_millis(400)); + + assert_eq!(recorder.failed_tracks(), vec!["out"]); + let duration = recorder.stop().unwrap(); + + let mic_samples = read_wav(&mic_path); + let out_samples = read_wav(&out_path); + assert_eq!(mic_samples.len(), out_samples.len()); + assert!(duration >= Duration::from_millis(350), "{duration:?}"); + assert!(mic_samples.iter().any(|&s| s != 0), "mic keeps recording"); + assert!( + out_samples.iter().all(|&s| s == 0), + "failed track is silence" + ); + } + + #[test] + fn recorder_fails_when_both_sources_fail() { + let dir = tempfile::tempdir().unwrap(); + let mic_path = dir.path().join("mic.wav"); + let out_path = dir.path().join("out.wav"); + + let result = Recorder::start( + Box::new(FakeSource::failing()), + Box::new(FakeSource::failing()), + &mic_path, + &out_path, + ); + + assert!(matches!( + result, + Err(RecorderError::BothSourcesFailed { .. }) + )); + assert!(!mic_path.exists() && !out_path.exists()); + } +}