feat #14740: play back calls

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
hauke
2026-11-10 13:37:47 +01:00
co-authored by Claude Opus 5.5
parent 8d757fd639
commit a74cfa5864
16 changed files with 1335 additions and 12 deletions
+220
View File
@@ -0,0 +1,220 @@
<script lang="ts">
import { convertFileSrc } from '@tauri-apps/api/core'
import { onDestroy } from 'svelte'
import { errorText, preparePlayback, type CallId } from './api'
import { formatOffset } from './format'
let {
callId,
estimate,
ontime,
onplaying,
}: {
callId: CallId
/** The call's length from its times, shown until the mix is ready. */
estimate: number
/** The playback position, once the mix is loaded (also while paused). */
ontime?: (secs: number) => void
onplaying?: (playing: boolean) => void
} = $props()
let audio: HTMLAudioElement
/** Resolves once the mix is loaded; `false` if preparing it failed. */
let loading: Promise<boolean> | null = null
let preparing = $state(false)
let playing = $state(false)
// svelte-ignore state_referenced_locally
let duration = $state(estimate)
let position = $state(0)
/** While the slider is dragged it shows the drag, not the audio. */
let dragging = $state(false)
let error = $state('')
/** The mix held in memory where the webview cannot stream it (Linux). */
let blobUrl: string | null = null
/** Mixes the call (first use only) and loads the mix into the player. */
function load(): Promise<boolean> {
loading ??= (async () => {
preparing = true
error = ''
try {
const info = await preparePlayback(callId)
duration = info.duration_secs
const url = convertFileSrc(info.path)
try {
await loadSource(url)
} catch (err) {
if (!(err instanceof MediaError) || err.code !== MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED) {
throw err
}
// WebKitGTK (Linux) plays no media from a custom scheme such as
// asset:// (WebView2 gets http://asset.localhost instead), but
// fetches it fine: there the mix is played from memory.
const response = await fetch(url)
if (!response.ok) throw new Error(`Audio konnte nicht geladen werden (${response.status}).`)
blobUrl = URL.createObjectURL(await response.blob())
await loadSource(blobUrl)
}
return true
} catch (err) {
error = err instanceof MediaError ? 'Audio konnte nicht geladen werden.' : errorText(err)
loading = null // a later click tries again
return false
} finally {
preparing = false
}
})()
return loading
}
/** Loads `src` into the player; rejects with its `MediaError`. */
function loadSource(src: string): Promise<void> {
const ready = new Promise<void>((resolve, reject) => {
const ok = () => {
audio.removeEventListener('error', fail)
resolve()
}
const fail = () => {
audio.removeEventListener('loadedmetadata', ok)
reject(audio.error)
}
audio.addEventListener('loadedmetadata', ok, { once: true })
audio.addEventListener('error', fail, { once: true })
})
audio.src = src
return ready
}
async function start() {
try {
await audio.play()
} catch (err) {
error = `Wiedergabe nicht möglich: ${errorText(err)}`
}
}
async function toggle() {
if (playing) {
audio.pause()
} else if (await load()) {
await start()
}
}
/** Jumps to `secs` and plays from there (a click on a transcript line). */
export async function playFrom(secs: number) {
if (!(await load())) return
audio.currentTime = Math.min(secs, duration)
position = audio.currentTime
if (audio.paused) await start()
}
async function seekTo(secs: number) {
dragging = false
position = secs
if (!(await load())) return
audio.currentTime = secs
}
function ontimeupdate() {
if (!dragging) position = audio.currentTime
ontime?.(audio.currentTime)
}
function setPlaying(value: boolean) {
playing = value
onplaying?.(value)
}
onDestroy(() => {
// Lets go of the file, so the call can be deleted right away.
audio?.pause()
audio?.removeAttribute('src')
audio?.load()
if (blobUrl) URL.revokeObjectURL(blobUrl)
onplaying?.(false)
})
</script>
<div class="player">
<audio
bind:this={audio}
preload="metadata"
{ontimeupdate}
onplay={() => setPlaying(true)}
onpause={() => setPlaying(false)}
onended={() => setPlaying(false)}
></audio>
<button
class="play"
disabled={preparing}
title={playing ? 'Pause' : 'Abspielen'}
aria-label={playing ? 'Pause' : 'Abspielen'}
onclick={toggle}
>
{#if preparing}…{:else if playing}❚❚{:else}▶{/if}
</button>
<input
class="timeline"
type="range"
min="0"
max={duration || 1}
step="0.1"
value={position}
disabled={preparing}
aria-label="Position"
oninput={(e) => {
dragging = true
position = Number(e.currentTarget.value)
}}
onchange={(e) => seekTo(Number(e.currentTarget.value))}
/>
<span class="time">{formatOffset(position)} / {formatOffset(duration)}</span>
{#if preparing}
<span class="muted">Audio wird vorbereitet…</span>
{:else if error}
<span class="error">{error}</span>
{/if}
</div>
<style>
.player {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 6px 10px;
min-width: 0;
flex: 1;
}
.play {
flex-shrink: 0;
width: 30px;
height: 30px;
border-radius: 50%;
border: none;
background: var(--primary);
color: #fff;
font-size: 11px;
line-height: 1;
cursor: pointer;
}
.play:disabled {
opacity: 0.6;
cursor: default;
}
.timeline {
flex: 1 1 160px;
min-width: 120px;
accent-color: var(--primary);
}
.time {
color: var(--muted);
font-size: 12px;
font-variant-numeric: tabular-nums;
white-space: nowrap;
}
</style>
+101 -5
View File
@@ -8,6 +8,9 @@
onrename,
onmerge,
onunmerge,
activeIndex = null,
follow = false,
onseek,
}: {
/** The transcript with merges applied: merged speakers no longer appear. */
segments: SegmentView[]
@@ -18,8 +21,45 @@
onmerge: (fromKey: string, intoKey: string) => Promise<void>
/** Undoes the merge of `fromKey`. */
onunmerge: (fromKey: string) => Promise<void>
/** The line being heard; it is highlighted. */
activeIndex?: number | null
/** Scrolls the line being heard into view when it leaves the view. */
follow?: boolean
/** Plays from `secs`, a line's start (only while the audio is there). */
onseek?: (secs: number) => void
} = $props()
let lineElements: HTMLElement[] = $state([])
/** A click on a line's text plays from it, unless text was selected. */
function lineClick(start: number) {
if (window.getSelection()?.toString()) return
onseek?.(start)
}
/** The nearest ancestor that scrolls vertically. */
function scrollParent(node: HTMLElement): HTMLElement | null {
for (let el = node.parentElement; el; el = el.parentElement) {
if (/(auto|scroll)/.test(getComputedStyle(el).overflowY)) return el
}
return null
}
// Following the playback scrolls only once the line has left the view
// (also when it hides under the sticky player), and then centres it.
$effect(() => {
if (!follow || activeIndex === null) return
const line = lineElements[activeIndex]
const parent = line && scrollParent(line)
if (!line || !parent) return
const view = parent.getBoundingClientRect()
const box = line.getBoundingClientRect()
const covered = parseFloat(getComputedStyle(line).scrollMarginTop) || 0
if (box.top < view.top + covered || box.bottom > view.bottom) {
line.scrollIntoView({ block: 'center' })
}
})
// "Ich" is always the primary blue; remote speakers cycle through the rest.
const ME_COLOR = '#3b5bdb'
const REMOTE_COLORS = ['#2b8a3e', '#c2255c', '#e8590c', '#7048e8', '#0c8599', '#5c940d']
@@ -186,8 +226,27 @@
<div class="lines">
{#each segments as segment, i (i)}
<div class="line">
<span class="time">{formatOffset(segment.start)}</span>
<!-- The time is the keyboard-accessible way to play a line. -->
<!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_static_element_interactions -->
<div
class="line"
class:active={i === activeIndex}
class:seekable={onseek !== undefined}
bind:this={lineElements[i]}
onclick={() => lineClick(segment.start)}
>
{#if onseek}
<button
class="time"
title="Ab hier abspielen"
onclick={(e) => {
e.stopPropagation()
onseek(segment.start)
}}>{formatOffset(segment.start)}</button
>
{:else}
<span class="time">{formatOffset(segment.start)}</span>
{/if}
<span class="speaker" style:color={speakerColor(segment.speaker_key)}>{segment.label}</span>
<span class="text">{segment.text}</span>
</div>
@@ -324,12 +383,33 @@
.lines {
display: grid;
grid-template-columns: auto auto 1fr;
gap: 8px 12px;
margin-top: 14px;
column-gap: 12px;
margin: 10px -8px 0;
}
/* One row of the transcript grid that can be highlighted as a whole. */
.line {
display: contents;
display: grid;
grid-column: 1 / -1;
grid-template-columns: subgrid;
padding: 4px 8px;
border-radius: 6px;
/* Room for the sticky player when the line is scrolled into view. */
scroll-margin-top: 64px;
}
.line.seekable {
cursor: pointer;
}
.line.seekable:hover {
background: var(--bg);
}
.line.active,
.line.active:hover {
background: var(--primary-soft);
box-shadow: inset 3px 0 0 var(--primary);
}
.time {
@@ -339,6 +419,22 @@
padding-top: 1px;
}
button.time {
align-self: start;
border: none;
background: none;
padding: 1px 0 0;
font-family: inherit;
line-height: inherit;
text-align: left;
cursor: pointer;
}
button.time:hover {
color: var(--primary);
text-decoration: underline;
}
.speaker {
font-weight: 600;
max-width: 160px;
+12
View File
@@ -140,6 +140,15 @@ export interface CallDetail {
merges: SpeakerMerge[]
/** The transcript is being uploaded right now. */
uploading: boolean
/** The call's audio is still on this PC and can be played. */
audio_available: boolean
}
/** A call's two tracks mixed into one file for playback. */
export interface PlaybackInfo {
/** The mixed WAV in the playback cache; play it via `convertFileSrc`. */
path: string
duration_secs: number
}
/** A newer version the release feed offers. */
@@ -188,6 +197,9 @@ export const getCallBrief = (id: CallId) => invoke<CallBrief>('get_call_brief',
export const transcribeCall = (id: CallId) => invoke<void>('transcribe_call', { id })
export const retryCall = (id: CallId) => invoke<void>('retry_call', { id })
export const discardCall = (id: CallId) => invoke<void>('discard_call', { id })
/** Mixes the call's tracks (or reuses the cached mix); refused while the
* call is recorded or processed and once its audio is gone. */
export const preparePlayback = (id: CallId) => invoke<PlaybackInfo>('prepare_playback', { id })
export const listCalls = () => invoke<CallListItem[]>('list_calls')
export const getCall = (id: CallId) => invoke<CallDetail>('get_call', { id })
/** An empty name restores the default label. */
+21
View File
@@ -0,0 +1,21 @@
// Couples playback to the transcript: which line is being heard.
/** The part of a transcript line the lookup needs. */
export interface Timed {
start: number
end: number
}
/** Index of the line heard at `secs`: the one whose `[start, end)` holds
* it — of overlapping lines (both sides talking at once) the one that
* started last. `null` before the first line, in a pause between lines and
* after the last one. */
export function segmentAt(segments: readonly Timed[], secs: number): number | null {
let found: number | null = null
segments.forEach((segment, i) => {
if (segment.start <= secs && secs < segment.end) {
if (found === null || segment.start >= segments[found].start) found = i
}
})
return found
}
+123 -1
View File
@@ -5,6 +5,7 @@
errorText,
getCall,
getRecordingState,
getSettings,
listCalls,
mergeSpeakers,
onBackendLoginError,
@@ -27,6 +28,8 @@
} from '../lib/api'
import CallList from '../lib/CallList.svelte'
import { formatDateTime, formatDetailTime, formatMinutes } from '../lib/format'
import Player from '../lib/Player.svelte'
import { segmentAt } from '../lib/playback'
import StatusPill from '../lib/StatusPill.svelte'
import TranscriptView from '../lib/TranscriptView.svelte'
import UpdateBanner from '../lib/UpdateBanner.svelte'
@@ -56,6 +59,16 @@
let uploading = $state<CallId[]>([])
let uploadErrors = $state<Record<CallId, string>>({})
// Playback of the shown call (the player itself lives in `Player`).
let player = $state<ReturnType<typeof Player>>()
/** The playback position once the mix is loaded, else `null`. */
let playerTime = $state<number | null>(null)
let playing = $state(false)
/** Whether the transcript scrolls along; scrolling by hand ends it. */
let follow = $state(true)
/** For "… werden N Tage nach der Transkription gelöscht". */
let retentionDays = $state<number | null>(null)
let editingTitle = $state(false)
let titleDraft = $state('')
let savingTitle = $state(false)
@@ -103,9 +116,16 @@
}
}
function resetPlayback() {
playerTime = null
playing = false
follow = true
}
function select(id: CallId) {
if (id === selectedId) return
selectedId = id
resetPlayback()
detail = null
detailError = ''
actionError = ''
@@ -118,6 +138,7 @@
function clearSelection() {
selectedId = null
resetPlayback()
detail = null
detailError = ''
actionError = ''
@@ -145,6 +166,9 @@
}),
]
refreshList()
getSettings()
.then((settings) => (retentionDays = settings.retention_days))
.catch(() => {})
getRecordingState()
.then((state) => (recordingState = state))
.catch((err) => (recordError = errorText(err)))
@@ -310,6 +334,28 @@
node.select()
}
/** A click on a transcript line: plays from there and follows again. */
function playFrom(secs: number) {
follow = true
player?.playFrom(secs)
}
const SCROLL_KEYS = ['ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' ']
/** Scrolling the detail by hand while playing stops following the
* playback (until "folgen" or a click on a line). Only user input counts:
* the transcript's own scrolling raises none of these events. */
function manualScroll(event: Event) {
if (!playing) return
if (event instanceof KeyboardEvent) {
const target = event.target as HTMLElement
if (!SCROLL_KEYS.includes(event.key) || target.closest('input, textarea, button')) return
}
// A press on the scroll bar (not on the content).
if (event instanceof PointerEvent && event.target !== event.currentTarget) return
follow = false
}
/** "Microsoft Teams · heute 10:14 · 32 min" (no duration while recording). */
function defaultTitle(call: CallListItem): string {
const parts = [call.app, formatDetailTime(call.started_at)]
@@ -330,6 +376,11 @@
// Calls that still need the user's attention (not transcribed, or
// transcribed but not uploaded here) — shown as "Gespräche · N offen".
let openCount = $derived(calls.filter((c) => c.needs_action).length)
/** The transcript line being heard, highlighted while the mix is loaded. */
let activeIndex = $derived(
detail && playerTime !== null ? segmentAt(detail.segments, playerTime) : null,
)
</script>
<main class="main">
@@ -364,7 +415,15 @@
</div>
</aside>
<section class="detail">
<!-- The handlers only watch for scrolling by hand (see manualScroll). -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
<section
class="detail"
onwheel={manualScroll}
ontouchmove={manualScroll}
onkeydown={manualScroll}
onpointerdown={manualScroll}
>
{#if detail}
{@const call = detail.call}
<header class="detail-header">
@@ -433,6 +492,36 @@
{/if}
</header>
{#if call.status !== 'recording' && call.status !== 'processing'}
{#if detail.audio_available}
<div class="playback-bar">
{#key call.id}
<Player
bind:this={player}
callId={call.id}
estimate={call.duration_secs}
ontime={(secs) => (playerTime = secs)}
onplaying={(value) => (playing = value)}
/>
{/key}
{#if call.status === 'done' && detail.segments.length > 0}
<button
class="btn small follow"
class:on={follow}
aria-pressed={follow}
title="Das Transkript scrollt mit der Wiedergabe mit"
onclick={() => (follow = !follow)}>folgen</button
>
{/if}
</div>
{:else}
<p class="muted no-audio">
Audio nicht mehr vorhanden – Aufnahmen werden {retentionDays ?? '…'} Tage nach der
Transkription gelöscht (<a href="#/settings">Einstellungen</a>).
</p>
{/if}
{/if}
<div class="body">
{#if call.status === 'recording'}
<p class="state rec">● Aufnahme läuft</p>
@@ -456,6 +545,9 @@
onrename={rename}
onmerge={merge}
onunmerge={unmerge}
{activeIndex}
follow={follow && playing}
onseek={detail.audio_available ? playFrom : undefined}
/>
{/if}
</div>
@@ -681,6 +773,36 @@
margin: 10px 0 0;
}
/* Stays at the top while the transcript scrolls under it. */
.playback-bar {
position: sticky;
top: -18px; /* the detail's top padding: flush with its edge */
z-index: 2;
display: flex;
align-items: center;
gap: 10px;
margin: 0 -24px 14px;
padding: 8px 24px;
background: var(--bg);
border-bottom: 1px solid var(--border);
}
.follow {
flex-shrink: 0;
color: var(--muted);
font-weight: 600;
}
.follow.on {
border-color: var(--primary);
background: var(--primary-soft);
color: var(--primary);
}
.no-audio {
margin: 0 0 14px;
}
.state {
color: var(--muted);
}