diff --git a/app/features/scanner/README.md b/app/features/scanner/README.md index 7afe9129d..24582c500 100644 --- a/app/features/scanner/README.md +++ b/app/features/scanner/README.md @@ -80,8 +80,13 @@ scored `kills² + kills / span`. Both controllers run the same a `MediaStreamTrackProcessor` → `VideoEncoder` (hardware H.264, ~16 Mbps, keyframe every 2 s) into a ring of GOPs holding the last `RING_BUFFER_SECONDS`; the audio track through an `AudioEncoder` (AAC, - else Opus) into the same ring. Packets are stamped with the wall clock on - arrival. A window is cut once `windowClosed` (no kill can join and the + else Opus) into the same ring. Packets carry the wall-clock time their + frame was captured (noted at encoder input, claimed at output, so encoder + latency never shifts audio against video). The audio processor queues + `AUDIO_BUFFER_FRAMES` slices so a busy main thread does not drop any, and + the encoder's input is watched for signal: a device that opens but sends + silence shows on the live status line. A window is cut once + `windowClosed` (no kill can join and the tail is captured): the GOP at or before its start through its end, muxed to MP4 with mediabunny's `EncodedVideoPacketSource` — no decode. Audio is the chosen source's own input (`audioInputFor`: same `groupId`, else a diff --git a/app/features/scanner/capture/ring-buffer.ts b/app/features/scanner/capture/ring-buffer.ts index 6e23d4b6a..ef678a014 100644 --- a/app/features/scanner/capture/ring-buffer.ts +++ b/app/features/scanner/capture/ring-buffer.ts @@ -3,10 +3,12 @@ * VideoEncoder (hardware H.264 where available, a keyframe every * `KEYFRAME_INTERVAL_S`) into a ring of GOPs holding the last `seconds`; * the audio track through an AudioEncoder into the same ring. Every packet - * is stamped with the wall clock as it arrives, the clock the sampler stamps - * detections with, so a cut asks for wall-clock seconds and gets the GOP at - * or before its start through the packets up to its end, muxed to MP4 with - * mediabunny — no decode, so a cut takes milliseconds. + * is stamped with the wall clock its frame was captured at (noted as the + * frame enters the encoder, claimed as the packet comes out, so encoder + * latency does not shift it), the clock the sampler stamps detections with, + * so a cut asks for wall-clock seconds and gets the GOP at or before its + * start through the packets up to its end, muxed to MP4 with mediabunny — + * no decode, so a cut takes milliseconds. */ import { type AudioCodec, @@ -31,6 +33,10 @@ const AUDIO_CODECS: { codec: string; container: AudioCodec }[] = [ ]; const THUMBNAIL_WIDTH = 320; const THUMBNAIL_HEIGHT = 180; +/** 10 ms audio slices the processor queues while the main thread is busy; past it Chromium drops the oldest */ +const AUDIO_BUFFER_FRAMES = 200; +/** a slice whose loudest sample is under this (about -60 dBFS) carries no signal */ +const SILENCE_PEAK = 0.001; interface Stamped { packet: EncodedPacket; @@ -53,6 +59,7 @@ const TrackProcessor = ( globalThis as unknown as { MediaStreamTrackProcessor?: new (init: { track: MediaStreamTrack; + maxBufferSize?: number; }) => TrackProcessor; } ).MediaStreamTrackProcessor; @@ -85,6 +92,9 @@ export class ClipRingBuffer { #videoEncoder: VideoEncoder | null = null; #audioEncoder: AudioEncoder | null = null; #readers: ReadableStreamDefaultReader[] = []; + readonly #videoClock = new CaptureClock(); + readonly #audioClock = new CaptureClock(); + #audioSignalAt: number | null = null; #stopped = false; #lastKeyframeAt = Number.NEGATIVE_INFINITY; @@ -92,6 +102,15 @@ export class ClipRingBuffer { this.#seconds = seconds; } + /** + * Wall-clock seconds the audio encoder last got a slice with sound in it; + * null while no audio is being encoded. Stuck in the past = the input is + * open but silent. + */ + get audioSignalAt(): number | null { + return this.#audioSignalAt; + } + /** Starts encoding both tracks; resolves once the video encoder is configured. */ async start(stream: MediaStream): Promise { const videoTrack = stream.getVideoTracks()[0]; @@ -214,6 +233,9 @@ export class ClipRingBuffer { this.#audioEncoder = null; this.#gops.length = 0; this.#audio.length = 0; + this.#videoClock.clear(); + this.#audioClock.clear(); + this.#audioSignalAt = null; } async #pumpVideo(track: MediaStreamTrack): Promise { @@ -242,19 +264,24 @@ export class ClipRingBuffer { if (gop && !gop.thumbnail) gop.thumbnail = thumbnail; }); } + this.#videoClock.note(frame.timestamp, now); encoder.encode(frame, { keyFrame }); frame.close(); } } async #startAudio(track: MediaStreamTrack): Promise { - const reader = new TrackProcessor!({ track }).readable.getReader(); + const reader = new TrackProcessor!({ + track, + maxBufferSize: AUDIO_BUFFER_FRAMES, + }).readable.getReader(); this.#readers.push(reader); let configured = false; while (!this.#stopped) { const { value, done } = await reader.read(); if (done || !value) break; const data = value as AudioData; + const now = Date.now() / 1000; if (!configured) { configured = true; const choice = await firstSupportedAudioCodec( @@ -278,9 +305,13 @@ export class ClipRingBuffer { sampleRate: data.sampleRate, bitrate: AUDIO_BITRATE, }); + this.#audioSignalAt = now; } const encoder = this.#audioEncoder; if (encoder?.state === "configured" && encoder.encodeQueueSize < 32) { + const peak = peakOf(data); + if (peak === null || peak > SILENCE_PEAK) this.#audioSignalAt = now; + this.#audioClock.note(data.timestamp, now); encoder.encode(data); } data.close(); @@ -294,7 +325,7 @@ export class ClipRingBuffer { if (meta?.decoderConfig) this.#videoConfig = meta.decoderConfig; const stamped = { packet: EncodedPacket.fromEncodedChunk(chunk), - wall: Date.now() / 1000, + wall: this.#videoClock.claim(chunk.timestamp), }; if (chunk.type === "key" || this.#gops.length === 0) { this.#gops.push({ packets: [stamped] }); @@ -311,7 +342,7 @@ export class ClipRingBuffer { if (meta?.decoderConfig) this.#audioConfig = meta.decoderConfig; this.#audio.push({ packet: EncodedPacket.fromEncodedChunk(chunk), - wall: Date.now() / 1000, + wall: this.#audioClock.claim(chunk.timestamp), }); } @@ -348,6 +379,52 @@ export class ClipRingBuffer { } } +/** + * Wall-clock stamps of the frames handed to an encoder, claimed in order by + * the packets that come out: a packet's stamp is its frame's capture time, + * whatever the encoder's latency. Output timestamps trail the input ones + * only where the encoder saw a gap, so a claim also sweeps up everything + * older (frames the encoder dropped). + */ +class CaptureClock { + readonly #entries: { timestamp: number; wall: number }[] = []; + + /** `timestamp` in microseconds, as WebCodecs frames carry it */ + note(timestamp: number, wall: number): void { + this.#entries.push({ timestamp, wall }); + } + + /** Wall-clock seconds for the packet at `timestamp` (microseconds); now when nothing was noted for it. */ + claim(timestamp: number): number { + let wall = Date.now() / 1000; + let claimed = 0; + for (const entry of this.#entries) { + if (entry.timestamp > timestamp) break; + wall = entry.wall + (timestamp - entry.timestamp) / 1e6; + claimed++; + } + this.#entries.splice(0, claimed); + return wall; + } + + clear(): void { + this.#entries.length = 0; + } +} + +/** The loudest sample of the slice's first channel; null when it cannot be read. */ +function peakOf(data: AudioData): number | null { + try { + const samples = new Float32Array(data.numberOfFrames); + data.copyTo(samples, { planeIndex: 0, format: "f32-planar" }); + let peak = 0; + for (const sample of samples) peak = Math.max(peak, Math.abs(sample)); + return peak; + } catch { + return null; + } +} + async function firstSupportedVideoCodec( width: number, height: number, diff --git a/app/features/scanner/components/LiveView.tsx b/app/features/scanner/components/LiveView.tsx index 9776cb833..b57e0f4ab 100644 --- a/app/features/scanner/components/LiveView.tsx +++ b/app/features/scanner/components/LiveView.tsx @@ -57,7 +57,11 @@ export function LiveView() { const clipsNote = live.clips === "on" ? live.hasAudio - ? "Clips on · Audio ✓" + ? live.audioSignal === "muted" + ? "Clips on · audio input muted by the browser" + : live.audioSignal === "silent" + ? "Clips on · Audio ✓ but only silence is coming in" + : "Clips on · Audio ✓" : `Clips on · no audio${live.audioError ? ` (${live.audioError})` : ""}` : live.clips === "unsupported" ? "Clips need a Chromium browser" diff --git a/app/features/scanner/components/live-session.ts b/app/features/scanner/components/live-session.ts index cebc55088..bd2cc5e32 100644 --- a/app/features/scanner/components/live-session.ts +++ b/app/features/scanner/components/live-session.ts @@ -82,6 +82,9 @@ const RING_BUFFER_SECONDS = MAX_CLIP_SECONDS + STREAK_MAX_GAP_S + 10; /** Windows close by time passing, not only by new events. */ const CLIP_TICK_MS = 5_000; +/** how often the audio input is checked for a signal, and how long without one counts as silent */ +const AUDIO_CHECK_MS = 1_000; +const AUDIO_SILENCE_MS = 5_000; /** Event types the ingested matches are built from — the only ones with a send status. */ const INGESTABLE_TYPES = [ @@ -95,6 +98,8 @@ const INGESTABLE_TYPES = [ export type LiveStatus = "idle" | "starting" | "running" | "error"; /** `unsupported`: no WebCodecs/track processor; `failed`: the encoder refused this stream */ export type ClipsState = "on" | "off" | "unsupported" | "failed"; +/** `muted`: the browser gets nothing from the device; `silent`: it gets samples, all of them silence */ +export type AudioSignal = "ok" | "silent" | "muted"; export interface LiveSnapshot { status: LiveStatus; @@ -106,6 +111,8 @@ export interface LiveSnapshot { hasAudio: boolean; /** why it does not, when an audio input was expected */ audioError: string | null; + /** what the clip encoder is getting from that track; null while clips are off */ + audioSignal: AudioSignal | null; clips: ClipsState; /** highest gate score on the latest frame (debug) */ gateScore: number | null; @@ -120,6 +127,7 @@ const IDLE: LiveSnapshot = { stream: null, hasAudio: false, audioError: null, + audioSignal: null, clips: "off", gateScore: null, detecting: false, @@ -134,6 +142,7 @@ let ring: ClipRingBuffer | null = null; let stopSampler: (() => void) | null = null; let retryTimer: ReturnType | null = null; let clipTimer: ReturnType | null = null; +let audioTimer: ReturnType | null = null; let unsubscribeFeed: (() => void) | null = null; let timeline = new TimelineBuilder(); const storedIds = new WeakMap(); @@ -244,6 +253,7 @@ export async function startCapture(): Promise { } }, UNLINKED_RETRY_TICK_MS); clipTimer = setInterval(clipTick, CLIP_TICK_MS); + audioTimer = setInterval(audioCheck, AUDIO_CHECK_MS); unsubscribeFeed = subscribeFeed(clipTick); void trimEvents().catch(() => {}); set({ @@ -290,6 +300,8 @@ function release(): void { retryTimer = null; if (clipTimer) clearInterval(clipTimer); clipTimer = null; + if (audioTimer) clearInterval(audioTimer); + audioTimer = null; unsubscribeFeed?.(); unsubscribeFeed = null; ring?.stop(); @@ -374,6 +386,23 @@ async function persist( refreshFeed(); } +/** Whether the clip encoder is getting sound; a device that opens fine but delivers silence shows up here. */ +function audioCheck(): void { + if (snapshot.status !== "running") return; + const track = snapshot.stream?.getAudioTracks()[0]; + const signalAt = ring?.audioSignalAt ?? null; + const next: AudioSignal | null = !track + ? null + : track.muted || track.readyState === "ended" + ? "muted" + : signalAt === null + ? null + : Date.now() / 1000 - signalAt > AUDIO_SILENCE_MS / 1000 + ? "silent" + : "ok"; + if (next !== snapshot.audioSignal) set({ audioSignal: next }); +} + /** Cuts every scored window of the running session whose post-roll is in the ring. */ function clipTick(): void { if (!ring || snapshot.status !== "running") return;