Show audio as silent in the UI

app/features/scanner/README.md

@@ -80,8 +80,13 @@ scored `kills² + kills / span`. Both controllers run the same
   a `MediaStreamTrackProcessor` → `VideoEncoder` (hardware H.264, ~16 Mbps,
   keyframe every 2 s) into a ring of GOPs holding the last
   `RING_BUFFER_SECONDS`; the audio track through an `AudioEncoder` (AAC,
-  else Opus) into the same ring. Packets are stamped with the wall clock on
-  arrival. A window is cut once `windowClosed` (no kill can join and the
+  else Opus) into the same ring. Packets carry the wall-clock time their
+  frame was captured (noted at encoder input, claimed at output, so encoder
+  latency never shifts audio against video). The audio processor queues
+  `AUDIO_BUFFER_FRAMES` slices so a busy main thread does not drop any, and
+  the encoder's input is watched for signal: a device that opens but sends
+  silence shows on the live status line. A window is cut once
+  `windowClosed` (no kill can join and the
   tail is captured): the GOP at or before its start through its end, muxed
   to MP4 with mediabunny's `EncodedVideoPacketSource` — no decode. Audio is
   the chosen source's own input (`audioInputFor`: same `groupId`, else a

app/features/scanner/capture/ring-buffer.ts

@@ -3,10 +3,12 @@
  * VideoEncoder (hardware H.264 where available, a keyframe every
  * `KEYFRAME_INTERVAL_S`) into a ring of GOPs holding the last `seconds`;
  * the audio track through an AudioEncoder into the same ring. Every packet
- * is stamped with the wall clock as it arrives, the clock the sampler stamps
- * detections with, so a cut asks for wall-clock seconds and gets the GOP at
- * or before its start through the packets up to its end, muxed to MP4 with
- * mediabunny — no decode, so a cut takes milliseconds.
+ * is stamped with the wall clock its frame was captured at (noted as the
+ * frame enters the encoder, claimed as the packet comes out, so encoder
+ * latency does not shift it), the clock the sampler stamps detections with,
+ * so a cut asks for wall-clock seconds and gets the GOP at or before its
+ * start through the packets up to its end, muxed to MP4 with mediabunny —
+ * no decode, so a cut takes milliseconds.
  */
 import {
 	type AudioCodec,
@@ -31,6 +33,10 @@ const AUDIO_CODECS: { codec: string; container: AudioCodec }[] = [
 ];
 const THUMBNAIL_WIDTH = 320;
 const THUMBNAIL_HEIGHT = 180;
+/** 10 ms audio slices the processor queues while the main thread is busy; past it Chromium drops the oldest */
+const AUDIO_BUFFER_FRAMES = 200;
+/** a slice whose loudest sample is under this (about -60 dBFS) carries no signal */
+const SILENCE_PEAK = 0.001;
 
 interface Stamped {
 	packet: EncodedPacket;
@@ -53,6 +59,7 @@ const TrackProcessor = (
 	globalThis as unknown as {
 		MediaStreamTrackProcessor?: new (init: {
 			track: MediaStreamTrack;
+			maxBufferSize?: number;
 		}) => TrackProcessor<VideoFrame | AudioData>;
 	}
 ).MediaStreamTrackProcessor;
@@ -85,6 +92,9 @@ export class ClipRingBuffer {
 	#videoEncoder: VideoEncoder | null = null;
 	#audioEncoder: AudioEncoder | null = null;
 	#readers: ReadableStreamDefaultReader<VideoFrame | AudioData>[] = [];
+	readonly #videoClock = new CaptureClock();
+	readonly #audioClock = new CaptureClock();
+	#audioSignalAt: number | null = null;
 	#stopped = false;
 	#lastKeyframeAt = Number.NEGATIVE_INFINITY;
 
@@ -92,6 +102,15 @@ export class ClipRingBuffer {
 		this.#seconds = seconds;
 	}
 
+	/**
+	 * Wall-clock seconds the audio encoder last got a slice with sound in it;
+	 * null while no audio is being encoded. Stuck in the past = the input is
+	 * open but silent.
+	 */
+	get audioSignalAt(): number | null {
+		return this.#audioSignalAt;
+	}
+
 	/** Starts encoding both tracks; resolves once the video encoder is configured. */
 	async start(stream: MediaStream): Promise<void> {
 		const videoTrack = stream.getVideoTracks()[0];
@@ -214,6 +233,9 @@ export class ClipRingBuffer {
 		this.#audioEncoder = null;
 		this.#gops.length = 0;
 		this.#audio.length = 0;
+		this.#videoClock.clear();
+		this.#audioClock.clear();
+		this.#audioSignalAt = null;
 	}
 
 	async #pumpVideo(track: MediaStreamTrack): Promise<void> {
@@ -242,19 +264,24 @@ export class ClipRingBuffer {
 					if (gop && !gop.thumbnail) gop.thumbnail = thumbnail;
 				});
 			}
+			this.#videoClock.note(frame.timestamp, now);
 			encoder.encode(frame, { keyFrame });
 			frame.close();
 		}
 	}
 
 	async #startAudio(track: MediaStreamTrack): Promise<void> {
-		const reader = new TrackProcessor!({ track }).readable.getReader();
+		const reader = new TrackProcessor!({
+			track,
+			maxBufferSize: AUDIO_BUFFER_FRAMES,
+		}).readable.getReader();
 		this.#readers.push(reader);
 		let configured = false;
 		while (!this.#stopped) {
 			const { value, done } = await reader.read();
 			if (done || !value) break;
 			const data = value as AudioData;
+			const now = Date.now() / 1000;
 			if (!configured) {
 				configured = true;
 				const choice = await firstSupportedAudioCodec(
@@ -278,9 +305,13 @@ export class ClipRingBuffer {
 					sampleRate: data.sampleRate,
 					bitrate: AUDIO_BITRATE,
 				});
+				this.#audioSignalAt = now;
 			}
 			const encoder = this.#audioEncoder;
 			if (encoder?.state === "configured" && encoder.encodeQueueSize < 32) {
+				const peak = peakOf(data);
+				if (peak === null || peak > SILENCE_PEAK) this.#audioSignalAt = now;
+				this.#audioClock.note(data.timestamp, now);
 				encoder.encode(data);
 			}
 			data.close();
@@ -294,7 +325,7 @@ export class ClipRingBuffer {
 		if (meta?.decoderConfig) this.#videoConfig = meta.decoderConfig;
 		const stamped = {
 			packet: EncodedPacket.fromEncodedChunk(chunk),
-			wall: Date.now() / 1000,
+			wall: this.#videoClock.claim(chunk.timestamp),
 		};
 		if (chunk.type === "key" || this.#gops.length === 0) {
 			this.#gops.push({ packets: [stamped] });
@@ -311,7 +342,7 @@ export class ClipRingBuffer {
 		if (meta?.decoderConfig) this.#audioConfig = meta.decoderConfig;
 		this.#audio.push({
 			packet: EncodedPacket.fromEncodedChunk(chunk),
-			wall: Date.now() / 1000,
+			wall: this.#audioClock.claim(chunk.timestamp),
 		});
 	}
 
@@ -348,6 +379,52 @@ export class ClipRingBuffer {
 	}
 }
 
+/**
+ * Wall-clock stamps of the frames handed to an encoder, claimed in order by
+ * the packets that come out: a packet's stamp is its frame's capture time,
+ * whatever the encoder's latency. Output timestamps trail the input ones
+ * only where the encoder saw a gap, so a claim also sweeps up everything
+ * older (frames the encoder dropped).
+ */
+class CaptureClock {
+	readonly #entries: { timestamp: number; wall: number }[] = [];
+
+	/** `timestamp` in microseconds, as WebCodecs frames carry it */
+	note(timestamp: number, wall: number): void {
+		this.#entries.push({ timestamp, wall });
+	}
+
+	/** Wall-clock seconds for the packet at `timestamp` (microseconds); now when nothing was noted for it. */
+	claim(timestamp: number): number {
+		let wall = Date.now() / 1000;
+		let claimed = 0;
+		for (const entry of this.#entries) {
+			if (entry.timestamp > timestamp) break;
+			wall = entry.wall + (timestamp - entry.timestamp) / 1e6;
+			claimed++;
+		}
+		this.#entries.splice(0, claimed);
+		return wall;
+	}
+
+	clear(): void {
+		this.#entries.length = 0;
+	}
+}
+
+/** The loudest sample of the slice's first channel; null when it cannot be read. */
+function peakOf(data: AudioData): number | null {
+	try {
+		const samples = new Float32Array(data.numberOfFrames);
+		data.copyTo(samples, { planeIndex: 0, format: "f32-planar" });
+		let peak = 0;
+		for (const sample of samples) peak = Math.max(peak, Math.abs(sample));
+		return peak;
+	} catch {
+		return null;
+	}
+}
+
 async function firstSupportedVideoCodec(
 	width: number,
 	height: number,

app/features/scanner/components/LiveView.tsx

@@ -57,7 +57,11 @@ export function LiveView() {
 	const clipsNote =
 		live.clips === "on"
 			? live.hasAudio
-				? "Clips on · Audio ✓"
+				? live.audioSignal === "muted"
+					? "Clips on · audio input muted by the browser"
+					: live.audioSignal === "silent"
+						? "Clips on · Audio ✓ but only silence is coming in"
+						: "Clips on · Audio ✓"
 				: `Clips on · no audio${live.audioError ? ` (${live.audioError})` : ""}`
 			: live.clips === "unsupported"
 				? "Clips need a Chromium browser"

app/features/scanner/components/live-session.ts

@@ -82,6 +82,9 @@ const RING_BUFFER_SECONDS = MAX_CLIP_SECONDS + STREAK_MAX_GAP_S + 10;
 
 /** Windows close by time passing, not only by new events. */
 const CLIP_TICK_MS = 5_000;
+/** how often the audio input is checked for a signal, and how long without one counts as silent */
+const AUDIO_CHECK_MS = 1_000;
+const AUDIO_SILENCE_MS = 5_000;
 
 /** Event types the ingested matches are built from — the only ones with a send status. */
 const INGESTABLE_TYPES = [
@@ -95,6 +98,8 @@ const INGESTABLE_TYPES = [
 export type LiveStatus = "idle" | "starting" | "running" | "error";
 /** `unsupported`: no WebCodecs/track processor; `failed`: the encoder refused this stream */
 export type ClipsState = "on" | "off" | "unsupported" | "failed";
+/** `muted`: the browser gets nothing from the device; `silent`: it gets samples, all of them silence */
+export type AudioSignal = "ok" | "silent" | "muted";
 
 export interface LiveSnapshot {
 	status: LiveStatus;
@@ -106,6 +111,8 @@ export interface LiveSnapshot {
 	hasAudio: boolean;
 	/** why it does not, when an audio input was expected */
 	audioError: string | null;
+	/** what the clip encoder is getting from that track; null while clips are off */
+	audioSignal: AudioSignal | null;
 	clips: ClipsState;
 	/** highest gate score on the latest frame (debug) */
 	gateScore: number | null;
@@ -120,6 +127,7 @@ const IDLE: LiveSnapshot = {
 	stream: null,
 	hasAudio: false,
 	audioError: null,
+	audioSignal: null,
 	clips: "off",
 	gateScore: null,
 	detecting: false,
@@ -134,6 +142,7 @@ let ring: ClipRingBuffer | null = null;
 let stopSampler: (() => void) | null = null;
 let retryTimer: ReturnType<typeof setInterval> | null = null;
 let clipTimer: ReturnType<typeof setInterval> | null = null;
+let audioTimer: ReturnType<typeof setInterval> | null = null;
 let unsubscribeFeed: (() => void) | null = null;
 let timeline = new TimelineBuilder();
 const storedIds = new WeakMap<DetectedEvent, number>();
@@ -244,6 +253,7 @@ export async function startCapture(): Promise<void> {
 			}
 		}, UNLINKED_RETRY_TICK_MS);
 		clipTimer = setInterval(clipTick, CLIP_TICK_MS);
+		audioTimer = setInterval(audioCheck, AUDIO_CHECK_MS);
 		unsubscribeFeed = subscribeFeed(clipTick);
 		void trimEvents().catch(() => {});
 		set({
@@ -290,6 +300,8 @@ function release(): void {
 	retryTimer = null;
 	if (clipTimer) clearInterval(clipTimer);
 	clipTimer = null;
+	if (audioTimer) clearInterval(audioTimer);
+	audioTimer = null;
 	unsubscribeFeed?.();
 	unsubscribeFeed = null;
 	ring?.stop();
@@ -374,6 +386,23 @@ async function persist(
 	refreshFeed();
 }
 
+/** Whether the clip encoder is getting sound; a device that opens fine but delivers silence shows up here. */
+function audioCheck(): void {
+	if (snapshot.status !== "running") return;
+	const track = snapshot.stream?.getAudioTracks()[0];
+	const signalAt = ring?.audioSignalAt ?? null;
+	const next: AudioSignal | null = !track
+		? null
+		: track.muted || track.readyState === "ended"
+			? "muted"
+			: signalAt === null
+				? null
+				: Date.now() / 1000 - signalAt > AUDIO_SILENCE_MS / 1000
+					? "silent"
+					: "ok";
+	if (next !== snapshot.audioSignal) set({ audioSignal: next });
+}
+
 /** Cuts every scored window of the running session whose post-roll is in the ring. */
 function clipTick(): void {
 	if (!ring || snapshot.status !== "running") return;