diff --git a/app/features/cv/capture/sampler.ts b/app/features/cv/capture/sampler.ts new file mode 100644 index 000000000..8875f4cde --- /dev/null +++ b/app/features/cv/capture/sampler.ts @@ -0,0 +1,71 @@ +/** + * Capture layer: OBS Virtual Camera in via getUserMedia, frames out as + * ImageBitmaps at a low sample rate. The interface downstream is just + * (bitmap, t) — a WHIP/MediaMTX transport can replace this file later. + */ + +export async function openVirtualCamera(deviceId?: string): Promise { + return navigator.mediaDevices.getUserMedia({ + video: { + deviceId: deviceId ? { exact: deviceId } : undefined, + width: { ideal: 1920 }, + height: { ideal: 1080 }, + }, + audio: false, + }); +} + +export async function listVideoInputs(): Promise { + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices.filter((d) => d.kind === "videoinput"); +} + +export type FrameHandler = (bitmap: ImageBitmap, t: number) => void; + +/** + * Sample frames from a playing video element at ~fps using + * requestVideoFrameCallback. Returns a stop function. + */ +export function startSampler( + video: HTMLVideoElement, + fps: number, + onFrame: FrameHandler, +): () => void { + const intervalMs = 1000 / fps; + let lastSample = -Infinity; + let lastMediaTime = -Infinity; + let stopped = false; + let handle = 0; + + const tick = async (now: number, metadata: VideoFrameCallbackMetadata) => { + if (stopped) return; + // Throttle on the callback clock, not metadata.mediaTime: Firefox never + // advances mediaTime for MediaStream-backed videos, which would freeze + // sampling after the first frame. + if (now - lastSample >= intervalMs) { + lastSample = now; + try { + const bitmap = await createImageBitmap(video); + if (stopped) { + bitmap.close(); + return; + } + // Same Firefox quirk for the frame timestamp: fall back to the clock + // when mediaTime isn't advancing so timestamps stay monotonic (the + // timeline's merge windows compare them). + const t = metadata.mediaTime > lastMediaTime ? metadata.mediaTime : now / 1000; + lastMediaTime = Math.max(lastMediaTime, metadata.mediaTime); + onFrame(bitmap, t); + } catch { + // video not ready / tab hidden — skip this frame + } + } + if (!stopped) handle = video.requestVideoFrameCallback(tick); + }; + handle = video.requestVideoFrameCallback(tick); + + return () => { + stopped = true; + video.cancelVideoFrameCallback(handle); + }; +} diff --git a/app/features/cv/capture/vod-frames.ts b/app/features/cv/capture/vod-frames.ts new file mode 100644 index 000000000..cbe6e13d9 --- /dev/null +++ b/app/features/cv/capture/vod-frames.ts @@ -0,0 +1,110 @@ +/** + * VoD frame extraction: step through a video file yielding (frame, t) as + * fast as decoding allows — no real-time playback. The primary path demuxes + * the file and decodes **every frame** sequentially with WebCodecs (via + * mediabunny), yielding the VideoFrames themselves (transferable to the + * analyzer workers with no main-thread conversion); when the container/codec + * can't be read that way, it falls back to seek-stepping a