diff --git a/app/features/scanner/README.md b/app/features/scanner/README.md index 24582c500..639585cba 100644 --- a/app/features/scanner/README.md +++ b/app/features/scanner/README.md @@ -76,22 +76,47 @@ clip would outgrow `MAX_CLIP_SECONDS`; `MIN_KILLS` (4) makes it a window, scored `kills² + kills / span`. Both controllers run the same `scoreWindows(match, deaths)` → cut → `store/clips.ts` path: -- **Live** (`capture/ring-buffer.ts`): the stream's video track runs through - a `MediaStreamTrackProcessor` → `VideoEncoder` (hardware H.264, ~16 Mbps, - keyframe every 2 s) into a ring of GOPs holding the last - `RING_BUFFER_SECONDS`; the audio track through an `AudioEncoder` (AAC, - else Opus) into the same ring. Packets carry the wall-clock time their - frame was captured (noted at encoder input, claimed at output, so encoder - latency never shifts audio against video). The audio processor queues - `AUDIO_BUFFER_FRAMES` slices so a busy main thread does not drop any, and - the encoder's input is watched for signal: a device that opens but sends - silence shows on the live status line. A window is cut once - `windowClosed` (no kill can join and the +- One capture per browser profile: `startCapture` holds a Web Lock + (`CAPTURE_LOCK`) for its lifetime, so a second tab gets an error instead + of a second pipeline writing every game twice (two samplers on one store + double each event, and the repeated scoreboard opens a duplicate match). +- **Live** (`capture/ring-buffer.ts` + `ring-buffer.worker.ts`): the + stream's tracks become `MediaStreamTrackProcessor` streams, transferred to + a worker so a busy page never costs the footage a frame (on the main + thread ~12% of a 60 fps track was lost to the one-frame processor + buffer). There the video runs through a `VideoEncoder` (hardware H.264, + ~16 Mbps, keyframe every 2 s) into a ring of GOPs holding the last + `RING_BUFFER_SECONDS`; the audio through an `AudioEncoder` (AAC, else + Opus) into the same ring. `openCapture` asks for 60 fps explicitly, as + Chromium's default of 30 would halve a capture card. Packets carry the + wall-clock time their frame was captured (noted at encoder input, claimed + at output, so encoder latency never shifts audio against video): video by + frame timestamp, audio by sample position (`SampleClock`), because audio + timestamps drift against the sample count on some sources (a display + capture, an element's `captureStream`) while the AAC encoder counts + samples — matching by timestamp pushed later packets into the future and + a cut lost its second half of audio. In the MP4 the audio runs on by + sample count from its first packet and only jumps forward on a delivery + gap over `AUDIO_RESYNC_S`. The remaining constant offset between a + capture card's picture and another path's sound (desktop audio, a + loopback device) is the `audioOffsetMs` setting, applied at cut time. + The audio input is watched for signal: a device that opens but sends + silence, and an encoder that gives up, show on the live status line. A window is cut once `windowClosed` (no + kill can join and the tail is captured): the GOP at or before its start through its end, muxed - to MP4 with mediabunny's `EncodedVideoPacketSource` — no decode. Audio is - the chosen source's own input (`audioInputFor`: same `groupId`, else a - shared label prefix); OBS Virtual Camera carries none, so its clips are - silent, which the source select says. + to MP4 with mediabunny's `EncodedVideoPacketSource` — no decode; a ring + that only begins more than `MAX_MISSING_LEAD_S` after the asked start + (a capture restarted mid-streak) yields nothing rather than a clip + missing its kills. A kill belongs to one clip: a window over footage + already cut — redrawn by a late-read kill, or seen again because the + session outlived the capture that first cut it — is skipped unless it + scores higher, when it is cut and the clips it overlaps are deleted + (`cuts` plus the session's saved clips, `live-session.ts`). What + the clips hear is the `audioSource` setting: the source's own input + (`audioInputFor`: same `groupId`, else a shared label prefix; OBS Virtual + Camera carries none), the desktop's sound (`openDesktopAudio`: the share + picker's audio track, opened before the camera so the click's activation + still covers it, its video surface dropped), any audio input, or off. - **VoD** (`capture/vod-clips.ts`): packets from the keyframe at or before the window start are copied into a fresh MP4 (video + audio, no re-encode, so a minute of 1080p takes well under a second). mediabunny's diff --git a/app/features/scanner/capture/ring-buffer-protocol.ts b/app/features/scanner/capture/ring-buffer-protocol.ts new file mode 100644 index 000000000..3528e2685 --- /dev/null +++ b/app/features/scanner/capture/ring-buffer-protocol.ts @@ -0,0 +1,41 @@ +/** Messages between the ring buffer's main-thread client and its worker. */ + +export interface RingBufferClip { + blob: Blob; + /** wall-clock seconds the clip really starts at (the keyframe) */ + start: number; + end: number; + hasAudio: boolean; + thumbnail?: string; +} + +export type RingWorkerRequest = + | { + kind: "start"; + video: ReadableStream; + audio: ReadableStream | null; + width: number; + height: number; + framerate: number; + /** how much footage the ring keeps */ + seconds: number; + } + | { + kind: "cut"; + id: number; + start: number; + end: number; + /** seconds to move the sound later (negative: earlier) against the picture */ + audioOffset: number; + }; + +export type RingWorkerResponse = + | { kind: "started" } + /** the encoder could not be set up, or gave up later; no more clips this session */ + | { kind: "error"; message: string } + | { kind: "cut"; id: number; clip: RingBufferClip | null } + | { kind: "cutError"; id: number; message: string } + /** the audio encoder gave up; clips from here on are silent */ + | { kind: "audioError"; message: string } + /** wall-clock seconds the audio encoder last got a slice with sound in it */ + | { kind: "audioSignal"; at: number }; diff --git a/app/features/scanner/capture/ring-buffer.ts b/app/features/scanner/capture/ring-buffer.ts index ef678a014..4e9f52a66 100644 --- a/app/features/scanner/capture/ring-buffer.ts +++ b/app/features/scanner/capture/ring-buffer.ts @@ -1,54 +1,23 @@ /** - * Live clip footage: the capture stream's video track runs through a - * VideoEncoder (hardware H.264 where available, a keyframe every - * `KEYFRAME_INTERVAL_S`) into a ring of GOPs holding the last `seconds`; - * the audio track through an AudioEncoder into the same ring. Every packet - * is stamped with the wall clock its frame was captured at (noted as the - * frame enters the encoder, claimed as the packet comes out, so encoder - * latency does not shift it), the clock the sampler stamps detections with, - * so a cut asks for wall-clock seconds and gets the GOP at or before its - * start through the packets up to its end, muxed to MP4 with mediabunny — - * no decode, so a cut takes milliseconds. + * Live clip footage: the main-thread side of the ring buffer. It turns the + * capture's tracks into MediaStreamTrackProcessor streams and hands them to + * ring-buffer.worker.ts, which encodes and keeps the last `seconds` and + * cuts clips out of them; nothing here touches a frame, so the page being + * busy (React, the sampler, a cut) never costs the footage a frame. The + * processors queue a few frames for the moments the worker is behind. */ -import { - type AudioCodec, - BufferTarget, - EncodedAudioPacketSource, - EncodedPacket, - EncodedVideoPacketSource, - Mp4OutputFormat, - Output, -} from "mediabunny"; +import type { + RingBufferClip, + RingWorkerRequest, + RingWorkerResponse, +} from "./ring-buffer-protocol"; -/** ~indistinguishable from the source at 720p60 per the auto-clipper measurements */ -const VIDEO_BITRATE = 16_000_000; -const KEYFRAME_INTERVAL_S = 2; -const AUDIO_BITRATE = 160_000; -/** high profile at level 5.1 covers 1080p60; the fallbacks trade profile for reach */ -const VIDEO_CODECS = ["avc1.640033", "avc1.4d0033", "avc1.42e033"]; -/** AAC is the MP4-native choice; Chromium encodes it on most but not all platforms */ -const AUDIO_CODECS: { codec: string; container: AudioCodec }[] = [ - { codec: "mp4a.40.2", container: "aac" }, - { codec: "opus", container: "opus" }, -]; -const THUMBNAIL_WIDTH = 320; -const THUMBNAIL_HEIGHT = 180; -/** 10 ms audio slices the processor queues while the main thread is busy; past it Chromium drops the oldest */ +export type { RingBufferClip } from "./ring-buffer-protocol"; + +/** frames the video processor queues before Chromium drops the oldest */ +const VIDEO_BUFFER_FRAMES = 4; +/** 10 ms audio slices likewise: a couple of seconds, audio is cheap to hold */ const AUDIO_BUFFER_FRAMES = 200; -/** a slice whose loudest sample is under this (about -60 dBFS) carries no signal */ -const SILENCE_PEAK = 0.001; - -interface Stamped { - packet: EncodedPacket; - /** wall-clock seconds the frame/sample arrived at */ - wall: number; -} - -interface Gop { - packets: Stamped[]; - /** JPEG data URL of the keyframe, for the clip card */ - thumbnail?: string; -} interface TrackProcessor { readable: ReadableStream; @@ -64,19 +33,11 @@ const TrackProcessor = ( } ).MediaStreamTrackProcessor; -export interface RingBufferClip { - blob: Blob; - /** wall-clock seconds the clip really starts at (the keyframe) */ - start: number; - end: number; - hasAudio: boolean; - thumbnail?: string; -} - /** Whether this browser can keep a clip ring buffer at all. */ export function supportsRingBuffer(): boolean { return ( TrackProcessor !== undefined && + typeof Worker !== "undefined" && typeof VideoEncoder !== "undefined" && typeof VideoEncoder.isConfigSupported === "function" ); @@ -84,19 +45,18 @@ export function supportsRingBuffer(): boolean { export class ClipRingBuffer { readonly #seconds: number; - readonly #gops: Gop[] = []; - readonly #audio: Stamped[] = []; - #videoConfig: VideoDecoderConfig | undefined; - #audioConfig: AudioDecoderConfig | undefined; - #audioContainerCodec: AudioCodec | null = null; - #videoEncoder: VideoEncoder | null = null; - #audioEncoder: AudioEncoder | null = null; - #readers: ReadableStreamDefaultReader[] = []; - readonly #videoClock = new CaptureClock(); - readonly #audioClock = new CaptureClock(); + #worker: Worker | null = null; #audioSignalAt: number | null = null; - #stopped = false; - #lastKeyframeAt = Number.NEGATIVE_INFINITY; + #audioFailure: string | null = null; + #failure: string | null = null; + #nextCutId = 0; + readonly #cuts = new Map< + number, + { + resolve: (clip: RingBufferClip | null) => void; + reject: (error: Error) => void; + } + >(); constructor(seconds: number) { this.#seconds = seconds; @@ -111,359 +71,104 @@ export class ClipRingBuffer { return this.#audioSignalAt; } - /** Starts encoding both tracks; resolves once the video encoder is configured. */ + /** Why the audio encoder gave up, once it has; clips from then on are silent. */ + get audioFailure(): string | null { + return this.#audioFailure; + } + + /** Starts encoding both tracks; resolves once the worker's video encoder is configured. */ async start(stream: MediaStream): Promise { const videoTrack = stream.getVideoTracks()[0]; if (!videoTrack || !TrackProcessor) throw new Error("no video track"); const settings = videoTrack.getSettings(); - const width = settings.width ?? 1920; - const height = settings.height ?? 1080; - const framerate = settings.frameRate ?? 60; - const codec = await firstSupportedVideoCodec(width, height, framerate); - if (!codec) throw new Error("no H.264 encoder available for clips"); - - this.#videoEncoder = new VideoEncoder({ - output: (chunk, meta) => this.#onVideoChunk(chunk, meta), - error: () => this.#fail(), - }); - this.#videoEncoder.configure({ - codec, - width, - height, - bitrate: VIDEO_BITRATE, - framerate, - latencyMode: "realtime", - hardwareAcceleration: "prefer-hardware", - }); - void this.#pumpVideo(videoTrack); - + const video = new TrackProcessor({ + track: videoTrack, + maxBufferSize: VIDEO_BUFFER_FRAMES, + }).readable as ReadableStream; const audioTrack = stream.getAudioTracks()[0]; - if (audioTrack && typeof AudioEncoder !== "undefined") { - void this.#startAudio(audioTrack); - } + const audio = audioTrack + ? (new TrackProcessor({ + track: audioTrack, + maxBufferSize: AUDIO_BUFFER_FRAMES, + }).readable as ReadableStream) + : null; + + const worker = new Worker( + new URL("./ring-buffer.worker.ts", import.meta.url), + { type: "module" }, + ); + this.#worker = worker; + const started = new Promise((resolve, reject) => { + worker.onmessage = (e: MessageEvent) => { + const msg = e.data; + if (msg.kind === "started") resolve(); + else if (msg.kind === "error") { + reject(new Error(msg.message)); + this.#failure = msg.message; + for (const cut of this.#cuts.values()) { + cut.reject(new Error(msg.message)); + } + this.#cuts.clear(); + } else if (msg.kind === "cut") { + this.#cuts.get(msg.id)?.resolve(msg.clip); + this.#cuts.delete(msg.id); + } else if (msg.kind === "cutError") { + this.#cuts.get(msg.id)?.reject(new Error(msg.message)); + this.#cuts.delete(msg.id); + } else if (msg.kind === "audioSignal") { + this.#audioSignalAt = msg.at; + } else if (msg.kind === "audioError") { + this.#audioFailure = msg.message; + } + }; + worker.onerror = (event) => { + reject(new Error(event.message || "clip worker failed")); + }; + }); + this.#send( + { + kind: "start", + video, + audio, + width: settings.width ?? 1920, + height: settings.height ?? 1080, + framerate: settings.frameRate ?? 60, + seconds: this.#seconds, + }, + audio ? [video, audio] : [video], + ); + await started; } /** * The footage between two wall-clock seconds as an MP4, from the keyframe - * at or before `start`. Null when the ring holds nothing for the range. + * at or before `start`, the sound moved `audioOffset` seconds later + * (negative: earlier). Null when the ring holds nothing for the range. */ - async cut(start: number, end: number): Promise { - const gopIndex = this.#gops.findLastIndex( - (gop) => gop.packets[0]!.wall <= start, - ); - const gops = this.#gops.slice(Math.max(0, gopIndex)); - const video = gops - .flatMap((gop) => gop.packets) - .filter((stamped) => stamped.wall <= end); - const first = video[0]; - if (!first || !this.#videoConfig) return null; - const clipStart = first.wall; - const clipEnd = video.at(-1)!.wall; - const thumbnail = - gops.findLast((gop) => gop.thumbnail && gop.packets[0]!.wall <= end) - ?.thumbnail ?? gops[0]?.thumbnail; - - const format = new Mp4OutputFormat({ fastStart: "in-memory" }); - const target = new BufferTarget(); - const output = new Output({ format, target }); - const videoSource = new EncodedVideoPacketSource("avc"); - output.addVideoTrack(videoSource); - const audio = this.#audioContainerCodec - ? this.#audio.filter( - (stamped) => stamped.wall >= clipStart && stamped.wall <= clipEnd, - ) - : []; - const audioSource = - audio.length > 0 && this.#audioContainerCodec - ? new EncodedAudioPacketSource(this.#audioContainerCodec) - : null; - if (audioSource) output.addAudioTrack(audioSource); - await output.start(); - try { - const base = first.packet.timestamp; - let meta: { decoderConfig: VideoDecoderConfig } | undefined = { - decoderConfig: this.#videoConfig, - }; - for (const { packet } of video) { - await videoSource.add( - packet.clone({ timestamp: packet.timestamp - base }), - meta, - ); - meta = undefined; - } - videoSource.close(); - if (audioSource) { - // audio timestamps live on the audio track's own clock: align the - // first sample to where its arrival sits against the clip's first frame - const firstAudio = audio[0]!; - const audioBase = - firstAudio.packet.timestamp - (firstAudio.wall - clipStart); - let audioMeta: { decoderConfig?: AudioDecoderConfig } | undefined = { - decoderConfig: this.#audioConfig, - }; - for (const { packet } of audio) { - const timestamp = packet.timestamp - audioBase; - if (timestamp < 0) continue; - await audioSource.add(packet.clone({ timestamp }), audioMeta); - audioMeta = undefined; - } - audioSource.close(); - } - await output.finalize(); - } catch (error) { - await output.cancel(); - throw error; - } - return { - blob: new Blob([target.buffer!], { type: format.mimeType }), - start: clipStart, - end: clipEnd, - hasAudio: audioSource !== null, - thumbnail, - }; + cut( + start: number, + end: number, + audioOffset = 0, + ): Promise { + if (this.#failure) return Promise.reject(new Error(this.#failure)); + if (!this.#worker) return Promise.resolve(null); + const id = this.#nextCutId++; + return new Promise((resolve, reject) => { + this.#cuts.set(id, { resolve, reject }); + this.#send({ kind: "cut", id, start, end, audioOffset }); + }); } + /** Ends the worker; the tracks' processors close with the tracks. */ stop(): void { - this.#stopped = true; - for (const reader of this.#readers) void reader.cancel().catch(() => {}); - this.#readers = []; - this.#videoEncoder?.close(); - this.#videoEncoder = null; - this.#audioEncoder?.close(); - this.#audioEncoder = null; - this.#gops.length = 0; - this.#audio.length = 0; - this.#videoClock.clear(); - this.#audioClock.clear(); + this.#worker?.terminate(); + this.#worker = null; + for (const cut of this.#cuts.values()) cut.resolve(null); + this.#cuts.clear(); this.#audioSignalAt = null; } - async #pumpVideo(track: MediaStreamTrack): Promise { - const reader = new TrackProcessor!({ track }).readable.getReader(); - this.#readers.push(reader); - while (!this.#stopped) { - const { value, done } = await reader.read(); - if (done || !value) break; - const frame = value as VideoFrame; - const encoder = this.#videoEncoder; - if (encoder?.state !== "configured") { - frame.close(); - continue; - } - // back-pressure: a stalled encoder drops frames rather than queueing memory - if (encoder.encodeQueueSize > 4) { - frame.close(); - continue; - } - const now = Date.now() / 1000; - const keyFrame = now - this.#lastKeyframeAt >= KEYFRAME_INTERVAL_S; - if (keyFrame) { - this.#lastKeyframeAt = now; - void this.#thumbnail(frame).then((thumbnail) => { - const gop = this.#gops.at(-1); - if (gop && !gop.thumbnail) gop.thumbnail = thumbnail; - }); - } - this.#videoClock.note(frame.timestamp, now); - encoder.encode(frame, { keyFrame }); - frame.close(); - } - } - - async #startAudio(track: MediaStreamTrack): Promise { - const reader = new TrackProcessor!({ - track, - maxBufferSize: AUDIO_BUFFER_FRAMES, - }).readable.getReader(); - this.#readers.push(reader); - let configured = false; - while (!this.#stopped) { - const { value, done } = await reader.read(); - if (done || !value) break; - const data = value as AudioData; - const now = Date.now() / 1000; - if (!configured) { - configured = true; - const choice = await firstSupportedAudioCodec( - data.numberOfChannels, - data.sampleRate, - ); - if (!choice || this.#stopped) { - data.close(); - break; - } - this.#audioContainerCodec = choice.container; - this.#audioEncoder = new AudioEncoder({ - output: (chunk, meta) => this.#onAudioChunk(chunk, meta), - error: () => { - this.#audioEncoder = null; - }, - }); - this.#audioEncoder.configure({ - codec: choice.codec, - numberOfChannels: data.numberOfChannels, - sampleRate: data.sampleRate, - bitrate: AUDIO_BITRATE, - }); - this.#audioSignalAt = now; - } - const encoder = this.#audioEncoder; - if (encoder?.state === "configured" && encoder.encodeQueueSize < 32) { - const peak = peakOf(data); - if (peak === null || peak > SILENCE_PEAK) this.#audioSignalAt = now; - this.#audioClock.note(data.timestamp, now); - encoder.encode(data); - } - data.close(); - } - } - - #onVideoChunk( - chunk: EncodedVideoChunk, - meta: EncodedVideoChunkMetadata | undefined, - ): void { - if (meta?.decoderConfig) this.#videoConfig = meta.decoderConfig; - const stamped = { - packet: EncodedPacket.fromEncodedChunk(chunk), - wall: this.#videoClock.claim(chunk.timestamp), - }; - if (chunk.type === "key" || this.#gops.length === 0) { - this.#gops.push({ packets: [stamped] }); - this.#evict(stamped.wall); - } else { - this.#gops.at(-1)!.packets.push(stamped); - } - } - - #onAudioChunk( - chunk: EncodedAudioChunk, - meta: EncodedAudioChunkMetadata | undefined, - ): void { - if (meta?.decoderConfig) this.#audioConfig = meta.decoderConfig; - this.#audio.push({ - packet: EncodedPacket.fromEncodedChunk(chunk), - wall: this.#audioClock.claim(chunk.timestamp), - }); - } - - /** Drops whole GOPs (and audio) older than the window, always keeping the newest two GOPs. */ - #evict(now: number): void { - const horizon = now - this.#seconds; - while ( - this.#gops.length > 2 && - this.#gops[1]!.packets[0]!.wall <= horizon - ) { - this.#gops.shift(); - } - const audioHorizon = this.#gops[0]?.packets[0]?.wall ?? horizon; - while (this.#audio.length > 0 && this.#audio[0]!.wall < audioHorizon) { - this.#audio.shift(); - } - } - - async #thumbnail(frame: VideoFrame): Promise { - const bitmap = await createImageBitmap(frame, { - resizeWidth: THUMBNAIL_WIDTH, - resizeHeight: THUMBNAIL_HEIGHT, - }); - const canvas = document.createElement("canvas"); - canvas.width = THUMBNAIL_WIDTH; - canvas.height = THUMBNAIL_HEIGHT; - canvas.getContext("2d")!.drawImage(bitmap, 0, 0); - bitmap.close(); - return canvas.toDataURL("image/jpeg", 0.7); - } - - #fail(): void { - this.#videoEncoder = null; + #send(msg: RingWorkerRequest, transfer: Transferable[] = []): void { + this.#worker?.postMessage(msg, transfer); } } - -/** - * Wall-clock stamps of the frames handed to an encoder, claimed in order by - * the packets that come out: a packet's stamp is its frame's capture time, - * whatever the encoder's latency. Output timestamps trail the input ones - * only where the encoder saw a gap, so a claim also sweeps up everything - * older (frames the encoder dropped). - */ -class CaptureClock { - readonly #entries: { timestamp: number; wall: number }[] = []; - - /** `timestamp` in microseconds, as WebCodecs frames carry it */ - note(timestamp: number, wall: number): void { - this.#entries.push({ timestamp, wall }); - } - - /** Wall-clock seconds for the packet at `timestamp` (microseconds); now when nothing was noted for it. */ - claim(timestamp: number): number { - let wall = Date.now() / 1000; - let claimed = 0; - for (const entry of this.#entries) { - if (entry.timestamp > timestamp) break; - wall = entry.wall + (timestamp - entry.timestamp) / 1e6; - claimed++; - } - this.#entries.splice(0, claimed); - return wall; - } - - clear(): void { - this.#entries.length = 0; - } -} - -/** The loudest sample of the slice's first channel; null when it cannot be read. */ -function peakOf(data: AudioData): number | null { - try { - const samples = new Float32Array(data.numberOfFrames); - data.copyTo(samples, { planeIndex: 0, format: "f32-planar" }); - let peak = 0; - for (const sample of samples) peak = Math.max(peak, Math.abs(sample)); - return peak; - } catch { - return null; - } -} - -async function firstSupportedVideoCodec( - width: number, - height: number, - framerate: number, -): Promise { - for (const codec of VIDEO_CODECS) { - try { - const { supported } = await VideoEncoder.isConfigSupported({ - codec, - width, - height, - bitrate: VIDEO_BITRATE, - framerate, - latencyMode: "realtime", - }); - if (supported) return codec; - } catch { - // an unknown codec string throws rather than reporting unsupported - } - } - return null; -} - -async function firstSupportedAudioCodec( - numberOfChannels: number, - sampleRate: number, -): Promise<{ codec: string; container: AudioCodec } | null> { - for (const choice of AUDIO_CODECS) { - try { - const { supported } = await AudioEncoder.isConfigSupported({ - codec: choice.codec, - numberOfChannels, - sampleRate, - bitrate: AUDIO_BITRATE, - }); - if (supported) return choice; - } catch { - // same as the video probe - } - } - return null; -} diff --git a/app/features/scanner/capture/ring-buffer.worker.ts b/app/features/scanner/capture/ring-buffer.worker.ts new file mode 100644 index 000000000..6228ab4cb --- /dev/null +++ b/app/features/scanner/capture/ring-buffer.worker.ts @@ -0,0 +1,515 @@ +/** + * The clip ring buffer's engine, off the main thread so a busy page never + * costs it a frame: the capture's video track (a transferred + * MediaStreamTrackProcessor stream) runs through a VideoEncoder (hardware + * H.264 where available, a keyframe every `KEYFRAME_INTERVAL_S`) into a + * ring of GOPs holding the last `seconds`; the audio track through an + * AudioEncoder into the same ring. Every packet is stamped with the wall + * clock its frame was captured at — noted as the frame enters the encoder, + * claimed as the packet comes out, so encoder latency does not shift it; + * video by frame timestamp, audio by sample position, because audio + * timestamps cannot be trusted (a display capture's drift against its + * sample count, and the AAC encoder counts samples anyway) — the clock the + * sampler stamps detections with, so a cut asks for wall-clock seconds and + * gets the GOP at or before its start through the packets up to its end, + * muxed to MP4 with mediabunny — no decode, so a cut takes milliseconds. + */ +import { + type AudioCodec, + BufferTarget, + EncodedAudioPacketSource, + EncodedPacket, + EncodedVideoPacketSource, + Mp4OutputFormat, + Output, +} from "mediabunny"; +import type { + RingBufferClip, + RingWorkerRequest, + RingWorkerResponse, +} from "./ring-buffer-protocol"; + +/** ~indistinguishable from the source at 720p60 per the auto-clipper measurements */ +const VIDEO_BITRATE = 16_000_000; +const KEYFRAME_INTERVAL_S = 2; +const AUDIO_BITRATE = 160_000; +/** high profile at level 5.1 covers 1080p60; the fallbacks trade profile for reach */ +const VIDEO_CODECS = ["avc1.640033", "avc1.4d0033", "avc1.42e033"]; +/** AAC is the MP4-native choice; Chromium encodes it on most but not all platforms */ +const AUDIO_CODECS: { codec: string; container: AudioCodec }[] = [ + { codec: "mp4a.40.2", container: "aac" }, + { codec: "opus", container: "opus" }, +]; +const THUMBNAIL_WIDTH = 320; +const THUMBNAIL_HEIGHT = 180; +/** footage that begins this long after the asked start misses the action itself, not just the approach */ +const MAX_MISSING_LEAD_S = 5; +/** a slice whose loudest sample is under this (about -60 dBFS) carries no signal */ +const SILENCE_PEAK = 0.001; +/** how often at most the main thread hears that sound is coming in */ +const AUDIO_SIGNAL_REPORT_S = 1; +/** + * Audio is laid out by sample count from its first packet; a packet whose + * capture time is further ahead than this jumps forward (a delivery gap), + * anything less is jitter smoothed away. + */ +const AUDIO_RESYNC_S = 0.1; + +interface Stamped { + packet: EncodedPacket; + /** wall-clock seconds the frame/sample was captured at */ + wall: number; +} + +interface Gop { + packets: Stamped[]; + /** JPEG data URL of the keyframe, for the clip card */ + thumbnail?: string; +} + +let ring: Ring | null = null; + +self.onmessage = (e: MessageEvent) => { + const msg = e.data; + if (msg.kind === "start") { + ring = new Ring(msg.seconds); + void ring.start(msg).then( + () => post({ kind: "started" }), + (error) => post({ kind: "error", message: describe(error) }), + ); + } else if (msg.kind === "cut") { + void ( + ring?.cut(msg.start, msg.end, msg.audioOffset) ?? Promise.resolve(null) + ).then( + (clip) => post({ kind: "cut", id: msg.id, clip }), + (error) => + post({ kind: "cutError", id: msg.id, message: describe(error) }), + ); + } +}; + +function post(msg: RingWorkerResponse): void { + self.postMessage(msg); +} + +class Ring { + readonly #seconds: number; + readonly #gops: Gop[] = []; + readonly #audio: Stamped[] = []; + readonly #videoClock = new CaptureClock(); + readonly #audioClock = new SampleClock(); + #audioSampleRate = 48_000; + #videoConfig: VideoDecoderConfig | undefined; + #audioConfig: AudioDecoderConfig | undefined; + #audioContainerCodec: AudioCodec | null = null; + #videoEncoder: VideoEncoder | null = null; + #audioEncoder: AudioEncoder | null = null; + #lastKeyframeAt = Number.NEGATIVE_INFINITY; + #audioSignalReportedAt = Number.NEGATIVE_INFINITY; + + constructor(seconds: number) { + this.#seconds = seconds; + } + + /** Starts encoding both streams; resolves once the video encoder is configured. */ + async start({ + video, + audio, + width, + height, + framerate, + }: Extract): Promise { + const codec = await firstSupportedVideoCodec(width, height, framerate); + if (!codec) throw new Error("no H.264 encoder available for clips"); + + this.#videoEncoder = new VideoEncoder({ + output: (chunk, meta) => this.#onVideoChunk(chunk, meta), + error: (error) => this.#fail(error), + }); + this.#videoEncoder.configure({ + codec, + width, + height, + bitrate: VIDEO_BITRATE, + framerate, + latencyMode: "realtime", + hardwareAcceleration: "prefer-hardware", + }); + void this.#pumpVideo(video); + if (audio && typeof AudioEncoder !== "undefined") { + void this.#pumpAudio(audio); + } + } + + /** + * The footage between two wall-clock seconds as an MP4, from the keyframe + * at or before `start`, the sound moved `audioOffset` seconds later. Null + * when the ring holds nothing for the range. + */ + async cut( + start: number, + end: number, + audioOffset: number, + ): Promise { + const gopIndex = this.#gops.findLastIndex( + (gop) => gop.packets[0]!.wall <= start, + ); + const gops = this.#gops.slice(Math.max(0, gopIndex)); + const firstGopAt = gops[0]?.packets[0]?.wall; + if (firstGopAt === undefined || firstGopAt - start > MAX_MISSING_LEAD_S) { + return null; + } + const video = gops + .flatMap((gop) => gop.packets) + .filter((stamped) => stamped.wall <= end); + const first = video[0]; + if (!first || !this.#videoConfig) return null; + const clipStart = first.wall; + const clipEnd = video.at(-1)!.wall; + const thumbnail = + gops.findLast((gop) => gop.thumbnail && gop.packets[0]!.wall <= end) + ?.thumbnail ?? gops[0]?.thumbnail; + + const format = new Mp4OutputFormat({ fastStart: "in-memory" }); + const target = new BufferTarget(); + const output = new Output({ format, target }); + const videoSource = new EncodedVideoPacketSource("avc"); + output.addVideoTrack(videoSource); + const audio = this.#audioContainerCodec + ? this.#audio + .map(({ packet, wall }) => ({ packet, wall: wall + audioOffset })) + .filter( + (stamped) => stamped.wall >= clipStart && stamped.wall <= clipEnd, + ) + : []; + const audioSource = + audio.length > 0 && this.#audioContainerCodec + ? new EncodedAudioPacketSource(this.#audioContainerCodec) + : null; + if (audioSource) output.addAudioTrack(audioSource); + await output.start(); + try { + const base = first.packet.timestamp; + let meta: { decoderConfig: VideoDecoderConfig } | undefined = { + decoderConfig: this.#videoConfig, + }; + for (const { packet } of video) { + await videoSource.add( + packet.clone({ timestamp: packet.timestamp - base }), + meta, + ); + meta = undefined; + } + videoSource.close(); + if (audioSource) { + // each packet sits where its sound was captured against the clip's + // first frame, packets running on from one another unless a gap says + // otherwise + let audioMeta: { decoderConfig?: AudioDecoderConfig } | undefined = { + decoderConfig: this.#audioConfig, + }; + let next: number | null = null; + for (const { packet, wall } of audio) { + const captured = wall - clipStart; + const timestamp: number = + next !== null && captured < next + AUDIO_RESYNC_S ? next : captured; + next = timestamp + packet.duration; + if (timestamp < 0) continue; + await audioSource.add(packet.clone({ timestamp }), audioMeta); + audioMeta = undefined; + } + audioSource.close(); + } + await output.finalize(); + } catch (error) { + await output.cancel(); + throw error; + } + return { + blob: new Blob([target.buffer!], { type: format.mimeType }), + start: clipStart, + end: clipEnd, + hasAudio: audioSource !== null, + thumbnail, + }; + } + + async #pumpVideo(stream: ReadableStream): Promise { + const reader = stream.getReader(); + for (;;) { + const { value: frame, done } = await reader.read(); + if (done || !frame) break; + const encoder = this.#videoEncoder; + if (encoder?.state !== "configured") { + frame.close(); + continue; + } + // back-pressure: a stalled encoder drops frames rather than queueing memory + if (encoder.encodeQueueSize > 4) { + frame.close(); + continue; + } + const now = Date.now() / 1000; + const keyFrame = now - this.#lastKeyframeAt >= KEYFRAME_INTERVAL_S; + if (keyFrame) { + this.#lastKeyframeAt = now; + void thumbnailOf(frame.clone()).then((thumbnail) => { + const gop = this.#gops.at(-1); + if (gop && !gop.thumbnail) gop.thumbnail = thumbnail; + }); + } + this.#videoClock.note(frame.timestamp, now); + encoder.encode(frame, { keyFrame }); + frame.close(); + } + } + + async #pumpAudio(stream: ReadableStream): Promise { + const reader = stream.getReader(); + let configured = false; + for (;;) { + const { value: data, done } = await reader.read(); + if (done || !data) break; + const now = Date.now() / 1000; + if (!configured) { + configured = true; + const choice = await firstSupportedAudioCodec( + data.numberOfChannels, + data.sampleRate, + ); + if (!choice) { + data.close(); + break; + } + this.#audioContainerCodec = choice.container; + this.#audioSampleRate = data.sampleRate; + this.#audioEncoder = new AudioEncoder({ + output: (chunk, meta) => this.#onAudioChunk(chunk, meta), + error: (error) => { + this.#audioEncoder = null; + post({ kind: "audioError", message: describe(error) }); + }, + }); + this.#audioEncoder.configure({ + codec: choice.codec, + numberOfChannels: data.numberOfChannels, + sampleRate: data.sampleRate, + bitrate: AUDIO_BITRATE, + }); + this.#reportAudioSignal(now); + } + const encoder = this.#audioEncoder; + if (encoder?.state === "configured" && encoder.encodeQueueSize < 32) { + const peak = peakOf(data); + if (peak === null || peak > SILENCE_PEAK) this.#reportAudioSignal(now); + this.#audioClock.note(data.numberOfFrames, now); + encoder.encode(data); + } + data.close(); + } + } + + #reportAudioSignal(now: number): void { + if (now - this.#audioSignalReportedAt < AUDIO_SIGNAL_REPORT_S) return; + this.#audioSignalReportedAt = now; + post({ kind: "audioSignal", at: now }); + } + + #onVideoChunk( + chunk: EncodedVideoChunk, + meta: EncodedVideoChunkMetadata | undefined, + ): void { + if (meta?.decoderConfig) this.#videoConfig = meta.decoderConfig; + const stamped = { + packet: EncodedPacket.fromEncodedChunk(chunk), + wall: this.#videoClock.claim(chunk.timestamp), + }; + if (chunk.type === "key" || this.#gops.length === 0) { + this.#gops.push({ packets: [stamped] }); + this.#evict(stamped.wall); + } else { + this.#gops.at(-1)!.packets.push(stamped); + } + } + + #onAudioChunk( + chunk: EncodedAudioChunk, + meta: EncodedAudioChunkMetadata | undefined, + ): void { + if (meta?.decoderConfig) this.#audioConfig = meta.decoderConfig; + const packet = EncodedPacket.fromEncodedChunk(chunk); + this.#audio.push({ + packet, + wall: this.#audioClock.claim( + Math.round(packet.duration * this.#audioSampleRate), + this.#audioSampleRate, + ), + }); + } + + /** Drops whole GOPs (and audio) older than the window, always keeping the newest two GOPs. */ + #evict(now: number): void { + const horizon = now - this.#seconds; + while ( + this.#gops.length > 2 && + this.#gops[1]!.packets[0]!.wall <= horizon + ) { + this.#gops.shift(); + } + const audioHorizon = this.#gops[0]?.packets[0]?.wall ?? horizon; + while (this.#audio.length > 0 && this.#audio[0]!.wall < audioHorizon) { + this.#audio.shift(); + } + } + + #fail(error: unknown): void { + this.#videoEncoder = null; + post({ kind: "error", message: describe(error) }); + } +} + +/** + * Wall-clock stamps of the frames handed to an encoder, claimed in order by + * the packets that come out: a packet's stamp is its frame's capture time, + * whatever the encoder's latency. Output timestamps trail the input ones + * only where the encoder saw a gap, so a claim also sweeps up everything + * older (frames the encoder dropped). + */ +class CaptureClock { + readonly #entries: { timestamp: number; wall: number }[] = []; + + /** `timestamp` in microseconds, as WebCodecs frames carry it */ + note(timestamp: number, wall: number): void { + this.#entries.push({ timestamp, wall }); + } + + /** Wall-clock seconds for the packet at `timestamp` (microseconds); now when nothing was noted for it. */ + claim(timestamp: number): number { + let wall = Date.now() / 1000; + let claimed = 0; + for (const entry of this.#entries) { + if (entry.timestamp > timestamp) break; + wall = entry.wall + (timestamp - entry.timestamp) / 1e6; + claimed++; + } + this.#entries.splice(0, claimed); + return wall; + } +} + +/** + * Wall-clock stamps of the audio slices handed to the encoder, by sample + * position: an output packet is stamped from the slice its first sample + * came from, whatever any timestamp says. Slices the encoder has passed + * are dropped as packets claim beyond them. + */ +class SampleClock { + readonly #slices: { endSample: number; wall: number }[] = []; + #samplesIn = 0; + #samplesOut = 0; + + note(frames: number, wall: number): void { + this.#samplesIn += frames; + this.#slices.push({ endSample: this.#samplesIn, wall }); + } + + /** Wall-clock seconds the packet of `frames` samples begins at; now when nothing was noted. */ + claim(frames: number, sampleRate: number): number { + const start = this.#samplesOut; + this.#samplesOut += frames; + while (this.#slices.length > 1 && this.#slices[0]!.endSample <= start) { + this.#slices.shift(); + } + const slice = this.#slices[0]; + if (!slice) return Date.now() / 1000; + return slice.wall - Math.max(0, slice.endSample - start) / sampleRate; + } +} + +/** The loudest sample of the slice's first channel; null when it cannot be read. */ +function peakOf(data: AudioData): number | null { + try { + const samples = new Float32Array(data.numberOfFrames); + data.copyTo(samples, { planeIndex: 0, format: "f32-planar" }); + let peak = 0; + for (const sample of samples) peak = Math.max(peak, Math.abs(sample)); + return peak; + } catch { + return null; + } +} + +/** A JPEG data URL of the frame, which is closed here. */ +async function thumbnailOf(frame: VideoFrame): Promise { + try { + const bitmap = await createImageBitmap(frame, { + resizeWidth: THUMBNAIL_WIDTH, + resizeHeight: THUMBNAIL_HEIGHT, + }); + const canvas = new OffscreenCanvas(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT); + canvas.getContext("2d")!.drawImage(bitmap, 0, 0); + bitmap.close(); + const blob = await canvas.convertToBlob({ + type: "image/jpeg", + quality: 0.7, + }); + return `data:image/jpeg;base64,${base64Of(new Uint8Array(await blob.arrayBuffer()))}`; + } finally { + frame.close(); + } +} + +function base64Of(bytes: Uint8Array): string { + let binary = ""; + for (let i = 0; i < bytes.length; i += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); + } + return btoa(binary); +} + +async function firstSupportedVideoCodec( + width: number, + height: number, + framerate: number, +): Promise { + for (const codec of VIDEO_CODECS) { + try { + const { supported } = await VideoEncoder.isConfigSupported({ + codec, + width, + height, + bitrate: VIDEO_BITRATE, + framerate, + latencyMode: "realtime", + }); + if (supported) return codec; + } catch { + // an unknown codec string throws rather than reporting unsupported + } + } + return null; +} + +async function firstSupportedAudioCodec( + numberOfChannels: number, + sampleRate: number, +): Promise<{ codec: string; container: AudioCodec } | null> { + for (const choice of AUDIO_CODECS) { + try { + const { supported } = await AudioEncoder.isConfigSupported({ + codec: choice.codec, + numberOfChannels, + sampleRate, + bitrate: AUDIO_BITRATE, + }); + if (supported) return choice; + } catch { + // same as the video probe + } + } + return null; +} + +function describe(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/app/features/scanner/capture/sampler.ts b/app/features/scanner/capture/sampler.ts index 43caeb404..c9cdb933f 100644 --- a/app/features/scanner/capture/sampler.ts +++ b/app/features/scanner/capture/sampler.ts @@ -100,6 +100,8 @@ export async function openCapture({ deviceId: videoDeviceId ? { exact: videoDeviceId } : undefined, width: { ideal: 1920 }, height: { ideal: 1080 }, + // Chromium's own default is 30, which would halve a capture card's 60 + frameRate: { ideal: 60 }, }; let audioError: string | null = null; if (audioDeviceId) { @@ -125,6 +127,59 @@ export async function openCapture({ return { stream, audioError }; } +interface DesktopAudioOptions extends DisplayMediaStreamOptions { + /** Chromium's picker hints, not in TypeScript's DOM lib */ + systemAudio?: "include" | "exclude"; + selfBrowserSurface?: "include" | "exclude"; +} + +/** + * The desktop's sound as one audio track, through the browser's share + * picker: Chromium offers "Also share system audio" for a whole screen and + * a tab's audio for a tab. The picked video surface is dropped at once. + * Must run inside the click that starts the capture (the picker needs the + * activation). Null with the reason when nothing usable was shared. + */ +export async function openDesktopAudio(): Promise<{ + track: MediaStreamTrack | null; + error: string | null; +}> { + if (!navigator.mediaDevices?.getDisplayMedia) { + return { track: null, error: "this browser cannot share desktop audio" }; + } + try { + const options: DesktopAudioOptions = { + video: true, + audio: { + echoCancellation: false, + noiseSuppression: false, + autoGainControl: false, + // Chromium's own: keep the shared sound playing on the desktop too + suppressLocalAudioPlayback: false, + } as MediaTrackConstraints, + systemAudio: "include", + selfBrowserSurface: "exclude", + }; + const shared = await navigator.mediaDevices.getDisplayMedia(options); + for (const track of shared.getVideoTracks()) track.stop(); + const track = shared.getAudioTracks()[0] ?? null; + return { + track, + error: track + ? null + : 'nothing was shared with sound, pick a screen and tick "Also share system audio"', + }; + } catch (error) { + return { + track: null, + error: + error instanceof DOMException && error.name === "NotAllowedError" + ? "desktop audio sharing was cancelled" + : audioErrorText(error), + }; + } +} + function audioErrorText(error: unknown): string { if (error instanceof DOMException) { switch (error.name) { diff --git a/app/features/scanner/components/LandingView.module.css b/app/features/scanner/components/LandingView.module.css index dd2e67954..d1ce14eb5 100644 --- a/app/features/scanner/components/LandingView.module.css +++ b/app/features/scanner/components/LandingView.module.css @@ -134,6 +134,8 @@ align-items: center; gap: var(--s-3); padding: var(--s-2-5) var(--s-4); + /* the global link radius would curve the divider's ends */ + border-radius: 0; color: inherit; text-decoration: none; font-size: var(--font-xs); diff --git a/app/features/scanner/components/LiveView.tsx b/app/features/scanner/components/LiveView.tsx index b57e0f4ab..592a67d25 100644 --- a/app/features/scanner/components/LiveView.tsx +++ b/app/features/scanner/components/LiveView.tsx @@ -43,7 +43,12 @@ export function LiveView() { const session = currentSession(feed); const events = session?.events ?? []; - const sessionClips = clips.filter((clip) => clip.bucket === "session"); + const sessionClips = clips.filter( + (clip) => + clip.bucket === "session" && + clip.source.kind === "live" && + clip.source.sessionKey === session?.key, + ); const newest = session?.built.at(-1); const reading = newest !== undefined && @@ -57,12 +62,18 @@ export function LiveView() { const clipsNote = live.clips === "on" ? live.hasAudio - ? live.audioSignal === "muted" - ? "Clips on · audio input muted by the browser" - : live.audioSignal === "silent" - ? "Clips on · Audio ✓ but only silence is coming in" - : "Clips on · Audio ✓" - : `Clips on · no audio${live.audioError ? ` (${live.audioError})` : ""}` + ? live.audioSignal === "failed" + ? "Clips on · audio encoder failed, clips are silent" + : live.audioSignal === "muted" + ? "Clips on · audio input muted by the browser" + : live.audioSignal === "ended" + ? "Clips on · audio input stopped" + : live.audioSignal === "silent" + ? "Clips on · Audio ✓ but only silence is coming in" + : "Clips on · Audio ✓" + : settings.audioSource === "off" + ? "Clips on · audio off" + : `Clips on · no audio${live.audioError ? ` (${live.audioError})` : ""}` : live.clips === "unsupported" ? "Clips need a Chromium browser" : live.clips === "failed" diff --git a/app/features/scanner/components/ScannerApp.tsx b/app/features/scanner/components/ScannerApp.tsx index c9b989330..1f7bba77f 100644 --- a/app/features/scanner/components/ScannerApp.tsx +++ b/app/features/scanner/components/ScannerApp.tsx @@ -9,7 +9,7 @@ import { useEffect } from "react"; import { useUser } from "~/features/auth/core/user"; import { useSearchParam } from "~/modules/search-params/hooks"; import { scannerSearchParams } from "../scanner-search-params"; -import { deleteVodClips } from "../store/clips"; +import { deleteVodClips, rollSessionClipsIntoHistory } from "../store/clips"; import { ClipsView } from "./ClipsView"; import { refreshClips } from "./clips-feed"; import { FixturesPage } from "./FixturesPage"; @@ -24,8 +24,12 @@ import { useDebug } from "./use-debug"; import { VodView } from "./VodView"; import { cancelVodScan } from "./vod-scan"; -/** A file's clips live for one visit: the file is on disk, so a new page load starts without them. */ -let vodClipsPurged = false; +/** + * Once per page load: a file's clips live for one visit (the file is on + * disk), and session clips left by a capture that never reached Stop (a + * reload, a closed tab) belong to the history now, not the next capture. + */ +let storeSettled = false; export function ScannerApp() { const [view] = useSearchParam(scannerSearchParams, "view"); @@ -39,11 +43,12 @@ export function ScannerApp() { }, [user]); useEffect(() => { - if (vodClipsPurged) return; - vodClipsPurged = true; - void deleteVodClips() - .catch(() => {}) - .then(() => refreshClips()); + if (storeSettled) return; + storeSettled = true; + void Promise.allSettled([ + deleteVodClips(), + rollSessionClipsIntoHistory(), + ]).then(() => refreshClips()); }, []); // a file scan has no Cancel button: leaving the page is how it is stopped diff --git a/app/features/scanner/components/SessionView.tsx b/app/features/scanner/components/SessionView.tsx index 3eef1f838..310ee2545 100644 --- a/app/features/scanner/components/SessionView.tsx +++ b/app/features/scanner/components/SessionView.tsx @@ -16,6 +16,7 @@ import { SendouTabPanel, SendouTabs, } from "~/components/elements/Tabs"; +import { MAP_START_EVENT_TYPE } from "../core/detectors/map-start"; import type { IngestSkipReason } from "../core/match-builder"; import { type BuiltMatch, @@ -110,6 +111,13 @@ export function SessionView({ skipReasons, }; const justFormedKeys = useJustFormedKeys(built.map(keyOf)); + // the game being played is the only one still gathering events; a newer + // map intro means it is over even before that game has a card of its own + const lastBuiltT = + built.at(-1)?.sources.at(-1)?.t ?? Number.NEGATIVE_INFINITY; + const newerGameStarted = events.some( + (event) => event.type === MAP_START_EVENT_TYPE && event.t > lastBuiltT, + ); const groups = LOBBY_GROUPS.map((group) => ({ group, matches: built.filter((b) => lobbyGroup(b.match.lobby) === group), @@ -127,7 +135,12 @@ export function SessionView({ kind={kind} justFormed={justFormedKeys.has(key)} expandable={ - !(running && index === built.length - 1 && b.match.winner === null) + !( + running && + index === built.length - 1 && + b.match.winner === null && + !newerGameStarted + ) } upload={uploadStateOf({ send: aggregateSendStatus(b.sources), diff --git a/app/features/scanner/components/SettingsPopover.module.css b/app/features/scanner/components/SettingsPopover.module.css index 6d59d0c6f..90c7fef08 100644 --- a/app/features/scanner/components/SettingsPopover.module.css +++ b/app/features/scanner/components/SettingsPopover.module.css @@ -43,3 +43,27 @@ font-weight: var(--weight-semi); color: var(--color-text-high); } + +.number { + width: 6rem; + height: var(--field-size-sm); + padding: 0 var(--s-2); + border: var(--border-style); + border-radius: var(--radius-field); + background-color: var(--color-bg); + color: var(--color-text); + font: inherit; + font-size: var(--font-xs); + font-variant-numeric: tabular-nums; + + &:focus-visible { + outline: var(--focus-ring); + outline-offset: 1px; + } +} + +.hint { + flex-basis: 100%; + font-size: var(--font-3xs); + color: var(--color-text-high); +} diff --git a/app/features/scanner/components/SettingsPopover.tsx b/app/features/scanner/components/SettingsPopover.tsx index beb601e0a..2b6e5f28a 100644 --- a/app/features/scanner/components/SettingsPopover.tsx +++ b/app/features/scanner/components/SettingsPopover.tsx @@ -19,6 +19,7 @@ import { scannerSearchParams } from "../scanner-search-params"; import { MAX_HISTORY_CLIPS } from "../store/clips"; import styles from "./SettingsPopover.module.css"; import { + AUDIO_OFFSET_LIMIT_MS, CLIP_MIN_KILLS_OPTIONS, updateSettings, useScannerSettings, @@ -26,6 +27,9 @@ import { import { isLoggedIn } from "./upload"; import { useDebug } from "./use-debug"; +/** a step of one frame-ish: fine enough to tune by ear, coarse enough to reach a second in a few clicks */ +const AUDIO_OFFSET_STEP_MS = 25; + export function SettingsPopover() { const settings = useScannerSettings(); const debug = useDebug(); @@ -81,6 +85,35 @@ export function SettingsPopover() { ))} +
+ + { + const value = e.target.valueAsNumber; + if (Number.isFinite(value)) { + updateSettings({ + audioOffsetMs: Math.max( + -AUDIO_OFFSET_LIMIT_MS, + Math.min(AUDIO_OFFSET_LIMIT_MS, value), + ), + }); + } + }} + /> + + Sound ahead of the picture? Raise it. Behind? Lower it. Desktop + audio and a capture card usually need a few hundred ms. + +

Clip history keeps the {MAX_HISTORY_CLIPS} best; the lowest is diff --git a/app/features/scanner/components/SourceSelect.tsx b/app/features/scanner/components/SourceSelect.tsx index ec5b40d99..9b365a025 100644 --- a/app/features/scanner/components/SourceSelect.tsx +++ b/app/features/scanner/components/SourceSelect.tsx @@ -1,10 +1,11 @@ /** * The capture source: one select listing every video input, the capture - * card first and OBS Virtual Camera as just another entry. Browsers reveal - * ids and labels only once camera permission is granted, so the list is - * re-read whenever a capture starts or stops, and opening the select while - * it is still anonymous asks for the permission right there. The choice is - * remembered in localStorage. + * card first and OBS Virtual Camera as just another entry, and under it what + * clips hear: the source's own audio, the desktop's sound, any audio input + * or nothing. Browsers reveal ids and labels only once camera permission is + * granted, so the list is re-read whenever a capture starts or stops, and + * opening a select while it is still anonymous asks for the permission + * right there. Both choices are remembered in localStorage. */ import { useEffect, useState } from "react"; import { @@ -17,7 +18,11 @@ import { } from "../capture/sampler"; import { useLiveSession } from "./live-session"; import styles from "./SourceSelect.module.css"; -import { updateSettings, useScannerSettings } from "./settings"; +import { + type AudioSource, + updateSettings, + useScannerSettings, +} from "./settings"; const NO_INPUTS: MediaInputs = { video: [], audio: [] }; @@ -57,10 +62,11 @@ export function SourceSelect({ disabled }: { disabled?: boolean }) { // before the first granted permission Chromium lists devices with empty // ids and labels, which would collide with the default entry const videoInputs = inputs.video.filter((device) => device.deviceId !== ""); + const audioInputs = inputs.audio.filter((device) => device.deviceId !== ""); const selected = settings.sourceDeviceId ? videoInputs.find((device) => device.deviceId === settings.sourceDeviceId) : undefined; - const audio = selected ? audioInputFor(selected, inputs.audio) : null; + const sourceAudio = selected ? audioInputFor(selected, inputs.audio) : null; return (

@@ -80,17 +86,62 @@ export function SourceSelect({ disabled }: { disabled?: boolean }) { ))} - {selected ? ( - - {audio - ? `audio from ${audio.label || "the source"}` - : "audio: none, clips will be silent"} - - ) : null} + + + {audioNote(settings.audioSource, selected, sourceAudio)} +
); } +function audioNote( + source: AudioSource, + video: MediaDeviceInfo | undefined, + sourceAudio: MediaDeviceInfo | null, +): string { + switch (source) { + case "source": + if (!video) return "audio: pick a source to use its own audio"; + return sourceAudio + ? `audio from ${sourceAudio.label || "the source"}` + : "audio: none for this source, clips will be silent"; + case "desktop": + return 'a share picker opens with the capture: pick a screen and tick "Also share system audio"'; + case "off": + return "clips will be silent"; + default: + return "clips hear this input"; + } +} + +function supportsDesktopAudio(): boolean { + return ( + typeof navigator !== "undefined" && + Boolean(navigator.mediaDevices?.getDisplayMedia) + ); +} + function deviceLabel(device: MediaDeviceInfo): string { if (isVirtualCamera(device)) return "OBS Virtual Camera (no audio)"; return device.label || `Camera ${device.deviceId.slice(0, 6)}`; diff --git a/app/features/scanner/components/UploadChip.tsx b/app/features/scanner/components/UploadChip.tsx index e0570e5e1..8c9d5d9f6 100644 --- a/app/features/scanner/components/UploadChip.tsx +++ b/app/features/scanner/components/UploadChip.tsx @@ -22,7 +22,6 @@ import styles from "./UploadChip.module.css"; export type UploadState = | { kind: "uploaded"; link?: IngestedMatchLink } | { kind: "uploading" } - | { kind: "queued" } | { kind: "waiting"; onRetry?: () => void } | { kind: "failed"; error?: string; onRetry?: () => void } | { kind: "not-uploaded"; onUpload?: () => void } @@ -50,8 +49,6 @@ export function uploadStateOf({ return { kind: "uploaded", link: send.link }; case "sending": return { kind: "uploading" }; - case "queued": - return { kind: "queued" }; case "unlinked": return { kind: "waiting", onRetry: action }; case "failed": @@ -88,13 +85,6 @@ export function UploadChip({ state }: { state: UploadState }) { uploading… ); - case "queued": - return ( - - - queued - - ); case "waiting": return ( diff --git a/app/features/scanner/components/clips-feed.ts b/app/features/scanner/components/clips-feed.ts index 08898d0e7..f8cd37717 100644 --- a/app/features/scanner/components/clips-feed.ts +++ b/app/features/scanner/components/clips-feed.ts @@ -33,7 +33,8 @@ export function useClips(): ScannerClip[] { return useSyncExternalStore(subscribe, getClips, () => EMPTY); } -function getClips(): ScannerClip[] { +/** The list as last loaded, for the controllers; loads on first use. */ +export function getClips(): ScannerClip[] { if (!loaded && !refreshing) void refreshClips(); return clips; } diff --git a/app/features/scanner/components/live-session.ts b/app/features/scanner/components/live-session.ts index bd2cc5e32..f2c4ca22f 100644 --- a/app/features/scanner/components/live-session.ts +++ b/app/features/scanner/components/live-session.ts @@ -14,6 +14,7 @@ import { inputsRevealed, listMediaInputs, openCapture, + openDesktopAudio, requestInputAccess, startSampler, } from "../capture/sampler"; @@ -25,12 +26,10 @@ import { windowClosed, } from "../core/clips/scoring"; import { DEATH_EVENT_TYPE } from "../core/detectors/death/index"; -import { KILL_EVENT_TYPE } from "../core/detectors/kill/index"; import { MAP_START_EVENT_TYPE, type MapStartData, } from "../core/detectors/map-start/index"; -import { MINIMAP_EVENT_TYPE } from "../core/detectors/minimap/index"; import { OBJECTIVE_EVENT_TYPE } from "../core/detectors/objective/index"; import { PLAYER_STATUS_EVENT_TYPE } from "../core/detectors/objective/player-status"; import { SCOREBOARD_EVENT_TYPES } from "../core/detectors/registry"; @@ -38,13 +37,14 @@ import type { DetectedEvent, GateResult } from "../core/detectors/types"; import type { ScannerMatch } from "../core/scanner-match"; import { TimelineBuilder } from "../core/timeline/index"; import { + deleteClip, requestPersistentStorage, rollSessionClipsIntoHistory, saveClip, } from "../store/clips"; -import { saveEvent, trimEvents, updateEventsSend } from "../store/events"; +import { saveEvent, trimEvents } from "../store/events"; import { AnalyzerClient } from "../worker/client"; -import { refreshClips } from "./clips-feed"; +import { getClips, refreshClips } from "./clips-feed"; import { describeError } from "./errors"; import { currentSession, @@ -59,7 +59,7 @@ import { unsentClosedMatches, unsentMatches, } from "./sendou-ingest"; -import { readSettings } from "./settings"; +import { audioDeviceIdOf, readSettings } from "./settings"; import { thumbnailFromBlob } from "./thumbnail"; import { sendLive, uploadEnabled } from "./upload"; @@ -85,21 +85,14 @@ const CLIP_TICK_MS = 5_000; /** how often the audio input is checked for a signal, and how long without one counts as silent */ const AUDIO_CHECK_MS = 1_000; const AUDIO_SILENCE_MS = 5_000; - -/** Event types the ingested matches are built from — the only ones with a send status. */ -const INGESTABLE_TYPES = [ - MAP_START_EVENT_TYPE, - DEATH_EVENT_TYPE, - KILL_EVENT_TYPE, - MINIMAP_EVENT_TYPE, - ...SCOREBOARD_EVENT_TYPES, -]; +/** one capture per browser profile: two would write the same games twice */ +const CAPTURE_LOCK = "scanner:capture"; export type LiveStatus = "idle" | "starting" | "running" | "error"; /** `unsupported`: no WebCodecs/track processor; `failed`: the encoder refused this stream */ export type ClipsState = "on" | "off" | "unsupported" | "failed"; -/** `muted`: the browser gets nothing from the device; `silent`: it gets samples, all of them silence */ -export type AudioSignal = "ok" | "silent" | "muted"; +/** `muted`: the browser gets nothing from the device; `silent`: it gets samples, all of them silence; `ended`: the track stopped (a share ended, say); `failed`: the encoder gave up */ +export type AudioSignal = "ok" | "silent" | "muted" | "ended" | "failed"; export interface LiveSnapshot { status: LiveStatus; @@ -143,6 +136,7 @@ let stopSampler: (() => void) | null = null; let retryTimer: ReturnType | null = null; let clipTimer: ReturnType | null = null; let audioTimer: ReturnType | null = null; +let releaseCaptureLock: (() => void) | null = null; let unsubscribeFeed: (() => void) | null = null; let timeline = new TimelineBuilder(); const storedIds = new WeakMap(); @@ -152,7 +146,24 @@ let latestParse: { type: string; data: FixtureData } | null = null; // misreads of another mode's overlay and are not collected at all let objectiveBlocked = false; /** windows already cut, `${match first source id}:${window t}` */ -const cutWindows = new Set(); +/** + * Footage cut this capture, plus (at tick time) the session's saved clips + * from earlier captures — the session outlives a capture, so its windows + * come up again after a restart. A kill belongs to one clip: a window over + * footage already cut (redrawn by a late-read kill, or simply seen again) + * is skipped unless it scores higher, when it replaces the clips it overlaps. + */ +const cuts: LiveCut[] = []; +/** clips this capture saved, already represented in `cuts` */ +const ownClipIds = new Set(); + +interface LiveCut { + start: number; + end: number; + score: number; + /** the saved clip's id, once the cut lands; null when it yielded nothing */ + clipId: Promise; +} export function useLiveSession(): LiveSnapshot { return useSyncExternalStore( @@ -179,14 +190,28 @@ function set(patch: Partial): void { /** Opens the source, brings the worker up, then starts sampling; a second call while running is ignored. */ export async function startCapture(): Promise { if (snapshot.status === "starting" || snapshot.status === "running") return; + releaseCaptureLock = await acquireCaptureLock(); + if (!releaseCaptureLock) { + set({ + ...IDLE, + status: "error", + error: + "The scanner is already capturing in another tab. Stop it there first.", + }); + return; + } set({ ...IDLE, status: "starting" }); objectiveBlocked = false; gates.clear(); - cutWindows.clear(); + cuts.length = 0; + ownClipIds.clear(); timeline = new TimelineBuilder(); let stream: MediaStream | null = null; try { const settings = readSettings(); + // first thing, while the click's activation still covers the share picker + const desktop = + settings.audioSource === "desktop" ? await openDesktopAudio() : null; const noInputs = { video: [], audio: [] }; let inputs = await listMediaInputs().catch(() => noInputs); // the source's audio side is only listed with the microphone permission @@ -195,14 +220,17 @@ export async function startCapture(): Promise { } const videoInput = inputs.video.find((d) => d.deviceId === settings.sourceDeviceId) ?? null; - const audioInput = videoInput - ? audioInputFor(videoInput, inputs.audio) - : null; + const sourceAudio = + settings.audioSource === "source" && videoInput + ? audioInputFor(videoInput, inputs.audio) + : null; const opened = await openCapture({ videoDeviceId: settings.sourceDeviceId, - audioDeviceId: audioInput?.deviceId ?? null, + audioDeviceId: + sourceAudio?.deviceId ?? audioDeviceIdOf(settings.audioSource), }); stream = opened.stream; + if (desktop?.track) stream.addTrack(desktop.track); video = document.createElement("video"); video.muted = true; video.playsInline = true; @@ -261,7 +289,7 @@ export async function startCapture(): Promise { since: Date.now(), stream, hasAudio: stream.getAudioTracks().length > 0, - audioError: opened.audioError, + audioError: desktop ? desktop.error : opened.audioError, clips, }); } catch (error) { @@ -294,6 +322,8 @@ export function saveCurrentFrameAsFixture(): void { } function release(): void { + releaseCaptureLock?.(); + releaseCaptureLock = null; stopSampler?.(); stopSampler = null; if (retryTimer) clearInterval(retryTimer); @@ -312,6 +342,29 @@ function release(): void { video = null; } +/** + * Holds the capture lock until the returned function is called; null when + * another tab (or a page instance left running) holds it. The browser lets + * go of a tab's locks when it closes or crashes, so a dead capture never + * blocks the next one. + */ +function acquireCaptureLock(): Promise<(() => void) | null> { + if (!navigator.locks) return Promise.resolve(() => {}); + return new Promise((resolve) => { + void navigator.locks.request( + CAPTURE_LOCK, + { ifAvailable: true }, + (lock) => { + if (!lock) { + resolve(null); + return; + } + return new Promise((unlock) => resolve(unlock)); + }, + ); + }); +} + function stopTracks(stream: MediaStream): void { for (const track of stream.getTracks()) track.stop(); } @@ -369,16 +422,12 @@ async function persist( // repeat detections don't remount the cards const id = await saveEvent(event, thumbnail, frame, stale); storedIds.set(event, id); - if (uploadEnabled() && INGESTABLE_TYPES.includes(event.type)) { - if (SCOREBOARD_EVENT_TYPES.includes(event.type)) { - // a scoreboard closes its match — send it - refreshFeed(); - await sendLive( - (built) => matchContaining(id)(built) && unsentMatches(built), - ); - } else { - await updateEventsSend([id], { state: "queued", at: Date.now() }); - } + if (uploadEnabled() && SCOREBOARD_EVENT_TYPES.includes(event.type)) { + // a scoreboard closes its match — send it + refreshFeed(); + await sendLive( + (built) => matchContaining(id)(built) && unsentMatches(built), + ); } } catch (error) { set({ error: describeError(error) }); @@ -393,13 +442,17 @@ function audioCheck(): void { const signalAt = ring?.audioSignalAt ?? null; const next: AudioSignal | null = !track ? null - : track.muted || track.readyState === "ended" - ? "muted" - : signalAt === null - ? null - : Date.now() / 1000 - signalAt > AUDIO_SILENCE_MS / 1000 - ? "silent" - : "ok"; + : ring?.audioFailure + ? "failed" + : track.readyState === "ended" + ? "ended" + : track.muted + ? "muted" + : signalAt === null + ? null + : Date.now() / 1000 - signalAt > AUDIO_SILENCE_MS / 1000 + ? "silent" + : "ok"; if (next !== snapshot.audioSignal) set({ audioSignal: next }); } @@ -409,6 +462,19 @@ function clipTick(): void { const session = currentSession(getFeed()); if (!session) return; const nowT = Date.now() / 1000; + const saved: LiveCut[] = getClips() + .filter( + (clip) => + clip.source.kind === "live" && + clip.source.sessionKey === session.key && + !ownClipIds.has(clip.id), + ) + .map((clip) => ({ + start: clip.start, + end: clip.end, + score: clip.score, + clipId: Promise.resolve(clip.id), + })); for (const built of session.built) { const deaths = built.sources .filter((event) => event.type === DEATH_EVENT_TYPE) @@ -416,24 +482,42 @@ function clipTick(): void { for (const window of scoreWindows(built.match, deaths, { minKills: readSettings().clipMinKills, })) { - const key = `${built.sources[0]?.id ?? built.match.startsAt}:${window.t}`; - if (cutWindows.has(key) || !windowClosed(window, nowT)) continue; - cutWindows.add(key); - void cutClip(session.key, built.match, window); + if (!windowClosed(window, nowT)) continue; + const overlapping = [...cuts, ...saved].filter( + (cut) => window.start < cut.end && window.end > cut.start, + ); + if (overlapping.some((cut) => cut.score >= window.score)) continue; + for (const cut of overlapping) { + if (cuts.includes(cut)) cuts.splice(cuts.indexOf(cut), 1); + else saved.splice(saved.indexOf(cut), 1); + } + const clipId = cutClip(session.key, built.match, window, overlapping); + cuts.push({ + start: window.start, + end: window.end, + score: window.score, + clipId, + }); } } } +/** Cuts and saves the window, then drops the clips it `replaces`; resolves to the saved clip's id. */ async function cutClip( sessionKey: number, match: ScannerMatch, window: ClipWindow, -): Promise { - if (!ring) return; + replaces: readonly LiveCut[], +): Promise { + if (!ring) return null; try { - const clip = await ring.cut(window.start, window.end); - if (!clip) return; - await saveClip( + const clip = await ring.cut( + window.start, + window.end, + readSettings().audioOffsetMs / 1000, + ); + if (!clip) return null; + const saved = await saveClip( { createdAt: Date.now(), bucket: "session", @@ -451,9 +535,16 @@ async function cutClip( }, clip.blob, ); + ownClipIds.add(saved.id); + for (const cut of replaces) { + const id = await cut.clipId; + if (id !== null) await deleteClip(id); + } requestPersistentStorage(); await refreshClips(); + return saved.id; } catch (error) { set({ error: describeError(error) }); + return null; } } diff --git a/app/features/scanner/components/sendou-ingest.ts b/app/features/scanner/components/sendou-ingest.ts index 41ba37387..fc5d11304 100644 --- a/app/features/scanner/components/sendou-ingest.ts +++ b/app/features/scanner/components/sendou-ingest.ts @@ -66,7 +66,6 @@ export async function sendMatches({ buildScannerMatches(events.filter((e) => e.id !== undefined)), ); const selected = allBuilt.filter(include); - await clearOrphanedQueued(events, allBuilt, store); const result: SendResult = { sentMatches: 0, failedMatches: 0 }; for (const request of R.chunk(selected, MAX_MATCHES_PER_REQUEST)) { @@ -132,8 +131,8 @@ export function matchContaining( /** * The single send status a match displays, folded from its source events: an - * in-flight send wins, then failure, then success, then queued; within a - * state the most recent change is shown. + * in-flight send wins, then failure, then success; within a state the most + * recent change is shown. */ export function aggregateSendStatus( sources: readonly ScanEvent[], @@ -141,13 +140,7 @@ export function aggregateSendStatus( const statuses = sources .map((e) => e.send) .filter((status) => status !== undefined); - for (const state of [ - "sending", - "failed", - "unlinked", - "sent", - "queued", - ] as const) { + for (const state of ["sending", "failed", "unlinked", "sent"] as const) { const ofState = statuses.filter((status) => status.state === state); if (ofState.length > 0) { return ofState.reduce((a, b) => (a.at >= b.at ? a : b)); @@ -182,9 +175,7 @@ export function retryableUnlinkedMatches( export function unsentClosedMatches(built: BuiltMatch): boolean { return ( built.sources.some((e) => SCOREBOARD_EVENT_TYPES.includes(e.type)) && - built.sources.every( - (e) => e.send === undefined || e.send.state === "queued", - ) + built.sources.every((e) => e.send === undefined) ); } @@ -211,35 +202,6 @@ async function postIngestMatches( return res.json(); } -/** - * Live sending marks events "queued" as they arrive; ones the builder later - * leaves out (non-private match, older than the fallback window) would sit - * "queued" forever, so once a match boundary has passed them clear the status. - */ -async function clearOrphanedQueued( - events: readonly ScanEvent[], - allBuilt: BuiltMatch[], - store: string, -): Promise { - const lastBoundaryT = Math.max( - ...allBuilt.map((built) => built.sources.at(-1)!.t), - Number.NEGATIVE_INFINITY, - ); - const builtIds = new Set( - allBuilt.flatMap((built) => built.sources.map((e) => e.id)), - ); - const orphaned = events - .filter( - (e) => - e.send?.state === "queued" && - e.id !== undefined && - !builtIds.has(e.id) && - e.t <= lastBoundaryT, - ) - .map((e) => e.id!); - if (orphaned.length > 0) await updateEventsSend(orphaned, undefined, store); -} - async function errorText(res: Response): Promise { const text = await res.text().catch(() => ""); return `POST /ingest -> ${res.status}${text ? `: ${text.slice(0, 200)}` : ""}`; diff --git a/app/features/scanner/components/settings.ts b/app/features/scanner/components/settings.ts index 71bbd5ae3..612f92da3 100644 --- a/app/features/scanner/components/settings.ts +++ b/app/features/scanner/components/settings.ts @@ -1,7 +1,8 @@ /** - * The scanner's settings, kept in localStorage: the capture source, whether - * results upload to sendou.ink and whether live clips are saved. Read through - * a store so the controllers (outside React) and the views see one value. + * The scanner's settings, kept in localStorage: the capture source, what + * clips hear, whether results upload to sendou.ink and whether live clips + * are saved. Read through a store so the controllers (outside React) and + * the views see one value. */ import { useSyncExternalStore } from "react"; @@ -10,22 +11,38 @@ const STORAGE_KEY = "scanner:settings"; export interface ScannerSettings { /** `deviceId` of the video input; empty = the browser's default camera */ sourceDeviceId: string; + /** what live clips hear */ + audioSource: AudioSource; /** upload results to sendou.ink as games end (needs a login) */ upload: boolean; /** keep the live ring buffer and cut clips of the best moments */ saveClips: boolean; /** splats in a row that make a clip */ clipMinKills: ClipMinKills; + /** milliseconds the clips' sound is moved later (negative: earlier) against the picture */ + audioOffsetMs: number; } export const CLIP_MIN_KILLS_OPTIONS = [3, 4, 5] as const; export type ClipMinKills = (typeof CLIP_MIN_KILLS_OPTIONS)[number]; +/** + * `source`: the video input's own audio side; `desktop`: the system's sound + * through the share picker; `off`: silent clips; `device:`: a named + * audio input (a loopback device, say). + */ +export type AudioSource = "source" | "desktop" | "off" | `device:${string}`; + +/** two seconds either way covers any capture card against any audio path */ +export const AUDIO_OFFSET_LIMIT_MS = 2000; + const DEFAULT_SETTINGS: ScannerSettings = { sourceDeviceId: "", + audioSource: "source", upload: true, saveClips: true, clipMinKills: 4, + audioOffsetMs: 0, }; let settings: ScannerSettings | null = null; @@ -51,6 +68,20 @@ export function useScannerSettings(): ScannerSettings { return useSyncExternalStore(subscribe, readSettings, () => DEFAULT_SETTINGS); } +/** The `deviceId` a `device:` audio source names; null for the other kinds. */ +export function audioDeviceIdOf(source: AudioSource): string | null { + return source.startsWith("device:") ? source.slice("device:".length) : null; +} + +function isAudioSource(value: unknown): value is AudioSource { + return ( + value === "source" || + value === "desktop" || + value === "off" || + (typeof value === "string" && value.startsWith("device:")) + ); +} + function subscribe(listener: () => void): () => void { listeners.add(listener); return () => listeners.delete(listener); @@ -66,6 +97,9 @@ function load(): ScannerSettings { typeof parsed.sourceDeviceId === "string" ? parsed.sourceDeviceId : DEFAULT_SETTINGS.sourceDeviceId, + audioSource: isAudioSource(parsed.audioSource) + ? parsed.audioSource + : DEFAULT_SETTINGS.audioSource, upload: typeof parsed.upload === "boolean" ? parsed.upload @@ -77,6 +111,14 @@ function load(): ScannerSettings { clipMinKills: CLIP_MIN_KILLS_OPTIONS.find((n) => n === parsed.clipMinKills) ?? DEFAULT_SETTINGS.clipMinKills, + audioOffsetMs: + typeof parsed.audioOffsetMs === "number" && + Number.isFinite(parsed.audioOffsetMs) + ? Math.max( + -AUDIO_OFFSET_LIMIT_MS, + Math.min(AUDIO_OFFSET_LIMIT_MS, parsed.audioOffsetMs), + ) + : DEFAULT_SETTINGS.audioOffsetMs, }; } catch { return DEFAULT_SETTINGS; diff --git a/app/features/scanner/store/events.ts b/app/features/scanner/store/events.ts index 155f41599..815305d90 100644 --- a/app/features/scanner/store/events.ts +++ b/app/features/scanner/store/events.ts @@ -33,7 +33,7 @@ const RETENTION_INTERVAL_MS = 60_000; /** Where an event stands with sendou.ink /ingest; absent = never attempted. */ export interface SendStatus { /** "unlinked": sendou.ink stored the match but its game is not reported yet — resent on a backoff */ - state: "queued" | "sending" | "sent" | "unlinked" | "failed"; + state: "sending" | "sent" | "unlinked" | "failed"; /** wall-clock time of the last state change */ at: number; /** failure detail, set when state is "failed" */