This commit is contained in:
Kalle
2026-08-06 15:27:38 +03:00
parent 82413a64b5
commit 1da6ae37d0
24 changed files with 1507 additions and 551 deletions

View File

@@ -15,6 +15,7 @@ from the emberz repo; see `MIGRATION.md` there.
pnpm test:scanner # golden-file suite over tests/fixtures/ (Vitest, Node)
pnpm scanner:report # accuracy table + name character error rate across fixtures
pnpm scanner:fixtures [name-substring] # run detectors over matching fixtures, verbose
pnpm scanner:replay <dir> <startT> <fps> # replay ffmpeg-extracted frames through the scheduler+detectors
pnpm scanner:bootstrap-atlas # harvest labeled fixture crops into the glyph atlases
pnpm scanner:build-glyph-atlas # add the font-rendered charset (fonts required, see below)
pnpm scanner:build-localized-entries # regen localized closed sets from ../splat3
@@ -38,8 +39,8 @@ sequenceDiagram
participant UI as Live/VoD tab
participant ING as /ingest (scanner-ingest)
participant DB as IngestedMatch / IngestedScoreboard
Cap->>W: frame + t
W->>W: detectors gate() → parse()
Cap->>W: frame + t (live/screenshot/seek) — VoD: worker decodes its own slice
W->>W: scheduler dueDetectors() → gate() → parse()
W-->>TL: DetectedEvents
TL-->>UI: deduped timeline (IndexedDB on Live)
UI->>MB: buildScannerMatches(events)
@@ -91,13 +92,36 @@ sequenceDiagram
accuracy-critical matching internals in `core/glyphs.ts` and
`core/detectors/scoreboard/weapons.ts` — read those before touching
recognition code.
- A detector can declare `checkIntervalS` (objective: 1s) to cap how often
it is checked at all — the analyzer worker skips gate+parse in between
(`core/detectors/throttle.ts`) and exempts it from steady-frame
suppression — and `attachFrame: false` to keep continuously-firing events
from storing a frame PNG each. In the UI a match's objective reads render
as one step-line timeline (`components/ObjectiveTimeline.tsx`) instead of
per-event cards.
- Scheduling (`core/detectors/scheduler.ts`): per scan session the
DetectorScheduler decides which detectors look at a frame. While a gate
keeps failing its detector is checked every `searchIntervalS` (default
0.25s — produced/casted VoDs cut screens to ~1s with transition flicker
inside, so search cadence must not assume raw-gameplay screen lifetimes;
gates are ~ms-cheap, so this costs little); once the gate passes it
drops to the dense refine cadence for best-read refinement. Suppression ends a refinement streak
once it stagnates by parse count AND elapsed time (~3s — the time floor
keeps a dense cadence from suppressing during a screen's entry animation
before it is readable) or immediately at `sufficientConfidence`; death
adds `rearmCooldownS` (safe because it fits inside the Death timeline
merge window). `checkIntervalS` (objective: 1s)
still hard-caps both phases and exempts from suppression; a detector can
also declare `attachFrame: false` to keep continuously-firing events from
storing a frame PNG each. In the UI a match's objective reads render as
one step-line timeline (`components/ObjectiveTimeline.tsx`) instead of
per-event cards. Frames no detector is due for skip canvas readback
entirely, and everything is counted in `core/detectors/telemetry.ts`
(surfaced in the VoD tab's telemetry panel).
- VoD scans (`components/VodPage.tsx`): on the WebCodecs path the file's
duration is split into one contiguous slice per worker and each worker
demuxes + decodes its slice itself (mediabunny in the worker — no frames
cross the main thread). When the scheduler reports calm (no gate pass for
a quiet period, no open match — a confident map-start pins the scan
active until a scoreboard or timeout), the worker stops sequential decode
and skims keyframe-to-keyframe (single-frame decodes, hop capped at 2.5s
so short screens can't hide), snapping back to dense decode on any gate
pass. Chunks start dense, so slice boundaries are covered. The seek
fallback drives one worker and widens its stride over calm footage the
same way.
- Recognition is language-agnostic: OCR output snaps against every game
language at once (`core/localized-entries.ts`, generated) and events carry
sendou ids. English display names come from `components/labels.ts`.

View File

@@ -1,45 +1,27 @@
/**
* VoD frame extraction: step through a video file yielding (frame, t) as
* fast as decoding allows — no real-time playback. The primary path demuxes
* the file and decodes **every frame** sequentially with WebCodecs (via
* mediabunny), yielding the VideoFrames themselves (transferable to the
* analyzer workers with no main-thread conversion); when the container/codec
* can't be read that way, it falls back to seek-stepping a <video> element
* at a small fixed step, which handles anything the browser can play at the
* cost of per-seek latency and frame-exact coverage.
* VoD scan entry points. The primary path probes whether WebCodecs (via
* mediabunny) can decode the file — if so, the analyzer workers each demux
* and decode their own time slice of it (worker/analyzer.worker.ts) and no
* frames cross the main thread at all. When the container/codec can't be
* read that way, the fallback seek-steps a <video> element, which handles
* anything the browser can play at the cost of per-seek latency; its stride
* is supplied per step so the caller can widen it over calm footage.
*/
import { ALL_FORMATS, BlobSource, Input, VideoSampleSink } from "mediabunny";
/**
* Seek fallback step: a <video> element can't enumerate frames, so seek in
* increments small enough that anything but blink-and-miss overlays is caught.
*/
const SEEK_STEP_SECONDS = 0.25;
import { ALL_FORMATS, BlobSource, Input } from "mediabunny";
interface VodFrame {
/** the consumer owns the frame and must close() it */
frame: ImageBitmap | VideoFrame;
frame: ImageBitmap;
/** seconds into the video */
t: number;
}
export interface VodScan {
method: "webcodecs" | "seek";
duration: number;
frames: AsyncGenerator<VodFrame>;
dispose(): void;
}
/**
* Open a scan over `file`, yielding every decoded frame (or, on the seek
* fallback, one frame every SEEK_STEP_SECONDS). `video` must already have
* the file loaded (metadata not required yet); it is only driven by the
* seek fallback.
* Whether mediabunny + WebCodecs can decode `file`, and its duration if so.
*/
export async function openVodScan(
export async function probeWebCodecs(
file: File,
video: HTMLVideoElement,
): Promise<VodScan> {
): Promise<{ duration: number } | null> {
const input = new Input({
formats: ALL_FORMATS,
source: new BlobSource(file),
@@ -47,52 +29,40 @@ export async function openVodScan(
try {
const track = await input.getPrimaryVideoTrack();
if (track && (await track.canDecode())) {
const duration = await input.computeDuration([track]);
return {
method: "webcodecs",
duration,
frames: webCodecsFrames(input, new VideoSampleSink(track)),
dispose: () => input.dispose(),
};
return { duration: await input.computeDuration([track]) };
}
input.dispose();
return null;
} catch {
input.dispose();
}
await loadMetadata(video);
if (!Number.isFinite(video.duration)) {
throw new Error("video has no known duration — cannot scan by seeking");
}
return {
method: "seek",
duration: video.duration,
frames: seekFrames(video),
dispose: () => {},
};
}
async function* webCodecsFrames(
input: Input,
sink: VideoSampleSink,
): AsyncGenerator<VodFrame> {
try {
for await (const sample of sink.samples()) {
if (!sample) continue;
const t = sample.timestamp;
const frame = sample.toVideoFrame();
sample.close();
yield { frame, t };
}
return null;
} finally {
input.dispose();
}
}
async function* seekFrames(video: HTMLVideoElement): AsyncGenerator<VodFrame> {
for (let t = 0; t < video.duration; t += SEEK_STEP_SECONDS) {
/**
* Open a seek-stepping scan over `video` (which must already have the file
* loaded; metadata is awaited here). `nextStrideS` is consulted after every
* yielded frame, so analysis feedback can adjust the step on the fly.
*/
export async function openSeekScan(
video: HTMLVideoElement,
nextStrideS: () => number,
): Promise<{ duration: number; frames: AsyncGenerator<VodFrame> }> {
await loadMetadata(video);
if (!Number.isFinite(video.duration)) {
throw new Error("video has no known duration — cannot scan by seeking");
}
return { duration: video.duration, frames: seekFrames(video, nextStrideS) };
}
async function* seekFrames(
video: HTMLVideoElement,
nextStrideS: () => number,
): AsyncGenerator<VodFrame> {
for (let t = 0; t < video.duration; ) {
await seekTo(video, t);
yield { frame: await createImageBitmap(video), t };
t += Math.max(0.01, nextStrideS());
}
}

View File

@@ -1,25 +1,30 @@
/**
* VoD tab: load a video file and scan it for scoreboard matches as fast as
* decoding allows — no real-time playback (see src/capture/vod-frames.ts).
* Every frame is decoded and handed to a pool of analyzer workers; decode
* never waits on analysis (a frame arriving while all workers are busy is
* dropped — the next is ~1/60s away), so the scan runs at decode speed and
* analysis coverage stays as dense as the machine keeps up with — dense
* enough that even overlays visible for a fraction of a second are seen.
* Each match can be opened in the screenshot page with the exact frame
* that was analyzed.
* decoding allows — no real-time playback. On the primary (WebCodecs) path
* the file's duration is split into one contiguous slice per analyzer
* worker and each worker demuxes, decodes, schedules and analyzes its slice
* by itself (worker/analyzer.worker.ts): no frames cross the main thread,
* scheduling state is exact per slice, and calm stretches are skimmed by
* keyframe hops instead of decoded frame-by-frame. The seek fallback drives
* a <video> element through a single worker, widening its stride over calm
* footage. Each match can be opened in the screenshot page with the exact
* frame that was analyzed.
*
* Completed scans are persisted to IndexedDB keyed by file name
* (src/store/vods.ts); the default view lists them for reinspection.
*/
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { Link } from "react-router";
import { openVodScan } from "../capture/vod-frames";
import { openSeekScan, probeWebCodecs } from "../capture/vod-frames";
import { connectAbilities } from "../core/ability-harvest";
import {
OBJECTIVE_EVENT_TYPE,
type ObjectiveData,
} from "../core/detectors/objective/index";
import {
mergeScanTelemetry,
type ScanTelemetry,
} from "../core/detectors/telemetry";
import type { DetectedEvent } from "../core/detectors/types";
import {
buildScannerMatches,
@@ -36,7 +41,11 @@ import {
saveVod,
type VodSummary,
} from "../store/vods";
import { AnalyzerPool, defaultPoolSize } from "../worker/pool";
import {
AnalyzerClient,
type DoneInfo,
defaultScanWorkerCount,
} from "../worker/client";
import { withoutRepeatEvents } from "./dedupe-events";
import { EventCard, type GetFrame } from "./EventCard";
import { EventsSummary } from "./EventsSummary";
@@ -55,16 +64,14 @@ import {
import { sendouUpload } from "./sendou-upload";
import { thumbnailFromBlob } from "./thumbnail";
/** seek-fallback stride while the worker reports activity */
const SEEK_ACTIVE_STRIDE_S = 0.25;
/**
* Hard coverage floor: at most this much video may pass between two analyzed
* frames. Busy-dropping alone is not enough — around match results every
* gate fires and each analyzed frame runs a full parse (+ PNG encode), so
* the whole pool can stay busy for hundreds of ms while decode races ahead
* whole seconds of video; short screens (the own-results screen shows ~3s)
* then fall into the gap. When the budget is spent and no worker is free,
* decode waits.
* seek-fallback stride over calm footage (nothing detected for a while, no
* match open) — small enough that the screens that can start activity from
* dead air (results ~10s, match intro ~7s) still get sampled
*/
const MAX_ANALYSIS_GAP_SECONDS = 0.25;
const SEEK_CALM_STRIDE_S = 2.5;
type Status = "idle" | "scanning" | "done" | "error";
@@ -108,7 +115,12 @@ export function VodPage({
}) {
const videoRef = useRef<HTMLVideoElement>(null);
const previewRef = useRef<HTMLCanvasElement>(null);
const poolRef = useRef<AnalyzerPool | null>(null);
const clientsRef = useRef<AnalyzerClient[]>([]);
// cancels the in-flight chunk scans of the previous scan, if any
const abortScanRef = useRef<(() => void) | null>(null);
// seek fallback: latest per-frame done info + the waiter for the next one
const doneInfoRef = useRef<DoneInfo | null>(null);
const frameDoneRef = useRef<(() => void) | null>(null);
const timelineRef = useRef(new TimelineBuilder());
// latest gate score from any worker; flushed to state on the UI throttle
const gateScoreRef = useRef<number | null>(null);
@@ -131,6 +143,7 @@ export function VodPage({
const [matches, setMatches] = useState<VodMatch[]>([]);
const [vods, setVods] = useState<VodSummary[]>([]);
const [error, setError] = useState<string | null>(null);
const [telemetry, setTelemetry] = useState<ScanTelemetry | null>(null);
const [over, setOver] = useState(false);
const [eventsOpen, setEventsOpen] = useState(false);
const [resultsSend, setResultsSend] = useState<ResultsSend | null>(null);
@@ -207,7 +220,9 @@ export function VodPage({
void refreshVods();
return () => {
abortRef.current.aborted = true;
poolRef.current?.dispose();
abortScanRef.current?.();
for (const client of clientsRef.current) client.dispose();
clientsRef.current = [];
if (urlRef.current) URL.revokeObjectURL(urlRef.current);
};
}, [refreshVods]);
@@ -215,10 +230,12 @@ export function VodPage({
const scan = useCallback(
async (file: File) => {
abortRef.current.aborted = true;
abortScanRef.current?.();
const abort = { aborted: false };
abortRef.current = abort;
setError(null);
setTelemetry(null);
setMatches([]);
setResultsSend(null);
setEventsOpen(false);
@@ -229,7 +246,6 @@ export function VodPage({
setSource("scan");
setStatus("scanning");
let dispose = () => {};
try {
// the element is used by the seek fallback and for post-scan review
const video = videoRef.current!;
@@ -237,109 +253,83 @@ export function VodPage({
urlRef.current = URL.createObjectURL(file);
video.src = urlRef.current;
poolRef.current ??= new AnalyzerPool(
defaultPoolSize(),
(result) => {
gateScoreRef.current = result.gate.score;
if (!result.gate.pass) return;
for (const event of result.events as DetectedEvent<FixtureData>[]) {
const action = timelineRef.current.push(event);
if (action.action !== "added" && action.action !== "replaced")
continue;
const frame = result.frame;
sideWorkRef.current.push(
(async () => {
const thumbnail = frame
? await thumbnailFromBlob(frame)
: undefined;
const replaced =
action.action === "replaced"
? matchesRef.current.find(
(m) => m.event === action.replaced,
)
: undefined;
const next = matchesRef.current.filter((m) => m !== replaced);
next.push({
event,
key: replaced?.key ?? nextMatchKeyRef.current++,
thumbnail,
frame,
});
next.sort((a, b) => a.event.t - b.event.t);
matchesRef.current = next;
setMatches(next);
})().catch(() => {}),
);
}
},
(message) => {
setError(message);
setStatus("error");
},
);
const pool = poolRef.current;
await pool.whenReady();
// let frames still in flight from an aborted scan finish before the
// timeline resets, so their results can't bleed into this scan
await pool.whenIdle();
if (clientsRef.current.length === 0) {
clientsRef.current = Array.from(
{ length: defaultScanWorkerCount() },
() =>
new AnalyzerClient(
(result) => {
gateScoreRef.current = result.gate.score;
if (!result.gate.pass) return;
for (const event of result.events as DetectedEvent<FixtureData>[]) {
const action = timelineRef.current.push(event);
if (
action.action !== "added" &&
action.action !== "replaced"
)
continue;
const frame = result.frame;
sideWorkRef.current.push(
(async () => {
const thumbnail = frame
? await thumbnailFromBlob(frame)
: undefined;
const replaced =
action.action === "replaced"
? matchesRef.current.find(
(m) => m.event === action.replaced,
)
: undefined;
const next = matchesRef.current.filter(
(m) => m !== replaced,
);
next.push({
event,
key: replaced?.key ?? nextMatchKeyRef.current++,
thumbnail,
frame,
});
next.sort((a, b) => a.event.t - b.event.t);
matchesRef.current = next;
setMatches(next);
})().catch(() => {}),
);
}
},
(message) => {
frameDoneRef.current?.();
frameDoneRef.current = null;
setError(message);
setStatus("error");
},
(_t, info: DoneInfo) => {
doneInfoRef.current = info;
frameDoneRef.current?.();
frameDoneRef.current = null;
},
),
);
}
const clients = clientsRef.current;
await Promise.all(clients.map((c) => c.whenReady()));
// let work still in flight from an aborted scan finish before the
// timeline resets, so its results can't bleed into this scan
await Promise.all(clients.map((c) => c.whenIdle()));
if (abort.aborted) return;
matchesRef.current = [];
sideWorkRef.current = [];
timelineRef.current = new TimelineBuilder();
const vod = await openVodScan(file, video);
dispose = () => vod.dispose();
if (abort.aborted) return;
setMethod(vod.method);
const started = performance.now();
// each frame goes to an idle worker; with none free it is dropped
// unless MAX_ANALYSIS_GAP_SECONDS of video has passed unanalyzed,
// in which case decode waits for a worker. Preview/progress
// re-renders are throttled off the hot loop (the preview draw must
// precede tryAnalyze — transferring the frame to a worker detaches it).
let lastUiUpdate = Number.NEGATIVE_INFINITY;
let lastAnalyzedT = Number.NEGATIVE_INFINITY;
for await (const { frame, t } of vod.frames) {
if (abort.aborted) {
frame.close();
break;
}
const now = performance.now();
if (now - lastUiUpdate >= 250) {
lastUiUpdate = now;
drawPreview(previewRef.current, frame);
setGateScore(gateScoreRef.current);
const elapsed = (now - started) / 1000;
setProgress({
t,
duration: vod.duration,
rate: elapsed > 0 ? t / elapsed : 0,
});
}
if (!pool.hasIdle()) {
if (t - lastAnalyzedT < MAX_ANALYSIS_GAP_SECONDS) {
frame.close();
continue;
}
await pool.whenAnyIdle();
if (abort.aborted) {
frame.close();
break;
}
}
pool.tryAnalyze(frame, t);
lastAnalyzedT = t;
}
await pool.whenIdle();
await Promise.all(sideWorkRef.current);
if (!abort.aborted) {
const finalize = async (duration: number) => {
await Promise.all(sideWorkRef.current);
if (abort.aborted) return;
matchesRef.current = withoutInvalidObjectives(matchesRef.current);
setMatches(matchesRef.current);
setProgress((p) => (p ? { ...p, t: vod.duration } : p));
setProgress((p) => (p ? { ...p, t: duration } : p));
setStatus("done");
await saveVod(
{ name: file.name, savedAt: Date.now(), duration: vod.duration },
{ name: file.name, savedAt: Date.now(), duration },
matchesRef.current.map((m) => ({
type: m.event.type,
t: m.event.t,
@@ -350,14 +340,126 @@ export function VodPage({
})),
);
await refreshVods();
};
const probe = await probeWebCodecs(file);
if (abort.aborted) return;
if (probe) {
// each worker demuxes, decodes and analyzes its own slice of
// the file; the main thread only aggregates progress
setMethod("webcodecs");
const { duration } = probe;
const chunkSpan = duration / clients.length;
const chunks = clients.map((client, i) => ({
client,
tStart: i * chunkSpan,
tEnd: i === clients.length - 1 ? duration : (i + 1) * chunkSpan,
t: i * chunkSpan,
done: false,
telemetry: null as ScanTelemetry | null,
}));
abortScanRef.current = () => {
for (const client of clients) client.abortChunk();
};
const mergedTelemetry = () =>
mergeScanTelemetry(
chunks.flatMap((c) => (c.telemetry ? [c.telemetry] : [])),
);
let lastUiUpdate = Number.NEGATIVE_INFINITY;
const pushUiUpdate = () => {
const now = performance.now();
if (now - lastUiUpdate < 250) return;
lastUiUpdate = now;
const covered = chunks.reduce(
(sum, c) => sum + (Math.min(c.t, c.tEnd) - c.tStart),
0,
);
const elapsed = (now - started) / 1000;
setGateScore(gateScoreRef.current);
setProgress({
t: covered,
duration,
rate: elapsed > 0 ? covered / elapsed : 0,
});
setTelemetry(mergedTelemetry());
};
await Promise.all(
chunks.map((chunk, chunkIndex) =>
chunk.client
.scanChunk(
{ file, chunkIndex, tStart: chunk.tStart, tEnd: chunk.tEnd },
(progress) => {
chunk.t = progress.t;
chunk.telemetry = progress.telemetry;
if (progress.preview) {
// show one chunk at a time: the earliest still running
if (chunks.find((c) => !c.done) === chunk) {
drawPreview(previewRef.current, progress.preview);
}
progress.preview.close();
}
pushUiUpdate();
},
)
.then((chunkTelemetry) => {
chunk.done = true;
chunk.t = chunk.tEnd;
chunk.telemetry = chunkTelemetry;
}),
),
);
if (abort.aborted) return;
setTelemetry(mergedTelemetry());
await finalize(duration);
return;
}
// seek fallback: one worker, one frame in flight; the worker's calm
// signal widens the stride over dead air
setMethod("seek");
const strideRef = { current: SEEK_ACTIVE_STRIDE_S };
const vod = await openSeekScan(video, () => strideRef.current);
if (abort.aborted) return;
const client = clients[0]!;
let lastUiUpdate = Number.NEGATIVE_INFINITY;
for await (const { frame, t } of vod.frames) {
if (abort.aborted) {
frame.close();
break;
}
const now = performance.now();
if (now - lastUiUpdate >= 250) {
lastUiUpdate = now;
// the preview draw must precede analyze — transferring the
// frame to the worker detaches it
drawPreview(previewRef.current, frame);
setGateScore(gateScoreRef.current);
const elapsed = (now - started) / 1000;
setProgress({
t,
duration: vod.duration,
rate: elapsed > 0 ? t / elapsed : 0,
});
if (doneInfoRef.current)
setTelemetry(doneInfoRef.current.telemetry);
}
await new Promise<void>((resolve) => {
frameDoneRef.current = resolve;
if (!client.analyze(frame, t)) resolve();
});
strideRef.current = doneInfoRef.current?.calm
? SEEK_CALM_STRIDE_S
: SEEK_ACTIVE_STRIDE_S;
}
if (doneInfoRef.current) setTelemetry(doneInfoRef.current.telemetry);
await finalize(vod.duration);
} catch (e) {
if (!abort.aborted) {
abortScanRef.current?.();
setError(String(e));
setStatus("error");
}
} finally {
dispose();
}
},
[refreshVods],
@@ -365,6 +467,8 @@ export function VodPage({
const openStored = useCallback(async (name: string) => {
abortRef.current.aborted = true;
abortScanRef.current?.();
setTelemetry(null);
try {
const events = await loadVodEvents(name);
// VoDs saved before objective reads were mode-gated may carry them
@@ -408,6 +512,8 @@ export function VodPage({
const backToList = useCallback(() => {
abortRef.current.aborted = true;
abortScanRef.current?.();
setTelemetry(null);
matchesRef.current = [];
setMatches([]);
setResultsSend(null);
@@ -534,6 +640,9 @@ export function VodPage({
)}
</div>
{error && <p className="error">{error}</p>}
{showVodView && telemetry ? (
<TelemetryPanel telemetry={telemetry} />
) : null}
{!showVodView && (
<div className="vod-list">
{vods.length === 0 && (
@@ -689,6 +798,51 @@ function frameLoader(m: VodMatch): GetFrame | undefined {
: undefined;
}
function TelemetryPanel({ telemetry }: { telemetry: ScanTelemetry }) {
const detectors = Object.entries(telemetry.detectors).sort(([a], [b]) =>
a.localeCompare(b),
);
const coveredS = telemetry.activeVideoS + telemetry.skimVideoS;
return (
<details className="telemetry">
<summary>
telemetry · analyzed {telemetry.analyzedFrames}/
{telemetry.decodedFrames} decoded frames
{coveredS > 0 &&
` · skimmed ${formatTime(telemetry.skimVideoS)} of ${formatTime(coveredS)}`}
{telemetry.wallMs > 0 &&
` · ${formatTime(telemetry.wallMs / 1000)} cpu`}
</summary>
<table>
<thead>
<tr>
<th>detector</th>
<th>checks</th>
<th>gate pass</th>
<th>gate ms</th>
<th>parses</th>
<th>parse ms</th>
<th>suppressed</th>
</tr>
</thead>
<tbody>
{detectors.map(([id, d]) => (
<tr key={id}>
<td>{id}</td>
<td>{d.checks}</td>
<td>{d.gatePasses}</td>
<td>{Math.round(d.gateMs)}</td>
<td>{d.parses}</td>
<td>{Math.round(d.parseMs)}</td>
<td>{d.suppressedParses}</td>
</tr>
))}
</tbody>
</table>
</details>
);
}
function drawPreview(
canvas: HTMLCanvasElement | null,
frame: ImageBitmap | VideoFrame,

View File

@@ -1066,6 +1066,33 @@ html.light {
padding: 10px 12px;
}
.scanner-app .telemetry {
margin: 8px 0;
font-size: var(--font-2xs);
color: var(--color-text-high);
}
.scanner-app .telemetry summary {
cursor: pointer;
}
.scanner-app .telemetry table {
margin-top: 6px;
border-collapse: collapse;
}
.scanner-app .telemetry table th,
.scanner-app .telemetry table td {
padding: 2px 10px 2px 0;
text-align: right;
font-variant-numeric: tabular-nums;
}
.scanner-app .telemetry table th:first-child,
.scanner-app .telemetry table td:first-child {
text-align: left;
}
@media (prefers-reduced-motion: reduce) {
.scanner-app .match-card,
.scanner-app .match-card.sending::after,

View File

@@ -820,5 +820,14 @@ export function createDeathDetector(
];
}
return { id: "death", gate, parse };
// the death cam's animated background flickers the gate; after a
// sufficient read the 4s rearm hold stays inside the timeline's 8s
// Death merge window, so every parse it skips would merge anyway
return {
id: "death",
sufficientConfidence: 0.98,
rearmCooldownS: 4,
gate,
parse,
};
}

View File

@@ -365,5 +365,10 @@ export function createMapStartDetector(
];
}
return { id: "map-start", gate, parse };
return {
id: "map-start",
sufficientConfidence: 0.98,
gate,
parse,
};
}

View File

@@ -639,5 +639,10 @@ export function createMinimapDetector(
];
}
return { id: "minimap", gate, parse };
return {
id: "minimap",
sufficientConfidence: 0.98,
gate,
parse,
};
}

View File

@@ -0,0 +1,311 @@
/**
* DetectorScheduler: one temporally-ordered scan session's answer to "which
* detectors should look at the frame at time t, and which of those should
* pay for a parse?". It folds together three ideas:
*
* - Cadence. While a detector's gate keeps failing it is checked every
* `searchIntervalS` (default 0.25s — produced VoDs cut screens to as
* little as ~1s with transition flicker inside that, so the search
* cadence must not assume raw-gameplay screen lifetimes; gates are
* ~1.6ms, so searching at the analysis floor costs nothing). Once the
* gate passes it drops to the dense `refineIntervalS` so the best-read
* refinement loop still sees every frame it wants. `checkIntervalS`
* (objective counter) remains a hard cap on both phases and exempts the
* detector from suppression, as before. `nextDueT()` lets the caller
* skip a frame's canvas readback + normalize entirely when no detector
* is due.
*
* - Suppression. A static screen that keeps a gate firing stops paying for
* parses once a streak stagnates: at least `maxStagnantParses`
* consecutive parses AND `stagnantAfterS` seconds without the best
* confidence improving. The time floor matters — a gate often fires
* during a screen's entry animation while parses still read nothing, and
* at a dense sampling cadence a pure parse count would suppress before
* the screen ever becomes readable (with the gate held high, it would
* then never re-arm). A parse reaching the detector's
* `sufficientConfidence` suppresses immediately: the timeline keeps the
* best read per merge window, so once a read is that good further parses
* cannot change the outcome. A
* detector with `rearmCooldownS` (death: the overlay's animated
* background makes its gate flicker) additionally skips parses for that
* long after a sufficient read, across gate drops; safe only where the
* timeline merges purely on time and the cooldown fits inside the merge
* window.
*
* - Activity. The scheduler tracks when any gate last passed and whether a
* match is open (a confident map-start without a closing scoreboard), so
* the VoD chunk scanner can tell dead air ("calm") from footage that
* deserves dense decoding.
*
* All state is keyed to the scan's own clock; a `t` jumping backwards means
* a new capture session or rescan and resets everything.
*/
import type { DetectedEvent } from "./types";
export interface SchedulingInfo {
id: string;
checkIntervalS?: number;
searchIntervalS?: number;
sufficientConfidence?: number;
rearmCooldownS?: number;
}
export interface SchedulerOptions {
/**
* false = one-shot harness mode: every detector is due on every frame and
* parses are never suppressed
*/
suppressSteadyFrames: boolean;
/** check cadence while a detector's gate is passing */
refineIntervalS: number;
/** check cadence while a detector's gate is failing (per-detector
* searchIntervalS overrides) */
searchIntervalS: number;
/** consecutive non-improving parses tolerated before suppression */
maxStagnantParses: number;
/** seconds without improvement tolerated before suppression */
stagnantAfterS: number;
/** minimum confidence gain that counts as an improvement */
minImprovement: number;
/** seconds without any gate pass before the scan counts as calm */
quietAfterS: number;
/** a match opened this long ago without closing is assumed abandoned */
matchOpenMaxS: number;
/** event types that open a match (map-start) */
matchOpeningTypes: readonly string[];
/** event types that close a match (the scoreboard family) */
matchClosingTypes: readonly string[];
}
const DEFAULT_SCHEDULER_OPTIONS: SchedulerOptions = {
suppressSteadyFrames: true,
refineIntervalS: 0.15,
searchIntervalS: 0.25,
maxStagnantParses: 6,
// give a screen ~3s to animate in and produce its best read, whatever
// the sampling cadence
stagnantAfterS: 3,
minImprovement: 0.001,
quietAfterS: 15,
matchOpenMaxS: 8 * 60,
matchOpeningTypes: [],
matchClosingTypes: [],
};
/** events below this are too dubious to drive match-open/close state */
const MATCH_STATE_MIN_CONFIDENCE = 0.6;
/** tolerated backwards jitter in t before the session counts as restarted */
const RESET_TOLERANCE_S = 5;
const INTERVAL_EPSILON_S = 1e-6;
interface StreakState {
best: number;
stagnant: number;
lastImprovementT: number;
suppressed: boolean;
}
interface DetectorState {
info: SchedulingInfo;
lastCheckT: number | undefined;
gatePassing: boolean;
streak: StreakState | null;
/** parses skipped until this t after a sufficient read (rearmCooldownS) */
parseHoldUntilT: number;
}
export class DetectorScheduler {
#options: SchedulerOptions;
#states = new Map<string, DetectorState>();
#maxT = Number.NEGATIVE_INFINITY;
#lastActivityT = Number.NEGATIVE_INFINITY;
#matchOpenUntilT = Number.NEGATIVE_INFINITY;
constructor(
detectors: readonly SchedulingInfo[],
options: Partial<SchedulerOptions> = {},
) {
this.#options = { ...DEFAULT_SCHEDULER_OPTIONS, ...options };
for (const info of detectors) {
this.#states.set(info.id, freshState(info));
}
}
/** Drop all session state; `t` seeds the activity clock (chunk start). */
reset(t = Number.NEGATIVE_INFINITY): void {
for (const [id, state] of this.#states) {
this.#states.set(id, freshState(state.info));
}
this.#maxT = t;
this.#lastActivityT = t;
this.#matchOpenUntilT = Number.NEGATIVE_INFINITY;
}
/**
* Earliest t at which any detector wants a check — frames before it can
* skip analysis (and its canvas readback) entirely.
*/
nextDueT(): number {
let next = Number.POSITIVE_INFINITY;
for (const state of this.#states.values()) {
if (state.lastCheckT === undefined) return Number.NEGATIVE_INFINITY;
next = Math.min(next, state.lastCheckT + this.#interval(state));
}
return next;
}
/** Detector ids that should gate the frame at `t`. */
dueDetectors(t: number): string[] {
if (t + RESET_TOLERANCE_S < this.#maxT) this.reset(t);
// the first frame seeds the activity clock so a fresh session is never
// instantly calm — it has to earn its quiet period first
if (this.#lastActivityT === Number.NEGATIVE_INFINITY) {
this.#lastActivityT = t;
}
this.#maxT = Math.max(this.#maxT, t);
const due: string[] = [];
for (const [id, state] of this.#states) {
if (state.lastCheckT === undefined) {
due.push(id);
continue;
}
if (t - state.lastCheckT >= this.#interval(state) - INTERVAL_EPSILON_S) {
due.push(id);
}
}
return due;
}
/** Report a gate outcome for a detector this scheduler marked due. */
recordGate(id: string, t: number, pass: boolean): void {
const state = this.#states.get(id);
if (!state) return;
state.lastCheckT = t;
state.gatePassing = pass;
if (pass) {
this.#lastActivityT = Math.max(this.#lastActivityT, t);
} else {
state.streak = null;
}
}
/** Whether the (passed) gate should be followed by a parse at `t`. */
shouldParse(id: string, t: number): boolean {
if (!this.#options.suppressSteadyFrames) return true;
const state = this.#states.get(id);
if (!state) return true;
if (state.info.checkIntervalS !== undefined) return true;
if (t < state.parseHoldUntilT) return false;
return !state.streak?.suppressed;
}
/** Report the outcome of a parse this scheduler approved. */
recordParse(
id: string,
t: number,
events: readonly Pick<DetectedEvent, "type" | "confidence">[],
): void {
this.#recordMatchState(t, events);
const state = this.#states.get(id);
if (!state || state.info.checkIntervalS !== undefined) return;
// no events counts as confidence 0: a false-firing gate on a static
// screen stagnates and gets suppressed just like a parsed one
const confidence = events.reduce(
(max, e) => Math.max(max, e.confidence),
0,
);
const { sufficientConfidence, rearmCooldownS } = state.info;
if (
sufficientConfidence !== undefined &&
confidence >= sufficientConfidence
) {
state.streak = {
best: confidence,
stagnant: 0,
lastImprovementT: t,
suppressed: true,
};
if (rearmCooldownS !== undefined) {
state.parseHoldUntilT = t + rearmCooldownS;
}
return;
}
if (!state.streak) {
state.streak = {
best: confidence,
stagnant: 0,
lastImprovementT: t,
suppressed: false,
};
return;
}
const streak = state.streak;
if (confidence > streak.best + this.#options.minImprovement) {
streak.best = confidence;
streak.stagnant = 0;
streak.lastImprovementT = t;
return;
}
streak.stagnant += 1;
if (
streak.stagnant >= this.#options.maxStagnantParses &&
t - streak.lastImprovementT >= this.#options.stagnantAfterS
) {
streak.suppressed = true;
}
}
/**
* True when the footage at `t` is dead air as far as detection goes: no
* gate has passed for quietAfterS and no match is open — the VoD scanner
* may skim by keyframes instead of decoding densely.
*/
calm(t: number): boolean {
if (!this.#options.suppressSteadyFrames) return false;
return (
t - this.#lastActivityT >= this.#options.quietAfterS &&
t >= this.#matchOpenUntilT
);
}
#interval(state: DetectorState): number {
if (!this.#options.suppressSteadyFrames) return 0;
const { info } = state;
if (info.checkIntervalS !== undefined) return info.checkIntervalS;
const search = info.searchIntervalS ?? this.#options.searchIntervalS;
// while suppressed only the gate keeps running (to spot the screen
// changing), which the sparser search cadence covers
if (state.streak?.suppressed) return search;
return state.gatePassing ? this.#options.refineIntervalS : search;
}
#recordMatchState(
t: number,
events: readonly Pick<DetectedEvent, "type" | "confidence">[],
): void {
for (const event of events) {
if (event.confidence < MATCH_STATE_MIN_CONFIDENCE) continue;
if (this.#options.matchOpeningTypes.includes(event.type)) {
this.#matchOpenUntilT = Math.max(
this.#matchOpenUntilT,
t + this.#options.matchOpenMaxS,
);
} else if (this.#options.matchClosingTypes.includes(event.type)) {
this.#matchOpenUntilT = Math.min(this.#matchOpenUntilT, t);
}
}
}
}
function freshState(info: SchedulingInfo): DetectorState {
return {
info,
lastCheckT: undefined,
gatePassing: false,
streak: null,
parseHoldUntilT: Number.NEGATIVE_INFINITY,
};
}

View File

@@ -256,5 +256,10 @@ export function createScoreboardOwnDetector(
];
}
return { id: "scoreboard-own", gate, parse };
return {
id: "scoreboard-own",
sufficientConfidence: 0.98,
gate,
parse,
};
}

View File

@@ -426,5 +426,12 @@ export function createScoreboardReplayDetector(
];
}
return { id: "scoreboard-replay", gate, parse };
// no rearm cooldown — distinct replays browsed in quick succession are
// told apart by content
return {
id: "scoreboard-replay",
sufficientConfidence: 0.98,
gate,
parse,
};
}

View File

@@ -321,5 +321,11 @@ export function createScoreboardDetector(
];
}
return { id: "scoreboard", gate, parse };
// a 0.98 mean field score is a clean full read
return {
id: "scoreboard",
sufficientConfidence: 0.98,
gate,
parse,
};
}

View File

@@ -1,83 +0,0 @@
/**
* ParseSuppressor: bails out of re-parsing a static screen.
*
* A screen that keeps a detector's gate firing (the results scoreboard can
* sit for tens of seconds, a paused VoD indefinitely) makes the pipeline
* re-run the expensive parse() on every sampled frame even though nothing
* changes. Per detector this tracks the best event confidence seen during a
* continuous gate-pass streak; once `maxStagnantParses` consecutive parses
* fail to improve on it, parse() is skipped (the cheap gate keeps running)
* until the gate drops — i.e. the screen actually changed.
*/
export interface SuppressorOptions {
/** consecutive non-improving parses tolerated before suppression kicks in */
maxStagnantParses: number;
/** minimum confidence gain that counts as an improvement */
minImprovement: number;
}
const DEFAULT_SUPPRESSOR_OPTIONS: SuppressorOptions = {
// at the 2fps sample rate: give a stable screen ~3s to produce its best
// read, then stop paying for parses until the screen changes
maxStagnantParses: 6,
minImprovement: 0.001,
};
interface StreakState {
best: number;
stagnant: number;
suppressed: boolean;
}
export class ParseSuppressor {
#options: SuppressorOptions;
#streaks = new Map<string, StreakState>();
constructor(options: Partial<SuppressorOptions> = {}) {
this.#options = { ...DEFAULT_SUPPRESSOR_OPTIONS, ...options };
}
/**
* Call once per detector per frame with the gate outcome; returns whether
* parse() should run. A failed gate ends the streak, so the next gate pass
* starts a fresh, unsuppressed streak.
*/
shouldParse(detectorId: string, gatePass: boolean): boolean {
if (!gatePass) {
this.#streaks.delete(detectorId);
return false;
}
return !this.#streaks.get(detectorId)?.suppressed;
}
/** Report the outcome of a parse this suppressor approved. */
recordParse(
detectorId: string,
events: readonly { confidence: number }[],
): void {
// no events counts as confidence 0: a false-firing gate on a static
// screen stagnates and gets suppressed just like a parsed one
const confidence = events.reduce(
(max, e) => Math.max(max, e.confidence),
0,
);
const streak = this.#streaks.get(detectorId);
if (!streak) {
this.#streaks.set(detectorId, {
best: confidence,
stagnant: 0,
suppressed: false,
});
return;
}
if (confidence > streak.best + this.#options.minImprovement) {
streak.best = confidence;
streak.stagnant = 0;
return;
}
streak.stagnant += 1;
if (streak.stagnant >= this.#options.maxStagnantParses)
streak.suppressed = true;
}
}

View File

@@ -0,0 +1,85 @@
/**
* Scan telemetry: counters the analyzer accumulates so scan performance is
* measurable instead of guessed at — how many frames were decoded vs.
* actually analyzed, where detector time goes (gate vs. parse), how much
* work scheduling saved, and how much of a VoD was covered in skim mode.
* Plain JSON so it travels over the worker boundary as-is.
*/
export interface DetectorTelemetry {
/** frames on which the detector's gate ran */
checks: number;
gatePasses: number;
gateMs: number;
parses: number;
parseMs: number;
/** gate passed but parse() was skipped (suppression / cooldown) */
suppressedParses: number;
}
export interface ScanTelemetry {
decodedFrames: number;
/** frames that went through canvas readback + normalize + detectors */
analyzedFrames: number;
/** video seconds covered by dense sequential decode (chunk scan) */
activeVideoS: number;
/** video seconds covered by keyframe-hop skimming (chunk scan) */
skimVideoS: number;
wallMs: number;
detectors: Record<string, DetectorTelemetry>;
}
/** Fresh all-zero telemetry for one scan/session. */
export function createScanTelemetry(): ScanTelemetry {
return {
decodedFrames: 0,
analyzedFrames: 0,
activeVideoS: 0,
skimVideoS: 0,
wallMs: 0,
detectors: {},
};
}
/** Get-or-create the per-detector counters bucket. */
export function detectorTelemetry(
telemetry: ScanTelemetry,
id: string,
): DetectorTelemetry {
const existing = telemetry.detectors[id];
if (existing) return existing;
const created: DetectorTelemetry = {
checks: 0,
gatePasses: 0,
gateMs: 0,
parses: 0,
parseMs: 0,
suppressedParses: 0,
};
telemetry.detectors[id] = created;
return created;
}
/** Sum telemetry across parallel chunk scans into one report. */
export function mergeScanTelemetry(
parts: readonly ScanTelemetry[],
): ScanTelemetry {
const out = createScanTelemetry();
for (const part of parts) {
out.decodedFrames += part.decodedFrames;
out.analyzedFrames += part.analyzedFrames;
out.activeVideoS += part.activeVideoS;
out.skimVideoS += part.skimVideoS;
out.wallMs = Math.max(out.wallMs, part.wallMs);
for (const [id, d] of Object.entries(part.detectors)) {
const bucket = detectorTelemetry(out, id);
bucket.checks += d.checks;
bucket.gatePasses += d.gatePasses;
bucket.gateMs += d.gateMs;
bucket.parses += d.parses;
bucket.parseMs += d.parseMs;
bucket.suppressedParses += d.suppressedParses;
}
}
return out;
}

View File

@@ -1,24 +0,0 @@
/**
* CheckThrottle: caps how often a detector is checked at all (gate included),
* for detectors that declare a checkIntervalS — screens like the objective
* counter change at most once a second, so sampling them at the full frame
* rate buys nothing.
*/
export class CheckThrottle {
#lastCheckT = new Map<string, number>();
/**
* Whether detector `id` should run its check on the frame at `t` seconds.
* Approving a check starts the detector's next interval; a `t` earlier
* than the last approved check (a fresh capture session, a VoD rescan)
* resets the window instead of blocking until the old timeline catches up.
*/
shouldCheck(id: string, t: number, intervalS: number | undefined): boolean {
if (intervalS === undefined) return true;
const last = this.#lastCheckT.get(id);
if (last !== undefined && t >= last && t - last < intervalS) return false;
this.#lastCheckT.set(id, t);
return true;
}
}

View File

@@ -33,15 +33,41 @@ export interface GateResult {
export interface Detector<TData = unknown> {
id: string;
/**
* Minimum seconds between checks: frames inside the interval skip gate
* and parse entirely (core/detectors/throttle.ts, applied by the
* analyzer worker — per worker, so a VoD pool checks up to poolSize×
* this often). Declaring an interval also exempts the detector from
* steady-frame suppression: it marks a screen that stays up for minutes
* with *changing* content, which the stagnating-confidence heuristic
* would wrongly silence.
* Minimum seconds between checks in *both* scheduling phases: frames
* inside the interval skip gate and parse entirely
* (core/detectors/scheduler.ts, applied by the analyzer worker).
* Declaring an interval also exempts the detector from steady-frame
* suppression: it marks a screen that stays up for minutes with
* *changing* content, which the stagnating-confidence heuristic would
* wrongly silence.
*/
checkIntervalS?: number;
/**
* Check cadence while the gate keeps failing (the search phase); unset =
* the scheduler's default (0.25s). Do not reason from raw-gameplay
* screen lifetimes here: produced/casted VoDs cut screens (results,
* intros) to as little as ~1s, with transition flicker eating frames
* inside that window — the search cadence must sample such a window
* several times. Gates cost ~1.6ms, so searching at the analysis floor
* is effectively free; only override upward with strong evidence.
*/
searchIntervalS?: number;
/**
* A parse reaching this confidence ends the streak's refinement
* immediately: the timeline keeps the best read per merge window, so a
* read this good cannot be improved upon in any way that matters.
* Conservative by construction — if real reads never reach it, behavior
* falls back to stagnation-based suppression.
*/
sufficientConfidence?: number;
/**
* After a sufficient read, skip parses for this long even across gate
* drops — for overlays whose animated background flickers the gate
* (death). Only safe when the event type's timeline merge is purely
* time-based and the cooldown fits inside the merge window, so every
* parse the cooldown skips would have merged into the same event anyway.
*/
rearmCooldownS?: number;
/**
* false = worker results for this detector's events ship without the
* analyzed-frame PNG — for detectors whose events fire continuously,

View File

@@ -0,0 +1,218 @@
/**
* Unit tests for DetectorScheduler: search/refine check cadence,
* steady-frame suppression (stagnation and sufficient-confidence early
* stop), the rearm cooldown, checkIntervalS compatibility, and the calm
* signal that drives VoD skim mode.
*/
import assert from "node:assert/strict";
import {
DetectorScheduler,
type SchedulingInfo,
} from "../core/detectors/scheduler";
import test from "./node-test-compat";
const OPTS = {
refineIntervalS: 0.1,
searchIntervalS: 0.1,
maxStagnantParses: 3,
stagnantAfterS: 0.25,
minImprovement: 0.001,
quietAfterS: 10,
matchOpenMaxS: 60,
matchOpeningTypes: ["MapStart"],
matchClosingTypes: ["Scoreboard"],
};
function make(detector: Partial<SchedulingInfo>, options = {}) {
return new DetectorScheduler([{ id: "d", ...detector }], {
...OPTS,
...options,
});
}
function feed(
s: DetectorScheduler,
t: number,
options: { pass: boolean; confidence?: number; type?: string },
): "skipped" | "gated" | "parsed" {
if (!s.dueDetectors(t).includes("d")) return "skipped";
s.recordGate("d", t, options.pass);
if (!options.pass || !s.shouldParse("d", t)) return "gated";
s.recordParse(
"d",
t,
options.confidence === undefined
? []
: [{ type: options.type ?? "Event", confidence: options.confidence }],
);
return "parsed";
}
test("a failing gate is only re-checked at the search cadence", () => {
const s = make({ searchIntervalS: 1 });
assert.equal(feed(s, 0, { pass: false }), "gated");
assert.equal(feed(s, 0.5, { pass: false }), "skipped");
assert.equal(feed(s, 0.99, { pass: false }), "skipped");
assert.equal(feed(s, 1, { pass: false }), "gated");
});
test("a passing gate drops to the dense refine cadence", () => {
const s = make({ searchIntervalS: 1 });
assert.equal(feed(s, 0, { pass: true, confidence: 0.7 }), "parsed");
assert.equal(feed(s, 0.05, { pass: true, confidence: 0.7 }), "skipped");
assert.equal(feed(s, 0.1, { pass: true, confidence: 0.7 }), "parsed");
assert.equal(feed(s, 0.2, { pass: true, confidence: 0.7 }), "parsed");
});
test("suppresses after stagnant parses on a static screen", () => {
const s = make({});
assert.equal(feed(s, 0.0, { pass: true, confidence: 0.9 }), "parsed");
assert.equal(feed(s, 0.1, { pass: true, confidence: 0.9 }), "parsed");
assert.equal(feed(s, 0.2, { pass: true, confidence: 0.9 }), "parsed");
assert.equal(feed(s, 0.3, { pass: true, confidence: 0.9 }), "parsed");
assert.equal(feed(s, 0.4, { pass: true, confidence: 0.9 }), "gated");
assert.equal(feed(s, 0.5, { pass: true, confidence: 0.9 }), "gated");
});
test("an improving read resets the stagnation counter", () => {
const s = make({});
feed(s, 0.0, { pass: true, confidence: 0.7 });
feed(s, 0.1, { pass: true, confidence: 0.7 });
feed(s, 0.2, { pass: true, confidence: 0.7 });
assert.equal(feed(s, 0.3, { pass: true, confidence: 0.85 }), "parsed");
assert.equal(feed(s, 0.4, { pass: true, confidence: 0.85 }), "parsed");
assert.equal(feed(s, 0.5, { pass: true, confidence: 0.85 }), "parsed");
assert.equal(feed(s, 0.6, { pass: true, confidence: 0.85 }), "parsed");
assert.equal(feed(s, 0.7, { pass: true, confidence: 0.85 }), "gated");
});
test("a gate drop ends the streak and re-enables parsing", () => {
const s = make({});
for (let i = 0; i < 5; i++) feed(s, i * 0.1, { pass: true, confidence: 0.9 });
assert.equal(feed(s, 0.5, { pass: true, confidence: 0.9 }), "gated");
assert.equal(feed(s, 0.6, { pass: false }), "gated");
assert.equal(feed(s, 0.7, { pass: true, confidence: 0.9 }), "parsed");
});
test("gate firing without events stagnates too", () => {
const s = make({});
for (let i = 0; i < 4; i++) {
assert.equal(feed(s, i * 0.1, { pass: true }), "parsed");
}
assert.equal(feed(s, 0.4, { pass: true }), "gated");
});
test("dense no-read parses during an entry animation respect the time floor", () => {
const s = make({}, { stagnantAfterS: 3 });
// the gate fires from t=0 while the screen is still animating in and
// parses read nothing — a parse count alone must not suppress here
for (let i = 0; i < 8; i++) {
assert.equal(feed(s, i * 0.15, { pass: true }), "parsed");
}
// the screen becomes readable and the read still lands
assert.equal(feed(s, 1.35, { pass: true, confidence: 0.88 }), "parsed");
// once truly static past the time floor, suppression still kicks in
for (let i = 0; i < 25; i++) {
feed(s, 1.5 + i * 0.15, { pass: true, confidence: 0.88 });
}
assert.equal(feed(s, 5.4, { pass: true, confidence: 0.88 }), "gated");
});
test("a sufficient read suppresses immediately", () => {
const s = make({ sufficientConfidence: 0.95 });
assert.equal(feed(s, 0.0, { pass: true, confidence: 0.96 }), "parsed");
assert.equal(feed(s, 0.1, { pass: true, confidence: 0.99 }), "gated");
// gate drop = screen changed = fresh streak parses again
feed(s, 0.2, { pass: false });
assert.equal(feed(s, 0.3, { pass: true, confidence: 0.7 }), "parsed");
});
test("rearm cooldown skips parses across gate flicker", () => {
const s = make({ sufficientConfidence: 0.95, rearmCooldownS: 4 });
assert.equal(feed(s, 1.0, { pass: true, confidence: 0.96 }), "parsed");
feed(s, 1.5, { pass: false });
assert.equal(feed(s, 2.0, { pass: true, confidence: 0.9 }), "gated");
assert.equal(feed(s, 4.9, { pass: true, confidence: 0.9 }), "gated");
assert.equal(feed(s, 5.1, { pass: true, confidence: 0.9 }), "parsed");
});
test("checkIntervalS caps both phases and exempts from suppression", () => {
const s = make({ checkIntervalS: 1 });
assert.equal(feed(s, 0.0, { pass: true, confidence: 0.9 }), "parsed");
assert.equal(feed(s, 0.5, { pass: true, confidence: 0.9 }), "skipped");
for (let t = 1; t < 10; t++) {
assert.equal(feed(s, t, { pass: true, confidence: 0.9 }), "parsed");
}
});
test("suppressSteadyFrames=false checks and parses every frame", () => {
const s = make(
{ searchIntervalS: 1, sufficientConfidence: 0.5 },
{ suppressSteadyFrames: false },
);
for (let i = 0; i < 10; i++) {
assert.equal(feed(s, 0, { pass: true, confidence: 0.9 }), "parsed");
}
assert.equal(s.calm(1000), false);
});
test("calm needs a quiet period and no open match", () => {
const s = make({});
assert.equal(feed(s, 0, { pass: false }), "gated");
assert.equal(s.calm(5), false);
assert.equal(s.calm(10), true);
// a gate pass resets the quiet clock
feed(s, 10, { pass: true, confidence: 0.9 });
assert.equal(s.calm(15), false);
assert.equal(s.calm(20), true);
// a confident map-start keeps the scan active for matchOpenMaxS
feed(s, 21, { pass: true, confidence: 0.9, type: "MapStart" });
assert.equal(s.calm(50), false);
assert.equal(s.calm(21 + 60), true);
});
test("a scoreboard closes the match early", () => {
const s = make({});
feed(s, 0, { pass: true, confidence: 0.9, type: "MapStart" });
assert.equal(s.calm(30), false);
feed(s, 31, { pass: true, confidence: 0.9, type: "Scoreboard" });
// quiet period still applies after the closing read
assert.equal(s.calm(35), false);
assert.equal(s.calm(41), true);
});
test("a t jumping backwards resets the session", () => {
const s = make({ searchIntervalS: 1 });
feed(s, 100, { pass: false });
assert.equal(feed(s, 100.5, { pass: false }), "skipped");
// a rescan starts earlier than anything seen — fresh state, due again
assert.equal(feed(s, 0, { pass: false }), "gated");
});
test("detectors are tracked independently", () => {
const s = new DetectorScheduler(
[{ id: "a" }, { id: "b", searchIntervalS: 1 }],
OPTS,
);
assert.deepEqual(s.dueDetectors(0), ["a", "b"]);
s.recordGate("a", 0, true);
s.recordGate("b", 0, false);
assert.deepEqual(s.dueDetectors(0.5), ["a"]);
assert.deepEqual(s.dueDetectors(1), ["a", "b"]);
});
test("nextDueT lets frames skip analysis entirely", () => {
const s = new DetectorScheduler(
[
{ id: "a", searchIntervalS: 0.5 },
{ id: "b", searchIntervalS: 1 },
],
OPTS,
);
assert.equal(s.nextDueT(), Number.NEGATIVE_INFINITY);
s.dueDetectors(0);
s.recordGate("a", 0, false);
s.recordGate("b", 0, false);
assert.equal(s.nextDueT(), 0.5);
});

View File

@@ -1,64 +0,0 @@
/**
* Unit tests for ParseSuppressor: a static screen (gate keeps passing,
* confidence stops improving) gets its parse() suppressed after the
* stagnation budget, and any gate drop resets the streak.
*/
import assert from "node:assert/strict";
import { ParseSuppressor } from "../core/detectors/suppressor";
import test from "./node-test-compat";
const OPTS = { maxStagnantParses: 3, minImprovement: 0.001 };
function feed(
s: ParseSuppressor,
id: string,
confidence: number | null,
): boolean {
const allowed = s.shouldParse(id, true);
if (allowed) s.recordParse(id, confidence === null ? [] : [{ confidence }]);
return allowed;
}
test("suppresses after stagnant parses on a static screen", () => {
const s = new ParseSuppressor(OPTS);
assert.equal(feed(s, "scoreboard", 0.9), true); // sets the baseline
assert.equal(feed(s, "scoreboard", 0.9), true); // stagnant 1
assert.equal(feed(s, "scoreboard", 0.9), true); // stagnant 2
assert.equal(feed(s, "scoreboard", 0.9), true); // stagnant 3 -> suppressed
assert.equal(s.shouldParse("scoreboard", true), false);
assert.equal(s.shouldParse("scoreboard", true), false);
});
test("an improving read resets the stagnation counter", () => {
const s = new ParseSuppressor(OPTS);
feed(s, "scoreboard", 0.7);
feed(s, "scoreboard", 0.7);
feed(s, "scoreboard", 0.7);
assert.equal(feed(s, "scoreboard", 0.85), true); // improvement, counter resets
assert.equal(feed(s, "scoreboard", 0.85), true);
assert.equal(feed(s, "scoreboard", 0.85), true);
assert.equal(feed(s, "scoreboard", 0.85), true); // stagnant 3 -> suppressed
assert.equal(s.shouldParse("scoreboard", true), false);
});
test("a gate drop ends the streak and re-enables parsing", () => {
const s = new ParseSuppressor(OPTS);
for (let i = 0; i < 5; i++) feed(s, "scoreboard", 0.9);
assert.equal(s.shouldParse("scoreboard", true), false);
assert.equal(s.shouldParse("scoreboard", false), false); // screen changed
assert.equal(feed(s, "scoreboard", 0.9), true); // fresh streak parses again
});
test("gate firing without events stagnates too", () => {
const s = new ParseSuppressor(OPTS);
for (let i = 0; i < 4; i++) assert.equal(feed(s, "death", null), true);
assert.equal(s.shouldParse("death", true), false);
});
test("detectors are tracked independently", () => {
const s = new ParseSuppressor(OPTS);
for (let i = 0; i < 5; i++) feed(s, "scoreboard", 0.9);
assert.equal(s.shouldParse("scoreboard", true), false);
assert.equal(s.shouldParse("death", true), true);
});

View File

@@ -1,47 +0,0 @@
/**
* Unit tests for CheckThrottle: a detector with a check interval is checked
* at most once per interval, timeline regressions reset the window, and
* detectors without an interval are never throttled.
*/
import assert from "node:assert/strict";
import { CheckThrottle } from "../core/detectors/throttle";
import test from "./node-test-compat";
test("no interval means every frame is checked", () => {
const throttle = new CheckThrottle();
assert.equal(throttle.shouldCheck("scoreboard", 0, undefined), true);
assert.equal(throttle.shouldCheck("scoreboard", 0.1, undefined), true);
assert.equal(throttle.shouldCheck("scoreboard", 0.2, undefined), true);
});
test("frames inside the interval are skipped", () => {
const throttle = new CheckThrottle();
assert.equal(throttle.shouldCheck("objective", 10, 1), true);
assert.equal(throttle.shouldCheck("objective", 10.5, 1), false);
assert.equal(throttle.shouldCheck("objective", 10.99, 1), false);
assert.equal(throttle.shouldCheck("objective", 11, 1), true);
assert.equal(throttle.shouldCheck("objective", 11.5, 1), false);
});
test("a skipped frame does not push the next check later", () => {
const throttle = new CheckThrottle();
assert.equal(throttle.shouldCheck("objective", 0, 1), true);
assert.equal(throttle.shouldCheck("objective", 0.9, 1), false);
assert.equal(throttle.shouldCheck("objective", 1.1, 1), true);
});
test("a timeline regression resets the window", () => {
const throttle = new CheckThrottle();
assert.equal(throttle.shouldCheck("objective", 100, 1), true);
// new capture session / VoD rescan starts its clock over
assert.equal(throttle.shouldCheck("objective", 0.2, 1), true);
assert.equal(throttle.shouldCheck("objective", 0.6, 1), false);
});
test("detectors are tracked independently", () => {
const throttle = new CheckThrottle();
assert.equal(throttle.shouldCheck("objective", 0, 1), true);
assert.equal(throttle.shouldCheck("objective", 0.5, 1), false);
assert.equal(throttle.shouldCheck("other", 0.5, 1), true);
});

View File

@@ -1,22 +1,68 @@
/**
* AnalyzerWorker: owns OpenCV.js (WASM) and the detector registry.
* The main thread posts ImageBitmaps; results come back as plain JSON.
* AnalyzerWorker: owns OpenCV.js (WASM), the detector registry and a
* DetectorScheduler. Two ways in:
*
* - "frame": the main thread posts one ImageBitmap/VideoFrame at a time
* (live capture, screenshot harness, the VoD seek fallback). Results come
* back per detector, then a "done" carrying the scheduler's calm signal
* and telemetry.
* - "scanChunk": a VoD time slice is demuxed and decoded entirely in the
* worker with mediabunny. The worker owns a contiguous slice, so
* scheduling is exact, frames that no detector is due for skip canvas
* readback entirely, and calm stretches are skimmed by keyframe hops
* instead of decoding every frame — the big VoD speedup, since sequential
* decode is what bounds scan wall-clock time.
*/
import {
ALL_FORMATS,
BlobSource,
EncodedPacketSink,
Input,
type VideoSample,
VideoSampleSink,
} from "mediabunny";
import { loadOpenCV } from "../core/cv";
import { createAllDetectors } from "../core/detectors/registry";
import { ParseSuppressor } from "../core/detectors/suppressor";
import { CheckThrottle } from "../core/detectors/throttle";
import { MAP_START_EVENT_TYPE } from "../core/detectors/map-start/index";
import {
createAllDetectors,
SCOREBOARD_EVENT_TYPES,
} from "../core/detectors/registry";
import { DetectorScheduler } from "../core/detectors/scheduler";
import {
createScanTelemetry,
detectorTelemetry,
} from "../core/detectors/telemetry";
import type { Detector } from "../core/detectors/types";
import { normalizeFrame, toMat } from "../core/image";
import type { AnalyzeRequest, InitRequest, WorkerResponse } from "./protocol";
import type {
AnalyzeRequest,
InitRequest,
ScanChunkRequest,
WorkerRequest,
WorkerResponse,
} from "./protocol";
import { fetchScoreboardResources } from "./resources";
let detectors: Detector<unknown>[] = [];
let suppressor: ParseSuppressor | null = null;
let throttle = new CheckThrottle();
/**
* Widest skim hop: calm footage is sampled at the keyframe cadence, capped
* here so long-GOP recordings still cannot slip a results screen (~10s) or
* a match intro (~7s) between two samples.
*/
const MAX_SKIM_STRIDE_S = 2.5;
const PROGRESS_POST_INTERVAL_MS = 400;
const PREVIEW_POST_INTERVAL_MS = 600;
const PREVIEW_WIDTH = 480;
const PREVIEW_HEIGHT = 270;
function post(message: WorkerResponse): void {
self.postMessage(message);
let detectors: Detector<unknown>[] = [];
let scheduler: DetectorScheduler | null = null;
let telemetry = createScanTelemetry();
let chunkAborted = false;
/** last per-frame t, to reset telemetry when a new session rewinds the clock */
let lastFrameT = Number.NEGATIVE_INFINITY;
function post(message: WorkerResponse, transfer: Transferable[] = []): void {
self.postMessage(message, { transfer });
}
async function init({
@@ -27,20 +73,36 @@ async function init({
await loadOpenCV();
const resources = await fetchScoreboardResources(assetsBaseUrl);
detectors = createAllDetectors(resources);
suppressor = suppressSteadyFrames ? new ParseSuppressor() : null;
throttle = new CheckThrottle();
scheduler = new DetectorScheduler(detectors, {
suppressSteadyFrames,
matchOpeningTypes: [MAP_START_EVENT_TYPE],
matchClosingTypes: SCOREBOARD_EVENT_TYPES,
});
telemetry = createScanTelemetry();
post({ kind: "ready" });
} catch (error) {
post({ kind: "error", message: `init failed: ${String(error)}` });
}
}
async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
/**
* Run the due detectors over one frame; closes `bitmap`. When the scheduler
* has no detector due, the canvas readback and normalize are skipped too.
*/
async function analyzeFrame(
bitmap: ImageBitmap | VideoFrame,
t: number,
): Promise<void> {
const due = scheduler!.dueDetectors(t);
if (due.length === 0) {
bitmap.close();
return;
}
const width = "displayWidth" in bitmap ? bitmap.displayWidth : bitmap.width;
const height =
"displayHeight" in bitmap ? bitmap.displayHeight : bitmap.height;
const canvas = new OffscreenCanvas(width, height);
const ctx = canvas.getContext("2d")!;
const ctx = canvas.getContext("2d", { willReadFrequently: true })!;
ctx.drawImage(bitmap, 0, 0);
bitmap.close();
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
@@ -56,6 +118,7 @@ async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
} finally {
src.delete();
}
telemetry.analyzedFrames++;
// On detection, ship back the exact analyzed pixels (lossless, at capture
// resolution) so the UI never has to re-grab a later frame — encoded at
@@ -66,18 +129,24 @@ async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
try {
for (const detector of detectors) {
if (!throttle.shouldCheck(detector.id, t, detector.checkIntervalS))
continue;
if (!due.includes(detector.id)) continue;
const counters = detectorTelemetry(telemetry, detector.id);
counters.checks++;
const gateStart = performance.now();
const gate = detector.gate(frame);
// interval-limited detectors watch changing content and opt out of
// steady-frame suppression (see Detector.checkIntervalS)
const suppressible =
detector.checkIntervalS === undefined ? suppressor : null;
const runParse = suppressible
? suppressible.shouldParse(detector.id, gate.pass)
: gate.pass;
const events = runParse ? detector.parse(frame, t, gate) : [];
if (runParse) suppressible?.recordParse(detector.id, events);
counters.gateMs += performance.now() - gateStart;
scheduler!.recordGate(detector.id, t, gate.pass);
if (gate.pass) counters.gatePasses++;
const runParse = gate.pass && scheduler!.shouldParse(detector.id, t);
if (gate.pass && !runParse) counters.suppressedParses++;
let events: ReturnType<typeof detector.parse> = [];
if (runParse) {
const parseStart = performance.now();
events = detector.parse(frame, t, gate);
counters.parses++;
counters.parseMs += performance.now() - parseStart;
scheduler!.recordParse(detector.id, t, events);
}
const blob =
events.length > 0 && detector.attachFrame !== false
? await frameBlob()
@@ -91,16 +160,147 @@ async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
frame: blob,
});
}
} catch (error) {
post({ kind: "error", message: `analyze failed: ${String(error)}` });
} finally {
frame.delete();
post({ kind: "done", t });
}
}
async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
if (t + 5 < lastFrameT) telemetry = createScanTelemetry();
lastFrameT = t;
try {
await analyzeFrame(bitmap, t);
} catch (error) {
post({ kind: "error", message: `analyze failed: ${String(error)}` });
}
post({ kind: "done", t, calm: scheduler!.calm(t), telemetry });
}
async function scanChunk({
file,
chunkIndex,
tStart,
tEnd,
}: ScanChunkRequest): Promise<void> {
chunkAborted = false;
scheduler!.reset(tStart);
telemetry = createScanTelemetry();
const wallStart = performance.now();
let lastProgressAt = 0;
let lastPreviewAt = 0;
let cursor = tStart;
let mode: "active" | "skim" = "active";
const input = new Input({
formats: ALL_FORMATS,
source: new BlobSource(file),
});
try {
const track = await input.getPrimaryVideoTrack();
if (!track || !(await track.canDecode())) {
throw new Error("worker cannot decode this file");
}
const samples = new VideoSampleSink(track);
const packets = new EncodedPacketSink(track);
const handleSample = async (sample: VideoSample): Promise<void> => {
telemetry.decodedFrames++;
const t = sample.timestamp;
const span = Math.max(0, t - cursor);
if (mode === "active") telemetry.activeVideoS += span;
else telemetry.skimVideoS += span;
cursor = Math.max(cursor, t);
const frame = sample.toVideoFrame();
sample.close();
const now = performance.now();
let preview: ImageBitmap | undefined;
if (now - lastPreviewAt >= PREVIEW_POST_INTERVAL_MS) {
lastPreviewAt = now;
preview = await createImageBitmap(frame, {
resizeWidth: PREVIEW_WIDTH,
resizeHeight: PREVIEW_HEIGHT,
});
}
if (t >= scheduler!.nextDueT()) {
await analyzeFrame(frame, t);
} else {
frame.close();
}
if (preview || now - lastProgressAt >= PROGRESS_POST_INTERVAL_MS) {
lastProgressAt = now;
telemetry.wallMs = performance.now() - wallStart;
post(
{
kind: "chunkProgress",
chunkIndex,
t: cursor,
mode,
telemetry,
preview,
},
preview ? [preview] : [],
);
}
};
scan: while (!chunkAborted && cursor < tEnd) {
if (mode === "active") {
// dense sequential decode: every frame is seen, the scheduler
// decides which are worth analyzing
for await (const sample of samples.samples(cursor)) {
if (!sample) continue;
if (chunkAborted || sample.timestamp >= tEnd) {
sample.close();
break scan;
}
await handleSample(sample);
if (scheduler!.calm(cursor)) {
mode = "skim";
break;
}
}
if (mode === "active") break; // media ended before tEnd
} else {
// skim: hop keyframe to keyframe (single-frame decodes) while
// calm, capped so long GOPs cannot hide a short screen
const key = await packets.getKeyPacket(cursor + MAX_SKIM_STRIDE_S, {
verifyKeyPackets: true,
});
const target =
key && key.timestamp > cursor
? key.timestamp
: cursor + MAX_SKIM_STRIDE_S;
if (target >= tEnd) {
cursor = tEnd;
break;
}
const sample = await samples.getSample(target);
if (!sample) {
cursor = target;
continue;
}
await handleSample(sample);
cursor = Math.max(cursor, target);
if (!scheduler!.calm(cursor)) mode = "active";
}
}
telemetry.wallMs = performance.now() - wallStart;
post({ kind: "chunkDone", chunkIndex, telemetry });
} catch (error) {
post({
kind: "error",
message: `chunk ${chunkIndex} scan failed: ${String(error)}`,
});
} finally {
input.dispose();
}
}
self.onmessage = (e: MessageEvent) => {
const msg = e.data as { kind: string } & Record<string, unknown>;
if (msg.kind === "init") void init(msg as unknown as InitRequest);
else if (msg.kind === "frame") void analyze(msg as unknown as AnalyzeRequest);
const msg = e.data as WorkerRequest;
if (msg.kind === "init") void init(msg);
else if (msg.kind === "frame") void analyze(msg);
else if (msg.kind === "scanChunk") void scanChunk(msg);
else if (msg.kind === "abortChunk") chunkAborted = true;
};

View File

@@ -1,16 +1,38 @@
/**
* Main-thread wrapper around the AnalyzerWorker: init handshake, one
* in-flight frame at a time (the sampler drops frames while busy). Each
* frame yields one result per registered detector, then a single "done".
* Main-thread wrapper around the AnalyzerWorker: init handshake, then either
* one in-flight frame at a time (live capture / screenshot / seek fallback —
* each frame yields one result per due detector, then a "done" carrying the
* scheduler's calm signal and telemetry) or one in-flight chunk scan (the
* worker decodes and analyzes a VoD time slice by itself, streaming results
* and progress until "chunkDone").
*/
import { Config } from "../../../config";
import type { ScanTelemetry } from "../core/detectors/telemetry";
import type { WorkerResponse } from "./protocol";
export type ResultHandler = (
result: Extract<WorkerResponse, { kind: "result" }>,
) => void;
export type ErrorHandler = (message: string) => void;
export type DoneHandler = (t: number) => void;
export interface DoneInfo {
calm: boolean;
telemetry: ScanTelemetry;
}
export type DoneHandler = (t: number, info: DoneInfo) => void;
export type ChunkProgress = Extract<WorkerResponse, { kind: "chunkProgress" }>;
export type ChunkProgressHandler = (progress: ChunkProgress) => void;
/** each chunk-scanning worker decodes and analyzes on its own; leave a core
* for the main thread and one for the browser's media stack */
export function defaultScanWorkerCount(): number {
return Math.min(4, Math.max(1, (navigator.hardwareConcurrency || 4) - 2));
}
interface PendingChunk {
resolve(telemetry: ScanTelemetry): void;
reject(error: Error): void;
onProgress?: ChunkProgressHandler;
}
export class AnalyzerClient {
#worker: Worker;
@@ -20,6 +42,8 @@ export class AnalyzerClient {
#onError: ErrorHandler;
#onDone: DoneHandler | undefined;
#readyPromise: Promise<void>;
#idleWaiters: (() => void)[] = [];
#chunk: PendingChunk | null = null;
constructor(
onResult: ResultHandler,
@@ -49,23 +73,27 @@ export class AnalyzerClient {
} else if (msg.kind === "result") {
this.#onResult(msg);
} else if (msg.kind === "done") {
this.#busy = false;
this.#onDone?.(msg.t);
this.#settle();
this.#onDone?.(msg.t, { calm: msg.calm, telemetry: msg.telemetry });
} else if (msg.kind === "chunkProgress") {
this.#chunk?.onProgress?.(msg);
} else if (msg.kind === "chunkDone") {
const chunk = this.#chunk;
this.#chunk = null;
this.#settle();
chunk?.resolve(msg.telemetry);
} else if (msg.kind === "error") {
this.#busy = false;
this.#onError(msg.message);
this.#fail(msg.message);
}
};
// A throw outside the worker's own try/catch posts neither "error" nor
// "done"; without these handlers `busy` would stay true forever and the
// sampler / VoD scan would silently freeze.
this.#worker.onerror = (e: ErrorEvent) => {
this.#busy = false;
this.#onError(`worker error: ${e.message || String(e)}`);
this.#fail(`worker error: ${e.message || String(e)}`);
};
this.#worker.onmessageerror = () => {
this.#busy = false;
this.#onError("worker message deserialization failed");
this.#fail("worker message deserialization failed");
};
this.#worker.postMessage({
kind: "init",
@@ -82,6 +110,13 @@ export class AnalyzerClient {
return this.#busy || !this.#ready;
}
/** Resolves once no frame or chunk scan is in flight. Call whenReady() first. */
async whenIdle(): Promise<void> {
while (this.#busy) {
await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
}
}
/** Returns false (and closes the bitmap) if the worker is still busy. */
analyze(bitmap: ImageBitmap | VideoFrame, t: number): boolean {
if (this.busy) {
@@ -93,7 +128,46 @@ export class AnalyzerClient {
return true;
}
/**
* Scan [tStart, tEnd) of `file` inside the worker. Results stream to the
* shared result handler; resolves with the chunk's telemetry once done
* (an aborted chunk resolves too — abort is not an error).
*/
scanChunk(
request: { file: File; chunkIndex: number; tStart: number; tEnd: number },
onProgress?: ChunkProgressHandler,
): Promise<ScanTelemetry> {
if (this.busy) {
return Promise.reject(new Error("analyzer is busy"));
}
this.#busy = true;
return new Promise((resolve, reject) => {
this.#chunk = { resolve, reject, onProgress };
this.#worker.postMessage({ kind: "scanChunk", ...request });
});
}
/** Ask a running chunk scan to stop; it resolves after the current frame. */
abortChunk(): void {
if (this.#chunk) this.#worker.postMessage({ kind: "abortChunk" });
}
dispose(): void {
this.#worker.terminate();
}
#settle(): void {
this.#busy = false;
const waiters = this.#idleWaiters;
this.#idleWaiters = [];
for (const waiter of waiters) waiter();
}
#fail(message: string): void {
const chunk = this.#chunk;
this.#chunk = null;
this.#settle();
if (chunk) chunk.reject(new Error(message));
else this.#onError(message);
}
}

View File

@@ -1,80 +0,0 @@
/**
* AnalyzerPool: several AnalyzerClients so frame analysis parallelizes
* across cores. The VoD scan hands each decoded frame to any idle worker
* and never waits; when all workers are busy the frame is dropped — the
* next one is milliseconds of video away, so coverage stays as dense as
* the machine can analyze. Results arrive out of decode order, which the
* consumer must tolerate (TimelineBuilder does: it merges on |Δt| and
* keeps its list sorted, independent of arrival order).
*/
import {
AnalyzerClient,
type ErrorHandler,
type ResultHandler,
} from "./client";
/** leave cores for the main thread and the video decoder */
export function defaultPoolSize(): number {
return Math.min(4, Math.max(1, (navigator.hardwareConcurrency || 4) - 2));
}
export class AnalyzerPool {
#clients: AnalyzerClient[];
#idleWaiters: (() => void)[] = [];
constructor(size: number, onResult: ResultHandler, onError: ErrorHandler) {
const wake = () => {
const waiters = this.#idleWaiters;
this.#idleWaiters = [];
for (const waiter of waiters) waiter();
};
this.#clients = Array.from(
{ length: size },
() =>
new AnalyzerClient(
onResult,
(message) => {
wake(); // an errored frame is also a finished frame — don't hang whenIdle
onError(message);
},
wake,
),
);
}
whenReady(): Promise<void> {
return Promise.all(this.#clients.map((c) => c.whenReady())).then(() => {});
}
hasIdle(): boolean {
return this.#clients.some((c) => !c.busy);
}
/** Resolves once at least one worker is free. Call whenReady() first. */
async whenAnyIdle(): Promise<void> {
while (!this.hasIdle()) {
await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
}
}
/** Hand the frame to an idle worker; false (and the frame closed) when all are busy. */
tryAnalyze(frame: ImageBitmap | VideoFrame, t: number): boolean {
const idle = this.#clients.find((c) => !c.busy);
if (!idle) {
frame.close();
return false;
}
return idle.analyze(frame, t);
}
/** Resolves once no worker has a frame in flight. Call whenReady() first. */
async whenIdle(): Promise<void> {
while (this.#clients.some((c) => c.busy)) {
await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
}
}
dispose(): void {
for (const client of this.#clients) client.dispose();
}
}

View File

@@ -1,3 +1,4 @@
import type { ScanTelemetry } from "../core/detectors/telemetry";
import type { DetectedEvent, GateResult } from "../core/detectors/types";
export interface InitRequest {
@@ -10,8 +11,9 @@ export interface InitRequest {
assetsBaseUrl: string;
/**
* skip parse() for a detector whose gate keeps firing without confidence
* improving (static screen); default true — one-shot consumers like the
* screenshot harness turn it off
* improving (static screen), and let the scheduler thin out checks;
* default true — one-shot consumers like the screenshot harness turn it
* off to get every detector on every frame
*/
suppressSteadyFrames?: boolean;
}
@@ -25,6 +27,32 @@ export interface AnalyzeRequest {
t: number;
}
/**
* Scan a time slice of a VoD entirely inside the worker: demux + decode with
* mediabunny, schedule detectors, post results as they fire. Decoding in the
* worker removes the per-frame main-thread hop and lets each worker own a
* contiguous slice, so scheduler state (cadence, suppression, calm) is exact
* instead of split across a pool.
*/
export interface ScanChunkRequest {
kind: "scanChunk";
file: File;
chunkIndex: number;
/** seconds; the chunk scans [tStart, tEnd) */
tStart: number;
tEnd: number;
}
export interface AbortChunkRequest {
kind: "abortChunk";
}
export type WorkerRequest =
| InitRequest
| AnalyzeRequest
| ScanChunkRequest
| AbortChunkRequest;
export type WorkerResponse =
| { kind: "ready" }
| {
@@ -36,6 +64,23 @@ export type WorkerResponse =
/** lossless PNG of the exact frame that was analyzed; present when events fired */
frame?: Blob;
}
/** all detectors have reported for frame t */
| { kind: "done"; t: number }
/** all due detectors have reported for frame t (per-frame path only) */
| {
kind: "done";
t: number;
/** scheduler sees dead air — the caller may widen its sampling stride */
calm: boolean;
telemetry: ScanTelemetry;
}
| {
kind: "chunkProgress";
chunkIndex: number;
/** seconds of video the chunk scan has reached */
t: number;
mode: "active" | "skim";
telemetry: ScanTelemetry;
/** small bitmap of the latest decoded frame, for the preview canvas */
preview?: ImageBitmap;
}
| { kind: "chunkDone"; chunkIndex: number; telemetry: ScanTelemetry }
| { kind: "error"; message: string };

View File

@@ -33,6 +33,7 @@
"test:scanner": "vitest run --project scanner",
"scanner:report": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/report.ts",
"scanner:fixtures": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/run-fixtures.ts",
"scanner:replay": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/replay-frames.ts",
"scanner:bootstrap-atlas": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/bootstrap-atlas-from-fixture.ts",
"scanner:build-glyph-atlas": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/build-glyph-atlas.ts",
"scanner:build-localized-entries": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/build-localized-entries.ts",

View File

@@ -0,0 +1,82 @@
/** biome-ignore-all lint/suspicious/noConsole: CLI script output */
/**
* CLI harness: replay a directory of extracted VoD frames through the full
* detector registry driven by a DetectorScheduler, mirroring the analyzer
* worker's chunk scan — to reproduce scheduling-dependent misses offline.
* This is the tool for "the browser scan missed an event that a fixture
* parses fine": extract the surrounding footage with
* ffmpeg -ss <startT> -i vod.mkv -t 30 -vf fps=6 frames/f%04d.png
* then replay it and watch which checks the scheduler ran and what they saw.
*
* Usage: pnpm scanner:replay <framesDir> <startT> <fps>
* (frames are ffmpeg-numbered f0001.png..; t = startT + (n-1)/fps)
*/
import { readdirSync } from "node:fs";
import { join } from "node:path";
import { loadOpenCV } from "../../app/features/scanner/core/cv";
import { MAP_START_EVENT_TYPE } from "../../app/features/scanner/core/detectors/map-start/index";
import {
createAllDetectors,
SCOREBOARD_EVENT_TYPES,
} from "../../app/features/scanner/core/detectors/registry";
import { DetectorScheduler } from "../../app/features/scanner/core/detectors/scheduler";
import { normalizeFrame, toMat } from "../../app/features/scanner/core/image";
import { readImage } from "../../app/features/scanner/node/image-io";
import { loadScoreboardResources } from "../../app/features/scanner/node/resources";
const [framesDir, startTArg, fpsArg] = process.argv.slice(2);
if (!framesDir || !startTArg || !fpsArg) {
console.error("usage: pnpm scanner:replay <framesDir> <startT> <fps>");
process.exit(1);
}
const startT = Number(startTArg);
const fps = Number(fpsArg);
await loadOpenCV();
const detectors = createAllDetectors(await loadScoreboardResources());
const scheduler = new DetectorScheduler(detectors, {
matchOpeningTypes: [MAP_START_EVENT_TYPE],
matchClosingTypes: SCOREBOARD_EVENT_TYPES,
});
scheduler.reset(startT);
const files = readdirSync(framesDir)
.filter((f) => f.endsWith(".png"))
.sort();
for (const [i, file] of files.entries()) {
const t = startT + i / fps;
const due = scheduler.dueDetectors(t);
if (due.length === 0) continue;
const image = await readImage(join(framesDir, file));
const src = toMat(image);
const frame = normalizeFrame(src);
src.delete();
for (const detector of detectors) {
if (!due.includes(detector.id)) continue;
const gate = detector.gate(frame);
scheduler.recordGate(detector.id, t, gate.pass);
if (!gate.pass) {
if (process.env.LOG_GATES?.includes(detector.id)) {
console.log(
`${t.toFixed(2)} ${file} ${detector.id} gate=${gate.score.toFixed(3)} FAIL`,
);
}
continue;
}
if (!scheduler.shouldParse(detector.id, t)) {
console.log(
`${t.toFixed(2)} ${file} ${detector.id} gate=${gate.score.toFixed(3)} PARSE-SUPPRESSED`,
);
continue;
}
const events = detector.parse(frame, t, gate);
scheduler.recordParse(detector.id, t, events);
console.log(
`${t.toFixed(2)} ${file} ${detector.id} gate=${gate.score.toFixed(3)} events=[${events
.map((e) => `${e.type}@${e.confidence.toFixed(3)}`)
.join(", ")}]`,
);
}
frame.delete();
}