mirror of
https://github.com/Sendouc/sendou.ink.git
synced 2026-09-08 04:05:54 -05:00
Optimize
This commit is contained in:
@@ -15,6 +15,7 @@ from the emberz repo; see `MIGRATION.md` there.
|
||||
pnpm test:scanner # golden-file suite over tests/fixtures/ (Vitest, Node)
|
||||
pnpm scanner:report # accuracy table + name character error rate across fixtures
|
||||
pnpm scanner:fixtures [name-substring] # run detectors over matching fixtures, verbose
|
||||
pnpm scanner:replay <dir> <startT> <fps> # replay ffmpeg-extracted frames through the scheduler+detectors
|
||||
pnpm scanner:bootstrap-atlas # harvest labeled fixture crops into the glyph atlases
|
||||
pnpm scanner:build-glyph-atlas # add the font-rendered charset (fonts required, see below)
|
||||
pnpm scanner:build-localized-entries # regen localized closed sets from ../splat3
|
||||
@@ -38,8 +39,8 @@ sequenceDiagram
|
||||
participant UI as Live/VoD tab
|
||||
participant ING as /ingest (scanner-ingest)
|
||||
participant DB as IngestedMatch / IngestedScoreboard
|
||||
Cap->>W: frame + t
|
||||
W->>W: detectors gate() → parse()
|
||||
Cap->>W: frame + t (live/screenshot/seek) — VoD: worker decodes its own slice
|
||||
W->>W: scheduler dueDetectors() → gate() → parse()
|
||||
W-->>TL: DetectedEvents
|
||||
TL-->>UI: deduped timeline (IndexedDB on Live)
|
||||
UI->>MB: buildScannerMatches(events)
|
||||
@@ -91,13 +92,36 @@ sequenceDiagram
|
||||
accuracy-critical matching internals in `core/glyphs.ts` and
|
||||
`core/detectors/scoreboard/weapons.ts` — read those before touching
|
||||
recognition code.
|
||||
- A detector can declare `checkIntervalS` (objective: 1s) to cap how often
|
||||
it is checked at all — the analyzer worker skips gate+parse in between
|
||||
(`core/detectors/throttle.ts`) and exempts it from steady-frame
|
||||
suppression — and `attachFrame: false` to keep continuously-firing events
|
||||
from storing a frame PNG each. In the UI a match's objective reads render
|
||||
as one step-line timeline (`components/ObjectiveTimeline.tsx`) instead of
|
||||
per-event cards.
|
||||
- Scheduling (`core/detectors/scheduler.ts`): per scan session the
|
||||
DetectorScheduler decides which detectors look at a frame. While a gate
|
||||
keeps failing its detector is checked every `searchIntervalS` (default
|
||||
0.25s — produced/casted VoDs cut screens to ~1s with transition flicker
|
||||
inside, so search cadence must not assume raw-gameplay screen lifetimes;
|
||||
gates are ~ms-cheap, so this costs little); once the gate passes it
|
||||
drops to the dense refine cadence for best-read refinement. Suppression ends a refinement streak
|
||||
once it stagnates by parse count AND elapsed time (~3s — the time floor
|
||||
keeps a dense cadence from suppressing during a screen's entry animation
|
||||
before it is readable) or immediately at `sufficientConfidence`; death
|
||||
adds `rearmCooldownS` (safe because it fits inside the Death timeline
|
||||
merge window). `checkIntervalS` (objective: 1s)
|
||||
still hard-caps both phases and exempts from suppression; a detector can
|
||||
also declare `attachFrame: false` to keep continuously-firing events from
|
||||
storing a frame PNG each. In the UI a match's objective reads render as
|
||||
one step-line timeline (`components/ObjectiveTimeline.tsx`) instead of
|
||||
per-event cards. Frames no detector is due for skip canvas readback
|
||||
entirely, and everything is counted in `core/detectors/telemetry.ts`
|
||||
(surfaced in the VoD tab's telemetry panel).
|
||||
- VoD scans (`components/VodPage.tsx`): on the WebCodecs path the file's
|
||||
duration is split into one contiguous slice per worker and each worker
|
||||
demuxes + decodes its slice itself (mediabunny in the worker — no frames
|
||||
cross the main thread). When the scheduler reports calm (no gate pass for
|
||||
a quiet period, no open match — a confident map-start pins the scan
|
||||
active until a scoreboard or timeout), the worker stops sequential decode
|
||||
and skims keyframe-to-keyframe (single-frame decodes, hop capped at 2.5s
|
||||
so short screens can't hide), snapping back to dense decode on any gate
|
||||
pass. Chunks start dense, so slice boundaries are covered. The seek
|
||||
fallback drives one worker and widens its stride over calm footage the
|
||||
same way.
|
||||
- Recognition is language-agnostic: OCR output snaps against every game
|
||||
language at once (`core/localized-entries.ts`, generated) and events carry
|
||||
sendou ids. English display names come from `components/labels.ts`.
|
||||
|
||||
@@ -1,45 +1,27 @@
|
||||
/**
|
||||
* VoD frame extraction: step through a video file yielding (frame, t) as
|
||||
* fast as decoding allows — no real-time playback. The primary path demuxes
|
||||
* the file and decodes **every frame** sequentially with WebCodecs (via
|
||||
* mediabunny), yielding the VideoFrames themselves (transferable to the
|
||||
* analyzer workers with no main-thread conversion); when the container/codec
|
||||
* can't be read that way, it falls back to seek-stepping a <video> element
|
||||
* at a small fixed step, which handles anything the browser can play at the
|
||||
* cost of per-seek latency and frame-exact coverage.
|
||||
* VoD scan entry points. The primary path probes whether WebCodecs (via
|
||||
* mediabunny) can decode the file — if so, the analyzer workers each demux
|
||||
* and decode their own time slice of it (worker/analyzer.worker.ts) and no
|
||||
* frames cross the main thread at all. When the container/codec can't be
|
||||
* read that way, the fallback seek-steps a <video> element, which handles
|
||||
* anything the browser can play at the cost of per-seek latency; its stride
|
||||
* is supplied per step so the caller can widen it over calm footage.
|
||||
*/
|
||||
import { ALL_FORMATS, BlobSource, Input, VideoSampleSink } from "mediabunny";
|
||||
|
||||
/**
|
||||
* Seek fallback step: a <video> element can't enumerate frames, so seek in
|
||||
* increments small enough that anything but blink-and-miss overlays is caught.
|
||||
*/
|
||||
const SEEK_STEP_SECONDS = 0.25;
|
||||
import { ALL_FORMATS, BlobSource, Input } from "mediabunny";
|
||||
|
||||
interface VodFrame {
|
||||
/** the consumer owns the frame and must close() it */
|
||||
frame: ImageBitmap | VideoFrame;
|
||||
frame: ImageBitmap;
|
||||
/** seconds into the video */
|
||||
t: number;
|
||||
}
|
||||
|
||||
export interface VodScan {
|
||||
method: "webcodecs" | "seek";
|
||||
duration: number;
|
||||
frames: AsyncGenerator<VodFrame>;
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a scan over `file`, yielding every decoded frame (or, on the seek
|
||||
* fallback, one frame every SEEK_STEP_SECONDS). `video` must already have
|
||||
* the file loaded (metadata not required yet); it is only driven by the
|
||||
* seek fallback.
|
||||
* Whether mediabunny + WebCodecs can decode `file`, and its duration if so.
|
||||
*/
|
||||
export async function openVodScan(
|
||||
export async function probeWebCodecs(
|
||||
file: File,
|
||||
video: HTMLVideoElement,
|
||||
): Promise<VodScan> {
|
||||
): Promise<{ duration: number } | null> {
|
||||
const input = new Input({
|
||||
formats: ALL_FORMATS,
|
||||
source: new BlobSource(file),
|
||||
@@ -47,52 +29,40 @@ export async function openVodScan(
|
||||
try {
|
||||
const track = await input.getPrimaryVideoTrack();
|
||||
if (track && (await track.canDecode())) {
|
||||
const duration = await input.computeDuration([track]);
|
||||
return {
|
||||
method: "webcodecs",
|
||||
duration,
|
||||
frames: webCodecsFrames(input, new VideoSampleSink(track)),
|
||||
dispose: () => input.dispose(),
|
||||
};
|
||||
return { duration: await input.computeDuration([track]) };
|
||||
}
|
||||
input.dispose();
|
||||
return null;
|
||||
} catch {
|
||||
input.dispose();
|
||||
}
|
||||
|
||||
await loadMetadata(video);
|
||||
if (!Number.isFinite(video.duration)) {
|
||||
throw new Error("video has no known duration — cannot scan by seeking");
|
||||
}
|
||||
return {
|
||||
method: "seek",
|
||||
duration: video.duration,
|
||||
frames: seekFrames(video),
|
||||
dispose: () => {},
|
||||
};
|
||||
}
|
||||
|
||||
async function* webCodecsFrames(
|
||||
input: Input,
|
||||
sink: VideoSampleSink,
|
||||
): AsyncGenerator<VodFrame> {
|
||||
try {
|
||||
for await (const sample of sink.samples()) {
|
||||
if (!sample) continue;
|
||||
const t = sample.timestamp;
|
||||
const frame = sample.toVideoFrame();
|
||||
sample.close();
|
||||
yield { frame, t };
|
||||
}
|
||||
return null;
|
||||
} finally {
|
||||
input.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
async function* seekFrames(video: HTMLVideoElement): AsyncGenerator<VodFrame> {
|
||||
for (let t = 0; t < video.duration; t += SEEK_STEP_SECONDS) {
|
||||
/**
|
||||
* Open a seek-stepping scan over `video` (which must already have the file
|
||||
* loaded; metadata is awaited here). `nextStrideS` is consulted after every
|
||||
* yielded frame, so analysis feedback can adjust the step on the fly.
|
||||
*/
|
||||
export async function openSeekScan(
|
||||
video: HTMLVideoElement,
|
||||
nextStrideS: () => number,
|
||||
): Promise<{ duration: number; frames: AsyncGenerator<VodFrame> }> {
|
||||
await loadMetadata(video);
|
||||
if (!Number.isFinite(video.duration)) {
|
||||
throw new Error("video has no known duration — cannot scan by seeking");
|
||||
}
|
||||
return { duration: video.duration, frames: seekFrames(video, nextStrideS) };
|
||||
}
|
||||
|
||||
async function* seekFrames(
|
||||
video: HTMLVideoElement,
|
||||
nextStrideS: () => number,
|
||||
): AsyncGenerator<VodFrame> {
|
||||
for (let t = 0; t < video.duration; ) {
|
||||
await seekTo(video, t);
|
||||
yield { frame: await createImageBitmap(video), t };
|
||||
t += Math.max(0.01, nextStrideS());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,25 +1,30 @@
|
||||
/**
|
||||
* VoD tab: load a video file and scan it for scoreboard matches as fast as
|
||||
* decoding allows — no real-time playback (see src/capture/vod-frames.ts).
|
||||
* Every frame is decoded and handed to a pool of analyzer workers; decode
|
||||
* never waits on analysis (a frame arriving while all workers are busy is
|
||||
* dropped — the next is ~1/60s away), so the scan runs at decode speed and
|
||||
* analysis coverage stays as dense as the machine keeps up with — dense
|
||||
* enough that even overlays visible for a fraction of a second are seen.
|
||||
* Each match can be opened in the screenshot page with the exact frame
|
||||
* that was analyzed.
|
||||
* decoding allows — no real-time playback. On the primary (WebCodecs) path
|
||||
* the file's duration is split into one contiguous slice per analyzer
|
||||
* worker and each worker demuxes, decodes, schedules and analyzes its slice
|
||||
* by itself (worker/analyzer.worker.ts): no frames cross the main thread,
|
||||
* scheduling state is exact per slice, and calm stretches are skimmed by
|
||||
* keyframe hops instead of decoded frame-by-frame. The seek fallback drives
|
||||
* a <video> element through a single worker, widening its stride over calm
|
||||
* footage. Each match can be opened in the screenshot page with the exact
|
||||
* frame that was analyzed.
|
||||
*
|
||||
* Completed scans are persisted to IndexedDB keyed by file name
|
||||
* (src/store/vods.ts); the default view lists them for reinspection.
|
||||
*/
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { Link } from "react-router";
|
||||
import { openVodScan } from "../capture/vod-frames";
|
||||
import { openSeekScan, probeWebCodecs } from "../capture/vod-frames";
|
||||
import { connectAbilities } from "../core/ability-harvest";
|
||||
import {
|
||||
OBJECTIVE_EVENT_TYPE,
|
||||
type ObjectiveData,
|
||||
} from "../core/detectors/objective/index";
|
||||
import {
|
||||
mergeScanTelemetry,
|
||||
type ScanTelemetry,
|
||||
} from "../core/detectors/telemetry";
|
||||
import type { DetectedEvent } from "../core/detectors/types";
|
||||
import {
|
||||
buildScannerMatches,
|
||||
@@ -36,7 +41,11 @@ import {
|
||||
saveVod,
|
||||
type VodSummary,
|
||||
} from "../store/vods";
|
||||
import { AnalyzerPool, defaultPoolSize } from "../worker/pool";
|
||||
import {
|
||||
AnalyzerClient,
|
||||
type DoneInfo,
|
||||
defaultScanWorkerCount,
|
||||
} from "../worker/client";
|
||||
import { withoutRepeatEvents } from "./dedupe-events";
|
||||
import { EventCard, type GetFrame } from "./EventCard";
|
||||
import { EventsSummary } from "./EventsSummary";
|
||||
@@ -55,16 +64,14 @@ import {
|
||||
import { sendouUpload } from "./sendou-upload";
|
||||
import { thumbnailFromBlob } from "./thumbnail";
|
||||
|
||||
/** seek-fallback stride while the worker reports activity */
|
||||
const SEEK_ACTIVE_STRIDE_S = 0.25;
|
||||
/**
|
||||
* Hard coverage floor: at most this much video may pass between two analyzed
|
||||
* frames. Busy-dropping alone is not enough — around match results every
|
||||
* gate fires and each analyzed frame runs a full parse (+ PNG encode), so
|
||||
* the whole pool can stay busy for hundreds of ms while decode races ahead
|
||||
* whole seconds of video; short screens (the own-results screen shows ~3s)
|
||||
* then fall into the gap. When the budget is spent and no worker is free,
|
||||
* decode waits.
|
||||
* seek-fallback stride over calm footage (nothing detected for a while, no
|
||||
* match open) — small enough that the screens that can start activity from
|
||||
* dead air (results ~10s, match intro ~7s) still get sampled
|
||||
*/
|
||||
const MAX_ANALYSIS_GAP_SECONDS = 0.25;
|
||||
const SEEK_CALM_STRIDE_S = 2.5;
|
||||
|
||||
type Status = "idle" | "scanning" | "done" | "error";
|
||||
|
||||
@@ -108,7 +115,12 @@ export function VodPage({
|
||||
}) {
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const previewRef = useRef<HTMLCanvasElement>(null);
|
||||
const poolRef = useRef<AnalyzerPool | null>(null);
|
||||
const clientsRef = useRef<AnalyzerClient[]>([]);
|
||||
// cancels the in-flight chunk scans of the previous scan, if any
|
||||
const abortScanRef = useRef<(() => void) | null>(null);
|
||||
// seek fallback: latest per-frame done info + the waiter for the next one
|
||||
const doneInfoRef = useRef<DoneInfo | null>(null);
|
||||
const frameDoneRef = useRef<(() => void) | null>(null);
|
||||
const timelineRef = useRef(new TimelineBuilder());
|
||||
// latest gate score from any worker; flushed to state on the UI throttle
|
||||
const gateScoreRef = useRef<number | null>(null);
|
||||
@@ -131,6 +143,7 @@ export function VodPage({
|
||||
const [matches, setMatches] = useState<VodMatch[]>([]);
|
||||
const [vods, setVods] = useState<VodSummary[]>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [telemetry, setTelemetry] = useState<ScanTelemetry | null>(null);
|
||||
const [over, setOver] = useState(false);
|
||||
const [eventsOpen, setEventsOpen] = useState(false);
|
||||
const [resultsSend, setResultsSend] = useState<ResultsSend | null>(null);
|
||||
@@ -207,7 +220,9 @@ export function VodPage({
|
||||
void refreshVods();
|
||||
return () => {
|
||||
abortRef.current.aborted = true;
|
||||
poolRef.current?.dispose();
|
||||
abortScanRef.current?.();
|
||||
for (const client of clientsRef.current) client.dispose();
|
||||
clientsRef.current = [];
|
||||
if (urlRef.current) URL.revokeObjectURL(urlRef.current);
|
||||
};
|
||||
}, [refreshVods]);
|
||||
@@ -215,10 +230,12 @@ export function VodPage({
|
||||
const scan = useCallback(
|
||||
async (file: File) => {
|
||||
abortRef.current.aborted = true;
|
||||
abortScanRef.current?.();
|
||||
const abort = { aborted: false };
|
||||
abortRef.current = abort;
|
||||
|
||||
setError(null);
|
||||
setTelemetry(null);
|
||||
setMatches([]);
|
||||
setResultsSend(null);
|
||||
setEventsOpen(false);
|
||||
@@ -229,7 +246,6 @@ export function VodPage({
|
||||
setSource("scan");
|
||||
setStatus("scanning");
|
||||
|
||||
let dispose = () => {};
|
||||
try {
|
||||
// the element is used by the seek fallback and for post-scan review
|
||||
const video = videoRef.current!;
|
||||
@@ -237,109 +253,83 @@ export function VodPage({
|
||||
urlRef.current = URL.createObjectURL(file);
|
||||
video.src = urlRef.current;
|
||||
|
||||
poolRef.current ??= new AnalyzerPool(
|
||||
defaultPoolSize(),
|
||||
(result) => {
|
||||
gateScoreRef.current = result.gate.score;
|
||||
if (!result.gate.pass) return;
|
||||
for (const event of result.events as DetectedEvent<FixtureData>[]) {
|
||||
const action = timelineRef.current.push(event);
|
||||
if (action.action !== "added" && action.action !== "replaced")
|
||||
continue;
|
||||
const frame = result.frame;
|
||||
sideWorkRef.current.push(
|
||||
(async () => {
|
||||
const thumbnail = frame
|
||||
? await thumbnailFromBlob(frame)
|
||||
: undefined;
|
||||
const replaced =
|
||||
action.action === "replaced"
|
||||
? matchesRef.current.find(
|
||||
(m) => m.event === action.replaced,
|
||||
)
|
||||
: undefined;
|
||||
const next = matchesRef.current.filter((m) => m !== replaced);
|
||||
next.push({
|
||||
event,
|
||||
key: replaced?.key ?? nextMatchKeyRef.current++,
|
||||
thumbnail,
|
||||
frame,
|
||||
});
|
||||
next.sort((a, b) => a.event.t - b.event.t);
|
||||
matchesRef.current = next;
|
||||
setMatches(next);
|
||||
})().catch(() => {}),
|
||||
);
|
||||
}
|
||||
},
|
||||
(message) => {
|
||||
setError(message);
|
||||
setStatus("error");
|
||||
},
|
||||
);
|
||||
const pool = poolRef.current;
|
||||
await pool.whenReady();
|
||||
// let frames still in flight from an aborted scan finish before the
|
||||
// timeline resets, so their results can't bleed into this scan
|
||||
await pool.whenIdle();
|
||||
if (clientsRef.current.length === 0) {
|
||||
clientsRef.current = Array.from(
|
||||
{ length: defaultScanWorkerCount() },
|
||||
() =>
|
||||
new AnalyzerClient(
|
||||
(result) => {
|
||||
gateScoreRef.current = result.gate.score;
|
||||
if (!result.gate.pass) return;
|
||||
for (const event of result.events as DetectedEvent<FixtureData>[]) {
|
||||
const action = timelineRef.current.push(event);
|
||||
if (
|
||||
action.action !== "added" &&
|
||||
action.action !== "replaced"
|
||||
)
|
||||
continue;
|
||||
const frame = result.frame;
|
||||
sideWorkRef.current.push(
|
||||
(async () => {
|
||||
const thumbnail = frame
|
||||
? await thumbnailFromBlob(frame)
|
||||
: undefined;
|
||||
const replaced =
|
||||
action.action === "replaced"
|
||||
? matchesRef.current.find(
|
||||
(m) => m.event === action.replaced,
|
||||
)
|
||||
: undefined;
|
||||
const next = matchesRef.current.filter(
|
||||
(m) => m !== replaced,
|
||||
);
|
||||
next.push({
|
||||
event,
|
||||
key: replaced?.key ?? nextMatchKeyRef.current++,
|
||||
thumbnail,
|
||||
frame,
|
||||
});
|
||||
next.sort((a, b) => a.event.t - b.event.t);
|
||||
matchesRef.current = next;
|
||||
setMatches(next);
|
||||
})().catch(() => {}),
|
||||
);
|
||||
}
|
||||
},
|
||||
(message) => {
|
||||
frameDoneRef.current?.();
|
||||
frameDoneRef.current = null;
|
||||
setError(message);
|
||||
setStatus("error");
|
||||
},
|
||||
(_t, info: DoneInfo) => {
|
||||
doneInfoRef.current = info;
|
||||
frameDoneRef.current?.();
|
||||
frameDoneRef.current = null;
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
const clients = clientsRef.current;
|
||||
await Promise.all(clients.map((c) => c.whenReady()));
|
||||
// let work still in flight from an aborted scan finish before the
|
||||
// timeline resets, so its results can't bleed into this scan
|
||||
await Promise.all(clients.map((c) => c.whenIdle()));
|
||||
if (abort.aborted) return;
|
||||
matchesRef.current = [];
|
||||
sideWorkRef.current = [];
|
||||
timelineRef.current = new TimelineBuilder();
|
||||
|
||||
const vod = await openVodScan(file, video);
|
||||
dispose = () => vod.dispose();
|
||||
if (abort.aborted) return;
|
||||
setMethod(vod.method);
|
||||
|
||||
const started = performance.now();
|
||||
// each frame goes to an idle worker; with none free it is dropped
|
||||
// unless MAX_ANALYSIS_GAP_SECONDS of video has passed unanalyzed,
|
||||
// in which case decode waits for a worker. Preview/progress
|
||||
// re-renders are throttled off the hot loop (the preview draw must
|
||||
// precede tryAnalyze — transferring the frame to a worker detaches it).
|
||||
let lastUiUpdate = Number.NEGATIVE_INFINITY;
|
||||
let lastAnalyzedT = Number.NEGATIVE_INFINITY;
|
||||
for await (const { frame, t } of vod.frames) {
|
||||
if (abort.aborted) {
|
||||
frame.close();
|
||||
break;
|
||||
}
|
||||
const now = performance.now();
|
||||
if (now - lastUiUpdate >= 250) {
|
||||
lastUiUpdate = now;
|
||||
drawPreview(previewRef.current, frame);
|
||||
setGateScore(gateScoreRef.current);
|
||||
const elapsed = (now - started) / 1000;
|
||||
setProgress({
|
||||
t,
|
||||
duration: vod.duration,
|
||||
rate: elapsed > 0 ? t / elapsed : 0,
|
||||
});
|
||||
}
|
||||
if (!pool.hasIdle()) {
|
||||
if (t - lastAnalyzedT < MAX_ANALYSIS_GAP_SECONDS) {
|
||||
frame.close();
|
||||
continue;
|
||||
}
|
||||
await pool.whenAnyIdle();
|
||||
if (abort.aborted) {
|
||||
frame.close();
|
||||
break;
|
||||
}
|
||||
}
|
||||
pool.tryAnalyze(frame, t);
|
||||
lastAnalyzedT = t;
|
||||
}
|
||||
await pool.whenIdle();
|
||||
await Promise.all(sideWorkRef.current);
|
||||
if (!abort.aborted) {
|
||||
const finalize = async (duration: number) => {
|
||||
await Promise.all(sideWorkRef.current);
|
||||
if (abort.aborted) return;
|
||||
matchesRef.current = withoutInvalidObjectives(matchesRef.current);
|
||||
setMatches(matchesRef.current);
|
||||
setProgress((p) => (p ? { ...p, t: vod.duration } : p));
|
||||
setProgress((p) => (p ? { ...p, t: duration } : p));
|
||||
setStatus("done");
|
||||
await saveVod(
|
||||
{ name: file.name, savedAt: Date.now(), duration: vod.duration },
|
||||
{ name: file.name, savedAt: Date.now(), duration },
|
||||
matchesRef.current.map((m) => ({
|
||||
type: m.event.type,
|
||||
t: m.event.t,
|
||||
@@ -350,14 +340,126 @@ export function VodPage({
|
||||
})),
|
||||
);
|
||||
await refreshVods();
|
||||
};
|
||||
|
||||
const probe = await probeWebCodecs(file);
|
||||
if (abort.aborted) return;
|
||||
|
||||
if (probe) {
|
||||
// each worker demuxes, decodes and analyzes its own slice of
|
||||
// the file; the main thread only aggregates progress
|
||||
setMethod("webcodecs");
|
||||
const { duration } = probe;
|
||||
const chunkSpan = duration / clients.length;
|
||||
const chunks = clients.map((client, i) => ({
|
||||
client,
|
||||
tStart: i * chunkSpan,
|
||||
tEnd: i === clients.length - 1 ? duration : (i + 1) * chunkSpan,
|
||||
t: i * chunkSpan,
|
||||
done: false,
|
||||
telemetry: null as ScanTelemetry | null,
|
||||
}));
|
||||
abortScanRef.current = () => {
|
||||
for (const client of clients) client.abortChunk();
|
||||
};
|
||||
const mergedTelemetry = () =>
|
||||
mergeScanTelemetry(
|
||||
chunks.flatMap((c) => (c.telemetry ? [c.telemetry] : [])),
|
||||
);
|
||||
let lastUiUpdate = Number.NEGATIVE_INFINITY;
|
||||
const pushUiUpdate = () => {
|
||||
const now = performance.now();
|
||||
if (now - lastUiUpdate < 250) return;
|
||||
lastUiUpdate = now;
|
||||
const covered = chunks.reduce(
|
||||
(sum, c) => sum + (Math.min(c.t, c.tEnd) - c.tStart),
|
||||
0,
|
||||
);
|
||||
const elapsed = (now - started) / 1000;
|
||||
setGateScore(gateScoreRef.current);
|
||||
setProgress({
|
||||
t: covered,
|
||||
duration,
|
||||
rate: elapsed > 0 ? covered / elapsed : 0,
|
||||
});
|
||||
setTelemetry(mergedTelemetry());
|
||||
};
|
||||
await Promise.all(
|
||||
chunks.map((chunk, chunkIndex) =>
|
||||
chunk.client
|
||||
.scanChunk(
|
||||
{ file, chunkIndex, tStart: chunk.tStart, tEnd: chunk.tEnd },
|
||||
(progress) => {
|
||||
chunk.t = progress.t;
|
||||
chunk.telemetry = progress.telemetry;
|
||||
if (progress.preview) {
|
||||
// show one chunk at a time: the earliest still running
|
||||
if (chunks.find((c) => !c.done) === chunk) {
|
||||
drawPreview(previewRef.current, progress.preview);
|
||||
}
|
||||
progress.preview.close();
|
||||
}
|
||||
pushUiUpdate();
|
||||
},
|
||||
)
|
||||
.then((chunkTelemetry) => {
|
||||
chunk.done = true;
|
||||
chunk.t = chunk.tEnd;
|
||||
chunk.telemetry = chunkTelemetry;
|
||||
}),
|
||||
),
|
||||
);
|
||||
if (abort.aborted) return;
|
||||
setTelemetry(mergedTelemetry());
|
||||
await finalize(duration);
|
||||
return;
|
||||
}
|
||||
|
||||
// seek fallback: one worker, one frame in flight; the worker's calm
|
||||
// signal widens the stride over dead air
|
||||
setMethod("seek");
|
||||
const strideRef = { current: SEEK_ACTIVE_STRIDE_S };
|
||||
const vod = await openSeekScan(video, () => strideRef.current);
|
||||
if (abort.aborted) return;
|
||||
const client = clients[0]!;
|
||||
let lastUiUpdate = Number.NEGATIVE_INFINITY;
|
||||
for await (const { frame, t } of vod.frames) {
|
||||
if (abort.aborted) {
|
||||
frame.close();
|
||||
break;
|
||||
}
|
||||
const now = performance.now();
|
||||
if (now - lastUiUpdate >= 250) {
|
||||
lastUiUpdate = now;
|
||||
// the preview draw must precede analyze — transferring the
|
||||
// frame to the worker detaches it
|
||||
drawPreview(previewRef.current, frame);
|
||||
setGateScore(gateScoreRef.current);
|
||||
const elapsed = (now - started) / 1000;
|
||||
setProgress({
|
||||
t,
|
||||
duration: vod.duration,
|
||||
rate: elapsed > 0 ? t / elapsed : 0,
|
||||
});
|
||||
if (doneInfoRef.current)
|
||||
setTelemetry(doneInfoRef.current.telemetry);
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
frameDoneRef.current = resolve;
|
||||
if (!client.analyze(frame, t)) resolve();
|
||||
});
|
||||
strideRef.current = doneInfoRef.current?.calm
|
||||
? SEEK_CALM_STRIDE_S
|
||||
: SEEK_ACTIVE_STRIDE_S;
|
||||
}
|
||||
if (doneInfoRef.current) setTelemetry(doneInfoRef.current.telemetry);
|
||||
await finalize(vod.duration);
|
||||
} catch (e) {
|
||||
if (!abort.aborted) {
|
||||
abortScanRef.current?.();
|
||||
setError(String(e));
|
||||
setStatus("error");
|
||||
}
|
||||
} finally {
|
||||
dispose();
|
||||
}
|
||||
},
|
||||
[refreshVods],
|
||||
@@ -365,6 +467,8 @@ export function VodPage({
|
||||
|
||||
const openStored = useCallback(async (name: string) => {
|
||||
abortRef.current.aborted = true;
|
||||
abortScanRef.current?.();
|
||||
setTelemetry(null);
|
||||
try {
|
||||
const events = await loadVodEvents(name);
|
||||
// VoDs saved before objective reads were mode-gated may carry them
|
||||
@@ -408,6 +512,8 @@ export function VodPage({
|
||||
|
||||
const backToList = useCallback(() => {
|
||||
abortRef.current.aborted = true;
|
||||
abortScanRef.current?.();
|
||||
setTelemetry(null);
|
||||
matchesRef.current = [];
|
||||
setMatches([]);
|
||||
setResultsSend(null);
|
||||
@@ -534,6 +640,9 @@ export function VodPage({
|
||||
)}
|
||||
</div>
|
||||
{error && <p className="error">{error}</p>}
|
||||
{showVodView && telemetry ? (
|
||||
<TelemetryPanel telemetry={telemetry} />
|
||||
) : null}
|
||||
{!showVodView && (
|
||||
<div className="vod-list">
|
||||
{vods.length === 0 && (
|
||||
@@ -689,6 +798,51 @@ function frameLoader(m: VodMatch): GetFrame | undefined {
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function TelemetryPanel({ telemetry }: { telemetry: ScanTelemetry }) {
|
||||
const detectors = Object.entries(telemetry.detectors).sort(([a], [b]) =>
|
||||
a.localeCompare(b),
|
||||
);
|
||||
const coveredS = telemetry.activeVideoS + telemetry.skimVideoS;
|
||||
return (
|
||||
<details className="telemetry">
|
||||
<summary>
|
||||
telemetry · analyzed {telemetry.analyzedFrames}/
|
||||
{telemetry.decodedFrames} decoded frames
|
||||
{coveredS > 0 &&
|
||||
` · skimmed ${formatTime(telemetry.skimVideoS)} of ${formatTime(coveredS)}`}
|
||||
{telemetry.wallMs > 0 &&
|
||||
` · ${formatTime(telemetry.wallMs / 1000)} cpu`}
|
||||
</summary>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>detector</th>
|
||||
<th>checks</th>
|
||||
<th>gate pass</th>
|
||||
<th>gate ms</th>
|
||||
<th>parses</th>
|
||||
<th>parse ms</th>
|
||||
<th>suppressed</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{detectors.map(([id, d]) => (
|
||||
<tr key={id}>
|
||||
<td>{id}</td>
|
||||
<td>{d.checks}</td>
|
||||
<td>{d.gatePasses}</td>
|
||||
<td>{Math.round(d.gateMs)}</td>
|
||||
<td>{d.parses}</td>
|
||||
<td>{Math.round(d.parseMs)}</td>
|
||||
<td>{d.suppressedParses}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
||||
function drawPreview(
|
||||
canvas: HTMLCanvasElement | null,
|
||||
frame: ImageBitmap | VideoFrame,
|
||||
|
||||
@@ -1066,6 +1066,33 @@ html.light {
|
||||
padding: 10px 12px;
|
||||
}
|
||||
|
||||
.scanner-app .telemetry {
|
||||
margin: 8px 0;
|
||||
font-size: var(--font-2xs);
|
||||
color: var(--color-text-high);
|
||||
}
|
||||
|
||||
.scanner-app .telemetry summary {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.scanner-app .telemetry table {
|
||||
margin-top: 6px;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
.scanner-app .telemetry table th,
|
||||
.scanner-app .telemetry table td {
|
||||
padding: 2px 10px 2px 0;
|
||||
text-align: right;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.scanner-app .telemetry table th:first-child,
|
||||
.scanner-app .telemetry table td:first-child {
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.scanner-app .match-card,
|
||||
.scanner-app .match-card.sending::after,
|
||||
|
||||
@@ -820,5 +820,14 @@ export function createDeathDetector(
|
||||
];
|
||||
}
|
||||
|
||||
return { id: "death", gate, parse };
|
||||
// the death cam's animated background flickers the gate; after a
|
||||
// sufficient read the 4s rearm hold stays inside the timeline's 8s
|
||||
// Death merge window, so every parse it skips would merge anyway
|
||||
return {
|
||||
id: "death",
|
||||
sufficientConfidence: 0.98,
|
||||
rearmCooldownS: 4,
|
||||
gate,
|
||||
parse,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -365,5 +365,10 @@ export function createMapStartDetector(
|
||||
];
|
||||
}
|
||||
|
||||
return { id: "map-start", gate, parse };
|
||||
return {
|
||||
id: "map-start",
|
||||
sufficientConfidence: 0.98,
|
||||
gate,
|
||||
parse,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -639,5 +639,10 @@ export function createMinimapDetector(
|
||||
];
|
||||
}
|
||||
|
||||
return { id: "minimap", gate, parse };
|
||||
return {
|
||||
id: "minimap",
|
||||
sufficientConfidence: 0.98,
|
||||
gate,
|
||||
parse,
|
||||
};
|
||||
}
|
||||
|
||||
311
app/features/scanner/core/detectors/scheduler.ts
Normal file
311
app/features/scanner/core/detectors/scheduler.ts
Normal file
@@ -0,0 +1,311 @@
|
||||
/**
|
||||
* DetectorScheduler: one temporally-ordered scan session's answer to "which
|
||||
* detectors should look at the frame at time t, and which of those should
|
||||
* pay for a parse?". It folds together three ideas:
|
||||
*
|
||||
* - Cadence. While a detector's gate keeps failing it is checked every
|
||||
* `searchIntervalS` (default 0.25s — produced VoDs cut screens to as
|
||||
* little as ~1s with transition flicker inside that, so the search
|
||||
* cadence must not assume raw-gameplay screen lifetimes; gates are
|
||||
* ~1.6ms, so searching at the analysis floor costs nothing). Once the
|
||||
* gate passes it drops to the dense `refineIntervalS` so the best-read
|
||||
* refinement loop still sees every frame it wants. `checkIntervalS`
|
||||
* (objective counter) remains a hard cap on both phases and exempts the
|
||||
* detector from suppression, as before. `nextDueT()` lets the caller
|
||||
* skip a frame's canvas readback + normalize entirely when no detector
|
||||
* is due.
|
||||
*
|
||||
* - Suppression. A static screen that keeps a gate firing stops paying for
|
||||
* parses once a streak stagnates: at least `maxStagnantParses`
|
||||
* consecutive parses AND `stagnantAfterS` seconds without the best
|
||||
* confidence improving. The time floor matters — a gate often fires
|
||||
* during a screen's entry animation while parses still read nothing, and
|
||||
* at a dense sampling cadence a pure parse count would suppress before
|
||||
* the screen ever becomes readable (with the gate held high, it would
|
||||
* then never re-arm). A parse reaching the detector's
|
||||
* `sufficientConfidence` suppresses immediately: the timeline keeps the
|
||||
* best read per merge window, so once a read is that good further parses
|
||||
* cannot change the outcome. A
|
||||
* detector with `rearmCooldownS` (death: the overlay's animated
|
||||
* background makes its gate flicker) additionally skips parses for that
|
||||
* long after a sufficient read, across gate drops; safe only where the
|
||||
* timeline merges purely on time and the cooldown fits inside the merge
|
||||
* window.
|
||||
*
|
||||
* - Activity. The scheduler tracks when any gate last passed and whether a
|
||||
* match is open (a confident map-start without a closing scoreboard), so
|
||||
* the VoD chunk scanner can tell dead air ("calm") from footage that
|
||||
* deserves dense decoding.
|
||||
*
|
||||
* All state is keyed to the scan's own clock; a `t` jumping backwards means
|
||||
* a new capture session or rescan and resets everything.
|
||||
*/
|
||||
|
||||
import type { DetectedEvent } from "./types";
|
||||
|
||||
export interface SchedulingInfo {
|
||||
id: string;
|
||||
checkIntervalS?: number;
|
||||
searchIntervalS?: number;
|
||||
sufficientConfidence?: number;
|
||||
rearmCooldownS?: number;
|
||||
}
|
||||
|
||||
export interface SchedulerOptions {
|
||||
/**
|
||||
* false = one-shot harness mode: every detector is due on every frame and
|
||||
* parses are never suppressed
|
||||
*/
|
||||
suppressSteadyFrames: boolean;
|
||||
/** check cadence while a detector's gate is passing */
|
||||
refineIntervalS: number;
|
||||
/** check cadence while a detector's gate is failing (per-detector
|
||||
* searchIntervalS overrides) */
|
||||
searchIntervalS: number;
|
||||
/** consecutive non-improving parses tolerated before suppression */
|
||||
maxStagnantParses: number;
|
||||
/** seconds without improvement tolerated before suppression */
|
||||
stagnantAfterS: number;
|
||||
/** minimum confidence gain that counts as an improvement */
|
||||
minImprovement: number;
|
||||
/** seconds without any gate pass before the scan counts as calm */
|
||||
quietAfterS: number;
|
||||
/** a match opened this long ago without closing is assumed abandoned */
|
||||
matchOpenMaxS: number;
|
||||
/** event types that open a match (map-start) */
|
||||
matchOpeningTypes: readonly string[];
|
||||
/** event types that close a match (the scoreboard family) */
|
||||
matchClosingTypes: readonly string[];
|
||||
}
|
||||
|
||||
const DEFAULT_SCHEDULER_OPTIONS: SchedulerOptions = {
|
||||
suppressSteadyFrames: true,
|
||||
refineIntervalS: 0.15,
|
||||
searchIntervalS: 0.25,
|
||||
maxStagnantParses: 6,
|
||||
// give a screen ~3s to animate in and produce its best read, whatever
|
||||
// the sampling cadence
|
||||
stagnantAfterS: 3,
|
||||
minImprovement: 0.001,
|
||||
quietAfterS: 15,
|
||||
matchOpenMaxS: 8 * 60,
|
||||
matchOpeningTypes: [],
|
||||
matchClosingTypes: [],
|
||||
};
|
||||
|
||||
/** events below this are too dubious to drive match-open/close state */
|
||||
const MATCH_STATE_MIN_CONFIDENCE = 0.6;
|
||||
|
||||
/** tolerated backwards jitter in t before the session counts as restarted */
|
||||
const RESET_TOLERANCE_S = 5;
|
||||
|
||||
const INTERVAL_EPSILON_S = 1e-6;
|
||||
|
||||
interface StreakState {
|
||||
best: number;
|
||||
stagnant: number;
|
||||
lastImprovementT: number;
|
||||
suppressed: boolean;
|
||||
}
|
||||
|
||||
interface DetectorState {
|
||||
info: SchedulingInfo;
|
||||
lastCheckT: number | undefined;
|
||||
gatePassing: boolean;
|
||||
streak: StreakState | null;
|
||||
/** parses skipped until this t after a sufficient read (rearmCooldownS) */
|
||||
parseHoldUntilT: number;
|
||||
}
|
||||
|
||||
export class DetectorScheduler {
|
||||
#options: SchedulerOptions;
|
||||
#states = new Map<string, DetectorState>();
|
||||
#maxT = Number.NEGATIVE_INFINITY;
|
||||
#lastActivityT = Number.NEGATIVE_INFINITY;
|
||||
#matchOpenUntilT = Number.NEGATIVE_INFINITY;
|
||||
|
||||
constructor(
|
||||
detectors: readonly SchedulingInfo[],
|
||||
options: Partial<SchedulerOptions> = {},
|
||||
) {
|
||||
this.#options = { ...DEFAULT_SCHEDULER_OPTIONS, ...options };
|
||||
for (const info of detectors) {
|
||||
this.#states.set(info.id, freshState(info));
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop all session state; `t` seeds the activity clock (chunk start). */
|
||||
reset(t = Number.NEGATIVE_INFINITY): void {
|
||||
for (const [id, state] of this.#states) {
|
||||
this.#states.set(id, freshState(state.info));
|
||||
}
|
||||
this.#maxT = t;
|
||||
this.#lastActivityT = t;
|
||||
this.#matchOpenUntilT = Number.NEGATIVE_INFINITY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Earliest t at which any detector wants a check — frames before it can
|
||||
* skip analysis (and its canvas readback) entirely.
|
||||
*/
|
||||
nextDueT(): number {
|
||||
let next = Number.POSITIVE_INFINITY;
|
||||
for (const state of this.#states.values()) {
|
||||
if (state.lastCheckT === undefined) return Number.NEGATIVE_INFINITY;
|
||||
next = Math.min(next, state.lastCheckT + this.#interval(state));
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
/** Detector ids that should gate the frame at `t`. */
|
||||
dueDetectors(t: number): string[] {
|
||||
if (t + RESET_TOLERANCE_S < this.#maxT) this.reset(t);
|
||||
// the first frame seeds the activity clock so a fresh session is never
|
||||
// instantly calm — it has to earn its quiet period first
|
||||
if (this.#lastActivityT === Number.NEGATIVE_INFINITY) {
|
||||
this.#lastActivityT = t;
|
||||
}
|
||||
this.#maxT = Math.max(this.#maxT, t);
|
||||
const due: string[] = [];
|
||||
for (const [id, state] of this.#states) {
|
||||
if (state.lastCheckT === undefined) {
|
||||
due.push(id);
|
||||
continue;
|
||||
}
|
||||
if (t - state.lastCheckT >= this.#interval(state) - INTERVAL_EPSILON_S) {
|
||||
due.push(id);
|
||||
}
|
||||
}
|
||||
return due;
|
||||
}
|
||||
|
||||
/** Report a gate outcome for a detector this scheduler marked due. */
|
||||
recordGate(id: string, t: number, pass: boolean): void {
|
||||
const state = this.#states.get(id);
|
||||
if (!state) return;
|
||||
state.lastCheckT = t;
|
||||
state.gatePassing = pass;
|
||||
if (pass) {
|
||||
this.#lastActivityT = Math.max(this.#lastActivityT, t);
|
||||
} else {
|
||||
state.streak = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the (passed) gate should be followed by a parse at `t`. */
|
||||
shouldParse(id: string, t: number): boolean {
|
||||
if (!this.#options.suppressSteadyFrames) return true;
|
||||
const state = this.#states.get(id);
|
||||
if (!state) return true;
|
||||
if (state.info.checkIntervalS !== undefined) return true;
|
||||
if (t < state.parseHoldUntilT) return false;
|
||||
return !state.streak?.suppressed;
|
||||
}
|
||||
|
||||
/** Report the outcome of a parse this scheduler approved. */
|
||||
recordParse(
|
||||
id: string,
|
||||
t: number,
|
||||
events: readonly Pick<DetectedEvent, "type" | "confidence">[],
|
||||
): void {
|
||||
this.#recordMatchState(t, events);
|
||||
const state = this.#states.get(id);
|
||||
if (!state || state.info.checkIntervalS !== undefined) return;
|
||||
// no events counts as confidence 0: a false-firing gate on a static
|
||||
// screen stagnates and gets suppressed just like a parsed one
|
||||
const confidence = events.reduce(
|
||||
(max, e) => Math.max(max, e.confidence),
|
||||
0,
|
||||
);
|
||||
const { sufficientConfidence, rearmCooldownS } = state.info;
|
||||
if (
|
||||
sufficientConfidence !== undefined &&
|
||||
confidence >= sufficientConfidence
|
||||
) {
|
||||
state.streak = {
|
||||
best: confidence,
|
||||
stagnant: 0,
|
||||
lastImprovementT: t,
|
||||
suppressed: true,
|
||||
};
|
||||
if (rearmCooldownS !== undefined) {
|
||||
state.parseHoldUntilT = t + rearmCooldownS;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!state.streak) {
|
||||
state.streak = {
|
||||
best: confidence,
|
||||
stagnant: 0,
|
||||
lastImprovementT: t,
|
||||
suppressed: false,
|
||||
};
|
||||
return;
|
||||
}
|
||||
const streak = state.streak;
|
||||
if (confidence > streak.best + this.#options.minImprovement) {
|
||||
streak.best = confidence;
|
||||
streak.stagnant = 0;
|
||||
streak.lastImprovementT = t;
|
||||
return;
|
||||
}
|
||||
streak.stagnant += 1;
|
||||
if (
|
||||
streak.stagnant >= this.#options.maxStagnantParses &&
|
||||
t - streak.lastImprovementT >= this.#options.stagnantAfterS
|
||||
) {
|
||||
streak.suppressed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the footage at `t` is dead air as far as detection goes: no
|
||||
* gate has passed for quietAfterS and no match is open — the VoD scanner
|
||||
* may skim by keyframes instead of decoding densely.
|
||||
*/
|
||||
calm(t: number): boolean {
|
||||
if (!this.#options.suppressSteadyFrames) return false;
|
||||
return (
|
||||
t - this.#lastActivityT >= this.#options.quietAfterS &&
|
||||
t >= this.#matchOpenUntilT
|
||||
);
|
||||
}
|
||||
|
||||
#interval(state: DetectorState): number {
|
||||
if (!this.#options.suppressSteadyFrames) return 0;
|
||||
const { info } = state;
|
||||
if (info.checkIntervalS !== undefined) return info.checkIntervalS;
|
||||
const search = info.searchIntervalS ?? this.#options.searchIntervalS;
|
||||
// while suppressed only the gate keeps running (to spot the screen
|
||||
// changing), which the sparser search cadence covers
|
||||
if (state.streak?.suppressed) return search;
|
||||
return state.gatePassing ? this.#options.refineIntervalS : search;
|
||||
}
|
||||
|
||||
#recordMatchState(
|
||||
t: number,
|
||||
events: readonly Pick<DetectedEvent, "type" | "confidence">[],
|
||||
): void {
|
||||
for (const event of events) {
|
||||
if (event.confidence < MATCH_STATE_MIN_CONFIDENCE) continue;
|
||||
if (this.#options.matchOpeningTypes.includes(event.type)) {
|
||||
this.#matchOpenUntilT = Math.max(
|
||||
this.#matchOpenUntilT,
|
||||
t + this.#options.matchOpenMaxS,
|
||||
);
|
||||
} else if (this.#options.matchClosingTypes.includes(event.type)) {
|
||||
this.#matchOpenUntilT = Math.min(this.#matchOpenUntilT, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function freshState(info: SchedulingInfo): DetectorState {
|
||||
return {
|
||||
info,
|
||||
lastCheckT: undefined,
|
||||
gatePassing: false,
|
||||
streak: null,
|
||||
parseHoldUntilT: Number.NEGATIVE_INFINITY,
|
||||
};
|
||||
}
|
||||
@@ -256,5 +256,10 @@ export function createScoreboardOwnDetector(
|
||||
];
|
||||
}
|
||||
|
||||
return { id: "scoreboard-own", gate, parse };
|
||||
return {
|
||||
id: "scoreboard-own",
|
||||
sufficientConfidence: 0.98,
|
||||
gate,
|
||||
parse,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -426,5 +426,12 @@ export function createScoreboardReplayDetector(
|
||||
];
|
||||
}
|
||||
|
||||
return { id: "scoreboard-replay", gate, parse };
|
||||
// no rearm cooldown — distinct replays browsed in quick succession are
|
||||
// told apart by content
|
||||
return {
|
||||
id: "scoreboard-replay",
|
||||
sufficientConfidence: 0.98,
|
||||
gate,
|
||||
parse,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -321,5 +321,11 @@ export function createScoreboardDetector(
|
||||
];
|
||||
}
|
||||
|
||||
return { id: "scoreboard", gate, parse };
|
||||
// a 0.98 mean field score is a clean full read
|
||||
return {
|
||||
id: "scoreboard",
|
||||
sufficientConfidence: 0.98,
|
||||
gate,
|
||||
parse,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
/**
|
||||
* ParseSuppressor: bails out of re-parsing a static screen.
|
||||
*
|
||||
* A screen that keeps a detector's gate firing (the results scoreboard can
|
||||
* sit for tens of seconds, a paused VoD indefinitely) makes the pipeline
|
||||
* re-run the expensive parse() on every sampled frame even though nothing
|
||||
* changes. Per detector this tracks the best event confidence seen during a
|
||||
* continuous gate-pass streak; once `maxStagnantParses` consecutive parses
|
||||
* fail to improve on it, parse() is skipped (the cheap gate keeps running)
|
||||
* until the gate drops — i.e. the screen actually changed.
|
||||
*/
|
||||
|
||||
export interface SuppressorOptions {
|
||||
/** consecutive non-improving parses tolerated before suppression kicks in */
|
||||
maxStagnantParses: number;
|
||||
/** minimum confidence gain that counts as an improvement */
|
||||
minImprovement: number;
|
||||
}
|
||||
|
||||
const DEFAULT_SUPPRESSOR_OPTIONS: SuppressorOptions = {
|
||||
// at the 2fps sample rate: give a stable screen ~3s to produce its best
|
||||
// read, then stop paying for parses until the screen changes
|
||||
maxStagnantParses: 6,
|
||||
minImprovement: 0.001,
|
||||
};
|
||||
|
||||
interface StreakState {
|
||||
best: number;
|
||||
stagnant: number;
|
||||
suppressed: boolean;
|
||||
}
|
||||
|
||||
export class ParseSuppressor {
|
||||
#options: SuppressorOptions;
|
||||
#streaks = new Map<string, StreakState>();
|
||||
|
||||
constructor(options: Partial<SuppressorOptions> = {}) {
|
||||
this.#options = { ...DEFAULT_SUPPRESSOR_OPTIONS, ...options };
|
||||
}
|
||||
|
||||
/**
|
||||
* Call once per detector per frame with the gate outcome; returns whether
|
||||
* parse() should run. A failed gate ends the streak, so the next gate pass
|
||||
* starts a fresh, unsuppressed streak.
|
||||
*/
|
||||
shouldParse(detectorId: string, gatePass: boolean): boolean {
|
||||
if (!gatePass) {
|
||||
this.#streaks.delete(detectorId);
|
||||
return false;
|
||||
}
|
||||
return !this.#streaks.get(detectorId)?.suppressed;
|
||||
}
|
||||
|
||||
/** Report the outcome of a parse this suppressor approved. */
|
||||
recordParse(
|
||||
detectorId: string,
|
||||
events: readonly { confidence: number }[],
|
||||
): void {
|
||||
// no events counts as confidence 0: a false-firing gate on a static
|
||||
// screen stagnates and gets suppressed just like a parsed one
|
||||
const confidence = events.reduce(
|
||||
(max, e) => Math.max(max, e.confidence),
|
||||
0,
|
||||
);
|
||||
const streak = this.#streaks.get(detectorId);
|
||||
if (!streak) {
|
||||
this.#streaks.set(detectorId, {
|
||||
best: confidence,
|
||||
stagnant: 0,
|
||||
suppressed: false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (confidence > streak.best + this.#options.minImprovement) {
|
||||
streak.best = confidence;
|
||||
streak.stagnant = 0;
|
||||
return;
|
||||
}
|
||||
streak.stagnant += 1;
|
||||
if (streak.stagnant >= this.#options.maxStagnantParses)
|
||||
streak.suppressed = true;
|
||||
}
|
||||
}
|
||||
85
app/features/scanner/core/detectors/telemetry.ts
Normal file
85
app/features/scanner/core/detectors/telemetry.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Scan telemetry: counters the analyzer accumulates so scan performance is
|
||||
* measurable instead of guessed at — how many frames were decoded vs.
|
||||
* actually analyzed, where detector time goes (gate vs. parse), how much
|
||||
* work scheduling saved, and how much of a VoD was covered in skim mode.
|
||||
* Plain JSON so it travels over the worker boundary as-is.
|
||||
*/
|
||||
|
||||
export interface DetectorTelemetry {
|
||||
/** frames on which the detector's gate ran */
|
||||
checks: number;
|
||||
gatePasses: number;
|
||||
gateMs: number;
|
||||
parses: number;
|
||||
parseMs: number;
|
||||
/** gate passed but parse() was skipped (suppression / cooldown) */
|
||||
suppressedParses: number;
|
||||
}
|
||||
|
||||
export interface ScanTelemetry {
|
||||
decodedFrames: number;
|
||||
/** frames that went through canvas readback + normalize + detectors */
|
||||
analyzedFrames: number;
|
||||
/** video seconds covered by dense sequential decode (chunk scan) */
|
||||
activeVideoS: number;
|
||||
/** video seconds covered by keyframe-hop skimming (chunk scan) */
|
||||
skimVideoS: number;
|
||||
wallMs: number;
|
||||
detectors: Record<string, DetectorTelemetry>;
|
||||
}
|
||||
|
||||
/** Fresh all-zero telemetry for one scan/session. */
|
||||
export function createScanTelemetry(): ScanTelemetry {
|
||||
return {
|
||||
decodedFrames: 0,
|
||||
analyzedFrames: 0,
|
||||
activeVideoS: 0,
|
||||
skimVideoS: 0,
|
||||
wallMs: 0,
|
||||
detectors: {},
|
||||
};
|
||||
}
|
||||
|
||||
/** Get-or-create the per-detector counters bucket. */
|
||||
export function detectorTelemetry(
|
||||
telemetry: ScanTelemetry,
|
||||
id: string,
|
||||
): DetectorTelemetry {
|
||||
const existing = telemetry.detectors[id];
|
||||
if (existing) return existing;
|
||||
const created: DetectorTelemetry = {
|
||||
checks: 0,
|
||||
gatePasses: 0,
|
||||
gateMs: 0,
|
||||
parses: 0,
|
||||
parseMs: 0,
|
||||
suppressedParses: 0,
|
||||
};
|
||||
telemetry.detectors[id] = created;
|
||||
return created;
|
||||
}
|
||||
|
||||
/** Sum telemetry across parallel chunk scans into one report. */
|
||||
export function mergeScanTelemetry(
|
||||
parts: readonly ScanTelemetry[],
|
||||
): ScanTelemetry {
|
||||
const out = createScanTelemetry();
|
||||
for (const part of parts) {
|
||||
out.decodedFrames += part.decodedFrames;
|
||||
out.analyzedFrames += part.analyzedFrames;
|
||||
out.activeVideoS += part.activeVideoS;
|
||||
out.skimVideoS += part.skimVideoS;
|
||||
out.wallMs = Math.max(out.wallMs, part.wallMs);
|
||||
for (const [id, d] of Object.entries(part.detectors)) {
|
||||
const bucket = detectorTelemetry(out, id);
|
||||
bucket.checks += d.checks;
|
||||
bucket.gatePasses += d.gatePasses;
|
||||
bucket.gateMs += d.gateMs;
|
||||
bucket.parses += d.parses;
|
||||
bucket.parseMs += d.parseMs;
|
||||
bucket.suppressedParses += d.suppressedParses;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
/**
|
||||
* CheckThrottle: caps how often a detector is checked at all (gate included),
|
||||
* for detectors that declare a checkIntervalS — screens like the objective
|
||||
* counter change at most once a second, so sampling them at the full frame
|
||||
* rate buys nothing.
|
||||
*/
|
||||
|
||||
export class CheckThrottle {
|
||||
#lastCheckT = new Map<string, number>();
|
||||
|
||||
/**
|
||||
* Whether detector `id` should run its check on the frame at `t` seconds.
|
||||
* Approving a check starts the detector's next interval; a `t` earlier
|
||||
* than the last approved check (a fresh capture session, a VoD rescan)
|
||||
* resets the window instead of blocking until the old timeline catches up.
|
||||
*/
|
||||
shouldCheck(id: string, t: number, intervalS: number | undefined): boolean {
|
||||
if (intervalS === undefined) return true;
|
||||
const last = this.#lastCheckT.get(id);
|
||||
if (last !== undefined && t >= last && t - last < intervalS) return false;
|
||||
this.#lastCheckT.set(id, t);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -33,15 +33,41 @@ export interface GateResult {
|
||||
export interface Detector<TData = unknown> {
|
||||
id: string;
|
||||
/**
|
||||
* Minimum seconds between checks: frames inside the interval skip gate
|
||||
* and parse entirely (core/detectors/throttle.ts, applied by the
|
||||
* analyzer worker — per worker, so a VoD pool checks up to poolSize×
|
||||
* this often). Declaring an interval also exempts the detector from
|
||||
* steady-frame suppression: it marks a screen that stays up for minutes
|
||||
* with *changing* content, which the stagnating-confidence heuristic
|
||||
* would wrongly silence.
|
||||
* Minimum seconds between checks in *both* scheduling phases: frames
|
||||
* inside the interval skip gate and parse entirely
|
||||
* (core/detectors/scheduler.ts, applied by the analyzer worker).
|
||||
* Declaring an interval also exempts the detector from steady-frame
|
||||
* suppression: it marks a screen that stays up for minutes with
|
||||
* *changing* content, which the stagnating-confidence heuristic would
|
||||
* wrongly silence.
|
||||
*/
|
||||
checkIntervalS?: number;
|
||||
/**
|
||||
* Check cadence while the gate keeps failing (the search phase); unset =
|
||||
* the scheduler's default (0.25s). Do not reason from raw-gameplay
|
||||
* screen lifetimes here: produced/casted VoDs cut screens (results,
|
||||
* intros) to as little as ~1s, with transition flicker eating frames
|
||||
* inside that window — the search cadence must sample such a window
|
||||
* several times. Gates cost ~1.6ms, so searching at the analysis floor
|
||||
* is effectively free; only override upward with strong evidence.
|
||||
*/
|
||||
searchIntervalS?: number;
|
||||
/**
|
||||
* A parse reaching this confidence ends the streak's refinement
|
||||
* immediately: the timeline keeps the best read per merge window, so a
|
||||
* read this good cannot be improved upon in any way that matters.
|
||||
* Conservative by construction — if real reads never reach it, behavior
|
||||
* falls back to stagnation-based suppression.
|
||||
*/
|
||||
sufficientConfidence?: number;
|
||||
/**
|
||||
* After a sufficient read, skip parses for this long even across gate
|
||||
* drops — for overlays whose animated background flickers the gate
|
||||
* (death). Only safe when the event type's timeline merge is purely
|
||||
* time-based and the cooldown fits inside the merge window, so every
|
||||
* parse the cooldown skips would have merged into the same event anyway.
|
||||
*/
|
||||
rearmCooldownS?: number;
|
||||
/**
|
||||
* false = worker results for this detector's events ship without the
|
||||
* analyzed-frame PNG — for detectors whose events fire continuously,
|
||||
|
||||
218
app/features/scanner/tests/scheduler.test.ts
Normal file
218
app/features/scanner/tests/scheduler.test.ts
Normal file
@@ -0,0 +1,218 @@
|
||||
/**
|
||||
* Unit tests for DetectorScheduler: search/refine check cadence,
|
||||
* steady-frame suppression (stagnation and sufficient-confidence early
|
||||
* stop), the rearm cooldown, checkIntervalS compatibility, and the calm
|
||||
* signal that drives VoD skim mode.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
DetectorScheduler,
|
||||
type SchedulingInfo,
|
||||
} from "../core/detectors/scheduler";
|
||||
import test from "./node-test-compat";
|
||||
|
||||
const OPTS = {
|
||||
refineIntervalS: 0.1,
|
||||
searchIntervalS: 0.1,
|
||||
maxStagnantParses: 3,
|
||||
stagnantAfterS: 0.25,
|
||||
minImprovement: 0.001,
|
||||
quietAfterS: 10,
|
||||
matchOpenMaxS: 60,
|
||||
matchOpeningTypes: ["MapStart"],
|
||||
matchClosingTypes: ["Scoreboard"],
|
||||
};
|
||||
|
||||
function make(detector: Partial<SchedulingInfo>, options = {}) {
|
||||
return new DetectorScheduler([{ id: "d", ...detector }], {
|
||||
...OPTS,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
function feed(
|
||||
s: DetectorScheduler,
|
||||
t: number,
|
||||
options: { pass: boolean; confidence?: number; type?: string },
|
||||
): "skipped" | "gated" | "parsed" {
|
||||
if (!s.dueDetectors(t).includes("d")) return "skipped";
|
||||
s.recordGate("d", t, options.pass);
|
||||
if (!options.pass || !s.shouldParse("d", t)) return "gated";
|
||||
s.recordParse(
|
||||
"d",
|
||||
t,
|
||||
options.confidence === undefined
|
||||
? []
|
||||
: [{ type: options.type ?? "Event", confidence: options.confidence }],
|
||||
);
|
||||
return "parsed";
|
||||
}
|
||||
|
||||
test("a failing gate is only re-checked at the search cadence", () => {
|
||||
const s = make({ searchIntervalS: 1 });
|
||||
assert.equal(feed(s, 0, { pass: false }), "gated");
|
||||
assert.equal(feed(s, 0.5, { pass: false }), "skipped");
|
||||
assert.equal(feed(s, 0.99, { pass: false }), "skipped");
|
||||
assert.equal(feed(s, 1, { pass: false }), "gated");
|
||||
});
|
||||
|
||||
test("a passing gate drops to the dense refine cadence", () => {
|
||||
const s = make({ searchIntervalS: 1 });
|
||||
assert.equal(feed(s, 0, { pass: true, confidence: 0.7 }), "parsed");
|
||||
assert.equal(feed(s, 0.05, { pass: true, confidence: 0.7 }), "skipped");
|
||||
assert.equal(feed(s, 0.1, { pass: true, confidence: 0.7 }), "parsed");
|
||||
assert.equal(feed(s, 0.2, { pass: true, confidence: 0.7 }), "parsed");
|
||||
});
|
||||
|
||||
test("suppresses after stagnant parses on a static screen", () => {
|
||||
const s = make({});
|
||||
assert.equal(feed(s, 0.0, { pass: true, confidence: 0.9 }), "parsed");
|
||||
assert.equal(feed(s, 0.1, { pass: true, confidence: 0.9 }), "parsed");
|
||||
assert.equal(feed(s, 0.2, { pass: true, confidence: 0.9 }), "parsed");
|
||||
assert.equal(feed(s, 0.3, { pass: true, confidence: 0.9 }), "parsed");
|
||||
assert.equal(feed(s, 0.4, { pass: true, confidence: 0.9 }), "gated");
|
||||
assert.equal(feed(s, 0.5, { pass: true, confidence: 0.9 }), "gated");
|
||||
});
|
||||
|
||||
test("an improving read resets the stagnation counter", () => {
|
||||
const s = make({});
|
||||
feed(s, 0.0, { pass: true, confidence: 0.7 });
|
||||
feed(s, 0.1, { pass: true, confidence: 0.7 });
|
||||
feed(s, 0.2, { pass: true, confidence: 0.7 });
|
||||
assert.equal(feed(s, 0.3, { pass: true, confidence: 0.85 }), "parsed");
|
||||
assert.equal(feed(s, 0.4, { pass: true, confidence: 0.85 }), "parsed");
|
||||
assert.equal(feed(s, 0.5, { pass: true, confidence: 0.85 }), "parsed");
|
||||
assert.equal(feed(s, 0.6, { pass: true, confidence: 0.85 }), "parsed");
|
||||
assert.equal(feed(s, 0.7, { pass: true, confidence: 0.85 }), "gated");
|
||||
});
|
||||
|
||||
test("a gate drop ends the streak and re-enables parsing", () => {
|
||||
const s = make({});
|
||||
for (let i = 0; i < 5; i++) feed(s, i * 0.1, { pass: true, confidence: 0.9 });
|
||||
assert.equal(feed(s, 0.5, { pass: true, confidence: 0.9 }), "gated");
|
||||
assert.equal(feed(s, 0.6, { pass: false }), "gated");
|
||||
assert.equal(feed(s, 0.7, { pass: true, confidence: 0.9 }), "parsed");
|
||||
});
|
||||
|
||||
test("gate firing without events stagnates too", () => {
|
||||
const s = make({});
|
||||
for (let i = 0; i < 4; i++) {
|
||||
assert.equal(feed(s, i * 0.1, { pass: true }), "parsed");
|
||||
}
|
||||
assert.equal(feed(s, 0.4, { pass: true }), "gated");
|
||||
});
|
||||
|
||||
test("dense no-read parses during an entry animation respect the time floor", () => {
|
||||
const s = make({}, { stagnantAfterS: 3 });
|
||||
// the gate fires from t=0 while the screen is still animating in and
|
||||
// parses read nothing — a parse count alone must not suppress here
|
||||
for (let i = 0; i < 8; i++) {
|
||||
assert.equal(feed(s, i * 0.15, { pass: true }), "parsed");
|
||||
}
|
||||
// the screen becomes readable and the read still lands
|
||||
assert.equal(feed(s, 1.35, { pass: true, confidence: 0.88 }), "parsed");
|
||||
// once truly static past the time floor, suppression still kicks in
|
||||
for (let i = 0; i < 25; i++) {
|
||||
feed(s, 1.5 + i * 0.15, { pass: true, confidence: 0.88 });
|
||||
}
|
||||
assert.equal(feed(s, 5.4, { pass: true, confidence: 0.88 }), "gated");
|
||||
});
|
||||
|
||||
test("a sufficient read suppresses immediately", () => {
|
||||
const s = make({ sufficientConfidence: 0.95 });
|
||||
assert.equal(feed(s, 0.0, { pass: true, confidence: 0.96 }), "parsed");
|
||||
assert.equal(feed(s, 0.1, { pass: true, confidence: 0.99 }), "gated");
|
||||
// gate drop = screen changed = fresh streak parses again
|
||||
feed(s, 0.2, { pass: false });
|
||||
assert.equal(feed(s, 0.3, { pass: true, confidence: 0.7 }), "parsed");
|
||||
});
|
||||
|
||||
test("rearm cooldown skips parses across gate flicker", () => {
|
||||
const s = make({ sufficientConfidence: 0.95, rearmCooldownS: 4 });
|
||||
assert.equal(feed(s, 1.0, { pass: true, confidence: 0.96 }), "parsed");
|
||||
feed(s, 1.5, { pass: false });
|
||||
assert.equal(feed(s, 2.0, { pass: true, confidence: 0.9 }), "gated");
|
||||
assert.equal(feed(s, 4.9, { pass: true, confidence: 0.9 }), "gated");
|
||||
assert.equal(feed(s, 5.1, { pass: true, confidence: 0.9 }), "parsed");
|
||||
});
|
||||
|
||||
test("checkIntervalS caps both phases and exempts from suppression", () => {
|
||||
const s = make({ checkIntervalS: 1 });
|
||||
assert.equal(feed(s, 0.0, { pass: true, confidence: 0.9 }), "parsed");
|
||||
assert.equal(feed(s, 0.5, { pass: true, confidence: 0.9 }), "skipped");
|
||||
for (let t = 1; t < 10; t++) {
|
||||
assert.equal(feed(s, t, { pass: true, confidence: 0.9 }), "parsed");
|
||||
}
|
||||
});
|
||||
|
||||
test("suppressSteadyFrames=false checks and parses every frame", () => {
|
||||
const s = make(
|
||||
{ searchIntervalS: 1, sufficientConfidence: 0.5 },
|
||||
{ suppressSteadyFrames: false },
|
||||
);
|
||||
for (let i = 0; i < 10; i++) {
|
||||
assert.equal(feed(s, 0, { pass: true, confidence: 0.9 }), "parsed");
|
||||
}
|
||||
assert.equal(s.calm(1000), false);
|
||||
});
|
||||
|
||||
test("calm needs a quiet period and no open match", () => {
|
||||
const s = make({});
|
||||
assert.equal(feed(s, 0, { pass: false }), "gated");
|
||||
assert.equal(s.calm(5), false);
|
||||
assert.equal(s.calm(10), true);
|
||||
// a gate pass resets the quiet clock
|
||||
feed(s, 10, { pass: true, confidence: 0.9 });
|
||||
assert.equal(s.calm(15), false);
|
||||
assert.equal(s.calm(20), true);
|
||||
// a confident map-start keeps the scan active for matchOpenMaxS
|
||||
feed(s, 21, { pass: true, confidence: 0.9, type: "MapStart" });
|
||||
assert.equal(s.calm(50), false);
|
||||
assert.equal(s.calm(21 + 60), true);
|
||||
});
|
||||
|
||||
test("a scoreboard closes the match early", () => {
|
||||
const s = make({});
|
||||
feed(s, 0, { pass: true, confidence: 0.9, type: "MapStart" });
|
||||
assert.equal(s.calm(30), false);
|
||||
feed(s, 31, { pass: true, confidence: 0.9, type: "Scoreboard" });
|
||||
// quiet period still applies after the closing read
|
||||
assert.equal(s.calm(35), false);
|
||||
assert.equal(s.calm(41), true);
|
||||
});
|
||||
|
||||
test("a t jumping backwards resets the session", () => {
|
||||
const s = make({ searchIntervalS: 1 });
|
||||
feed(s, 100, { pass: false });
|
||||
assert.equal(feed(s, 100.5, { pass: false }), "skipped");
|
||||
// a rescan starts earlier than anything seen — fresh state, due again
|
||||
assert.equal(feed(s, 0, { pass: false }), "gated");
|
||||
});
|
||||
|
||||
test("detectors are tracked independently", () => {
|
||||
const s = new DetectorScheduler(
|
||||
[{ id: "a" }, { id: "b", searchIntervalS: 1 }],
|
||||
OPTS,
|
||||
);
|
||||
assert.deepEqual(s.dueDetectors(0), ["a", "b"]);
|
||||
s.recordGate("a", 0, true);
|
||||
s.recordGate("b", 0, false);
|
||||
assert.deepEqual(s.dueDetectors(0.5), ["a"]);
|
||||
assert.deepEqual(s.dueDetectors(1), ["a", "b"]);
|
||||
});
|
||||
|
||||
test("nextDueT lets frames skip analysis entirely", () => {
|
||||
const s = new DetectorScheduler(
|
||||
[
|
||||
{ id: "a", searchIntervalS: 0.5 },
|
||||
{ id: "b", searchIntervalS: 1 },
|
||||
],
|
||||
OPTS,
|
||||
);
|
||||
assert.equal(s.nextDueT(), Number.NEGATIVE_INFINITY);
|
||||
s.dueDetectors(0);
|
||||
s.recordGate("a", 0, false);
|
||||
s.recordGate("b", 0, false);
|
||||
assert.equal(s.nextDueT(), 0.5);
|
||||
});
|
||||
@@ -1,64 +0,0 @@
|
||||
/**
|
||||
* Unit tests for ParseSuppressor: a static screen (gate keeps passing,
|
||||
* confidence stops improving) gets its parse() suppressed after the
|
||||
* stagnation budget, and any gate drop resets the streak.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { ParseSuppressor } from "../core/detectors/suppressor";
|
||||
import test from "./node-test-compat";
|
||||
|
||||
const OPTS = { maxStagnantParses: 3, minImprovement: 0.001 };
|
||||
|
||||
function feed(
|
||||
s: ParseSuppressor,
|
||||
id: string,
|
||||
confidence: number | null,
|
||||
): boolean {
|
||||
const allowed = s.shouldParse(id, true);
|
||||
if (allowed) s.recordParse(id, confidence === null ? [] : [{ confidence }]);
|
||||
return allowed;
|
||||
}
|
||||
|
||||
test("suppresses after stagnant parses on a static screen", () => {
|
||||
const s = new ParseSuppressor(OPTS);
|
||||
assert.equal(feed(s, "scoreboard", 0.9), true); // sets the baseline
|
||||
assert.equal(feed(s, "scoreboard", 0.9), true); // stagnant 1
|
||||
assert.equal(feed(s, "scoreboard", 0.9), true); // stagnant 2
|
||||
assert.equal(feed(s, "scoreboard", 0.9), true); // stagnant 3 -> suppressed
|
||||
assert.equal(s.shouldParse("scoreboard", true), false);
|
||||
assert.equal(s.shouldParse("scoreboard", true), false);
|
||||
});
|
||||
|
||||
test("an improving read resets the stagnation counter", () => {
|
||||
const s = new ParseSuppressor(OPTS);
|
||||
feed(s, "scoreboard", 0.7);
|
||||
feed(s, "scoreboard", 0.7);
|
||||
feed(s, "scoreboard", 0.7);
|
||||
assert.equal(feed(s, "scoreboard", 0.85), true); // improvement, counter resets
|
||||
assert.equal(feed(s, "scoreboard", 0.85), true);
|
||||
assert.equal(feed(s, "scoreboard", 0.85), true);
|
||||
assert.equal(feed(s, "scoreboard", 0.85), true); // stagnant 3 -> suppressed
|
||||
assert.equal(s.shouldParse("scoreboard", true), false);
|
||||
});
|
||||
|
||||
test("a gate drop ends the streak and re-enables parsing", () => {
|
||||
const s = new ParseSuppressor(OPTS);
|
||||
for (let i = 0; i < 5; i++) feed(s, "scoreboard", 0.9);
|
||||
assert.equal(s.shouldParse("scoreboard", true), false);
|
||||
assert.equal(s.shouldParse("scoreboard", false), false); // screen changed
|
||||
assert.equal(feed(s, "scoreboard", 0.9), true); // fresh streak parses again
|
||||
});
|
||||
|
||||
test("gate firing without events stagnates too", () => {
|
||||
const s = new ParseSuppressor(OPTS);
|
||||
for (let i = 0; i < 4; i++) assert.equal(feed(s, "death", null), true);
|
||||
assert.equal(s.shouldParse("death", true), false);
|
||||
});
|
||||
|
||||
test("detectors are tracked independently", () => {
|
||||
const s = new ParseSuppressor(OPTS);
|
||||
for (let i = 0; i < 5; i++) feed(s, "scoreboard", 0.9);
|
||||
assert.equal(s.shouldParse("scoreboard", true), false);
|
||||
assert.equal(s.shouldParse("death", true), true);
|
||||
});
|
||||
@@ -1,47 +0,0 @@
|
||||
/**
|
||||
* Unit tests for CheckThrottle: a detector with a check interval is checked
|
||||
* at most once per interval, timeline regressions reset the window, and
|
||||
* detectors without an interval are never throttled.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { CheckThrottle } from "../core/detectors/throttle";
|
||||
import test from "./node-test-compat";
|
||||
|
||||
test("no interval means every frame is checked", () => {
|
||||
const throttle = new CheckThrottle();
|
||||
assert.equal(throttle.shouldCheck("scoreboard", 0, undefined), true);
|
||||
assert.equal(throttle.shouldCheck("scoreboard", 0.1, undefined), true);
|
||||
assert.equal(throttle.shouldCheck("scoreboard", 0.2, undefined), true);
|
||||
});
|
||||
|
||||
test("frames inside the interval are skipped", () => {
|
||||
const throttle = new CheckThrottle();
|
||||
assert.equal(throttle.shouldCheck("objective", 10, 1), true);
|
||||
assert.equal(throttle.shouldCheck("objective", 10.5, 1), false);
|
||||
assert.equal(throttle.shouldCheck("objective", 10.99, 1), false);
|
||||
assert.equal(throttle.shouldCheck("objective", 11, 1), true);
|
||||
assert.equal(throttle.shouldCheck("objective", 11.5, 1), false);
|
||||
});
|
||||
|
||||
test("a skipped frame does not push the next check later", () => {
|
||||
const throttle = new CheckThrottle();
|
||||
assert.equal(throttle.shouldCheck("objective", 0, 1), true);
|
||||
assert.equal(throttle.shouldCheck("objective", 0.9, 1), false);
|
||||
assert.equal(throttle.shouldCheck("objective", 1.1, 1), true);
|
||||
});
|
||||
|
||||
test("a timeline regression resets the window", () => {
|
||||
const throttle = new CheckThrottle();
|
||||
assert.equal(throttle.shouldCheck("objective", 100, 1), true);
|
||||
// new capture session / VoD rescan starts its clock over
|
||||
assert.equal(throttle.shouldCheck("objective", 0.2, 1), true);
|
||||
assert.equal(throttle.shouldCheck("objective", 0.6, 1), false);
|
||||
});
|
||||
|
||||
test("detectors are tracked independently", () => {
|
||||
const throttle = new CheckThrottle();
|
||||
assert.equal(throttle.shouldCheck("objective", 0, 1), true);
|
||||
assert.equal(throttle.shouldCheck("objective", 0.5, 1), false);
|
||||
assert.equal(throttle.shouldCheck("other", 0.5, 1), true);
|
||||
});
|
||||
@@ -1,22 +1,68 @@
|
||||
/**
|
||||
* AnalyzerWorker: owns OpenCV.js (WASM) and the detector registry.
|
||||
* The main thread posts ImageBitmaps; results come back as plain JSON.
|
||||
* AnalyzerWorker: owns OpenCV.js (WASM), the detector registry and a
|
||||
* DetectorScheduler. Two ways in:
|
||||
*
|
||||
* - "frame": the main thread posts one ImageBitmap/VideoFrame at a time
|
||||
* (live capture, screenshot harness, the VoD seek fallback). Results come
|
||||
* back per detector, then a "done" carrying the scheduler's calm signal
|
||||
* and telemetry.
|
||||
* - "scanChunk": a VoD time slice is demuxed and decoded entirely in the
|
||||
* worker with mediabunny. The worker owns a contiguous slice, so
|
||||
* scheduling is exact, frames that no detector is due for skip canvas
|
||||
* readback entirely, and calm stretches are skimmed by keyframe hops
|
||||
* instead of decoding every frame — the big VoD speedup, since sequential
|
||||
* decode is what bounds scan wall-clock time.
|
||||
*/
|
||||
import {
|
||||
ALL_FORMATS,
|
||||
BlobSource,
|
||||
EncodedPacketSink,
|
||||
Input,
|
||||
type VideoSample,
|
||||
VideoSampleSink,
|
||||
} from "mediabunny";
|
||||
import { loadOpenCV } from "../core/cv";
|
||||
import { createAllDetectors } from "../core/detectors/registry";
|
||||
import { ParseSuppressor } from "../core/detectors/suppressor";
|
||||
import { CheckThrottle } from "../core/detectors/throttle";
|
||||
import { MAP_START_EVENT_TYPE } from "../core/detectors/map-start/index";
|
||||
import {
|
||||
createAllDetectors,
|
||||
SCOREBOARD_EVENT_TYPES,
|
||||
} from "../core/detectors/registry";
|
||||
import { DetectorScheduler } from "../core/detectors/scheduler";
|
||||
import {
|
||||
createScanTelemetry,
|
||||
detectorTelemetry,
|
||||
} from "../core/detectors/telemetry";
|
||||
import type { Detector } from "../core/detectors/types";
|
||||
import { normalizeFrame, toMat } from "../core/image";
|
||||
import type { AnalyzeRequest, InitRequest, WorkerResponse } from "./protocol";
|
||||
import type {
|
||||
AnalyzeRequest,
|
||||
InitRequest,
|
||||
ScanChunkRequest,
|
||||
WorkerRequest,
|
||||
WorkerResponse,
|
||||
} from "./protocol";
|
||||
import { fetchScoreboardResources } from "./resources";
|
||||
|
||||
let detectors: Detector<unknown>[] = [];
|
||||
let suppressor: ParseSuppressor | null = null;
|
||||
let throttle = new CheckThrottle();
|
||||
/**
|
||||
* Widest skim hop: calm footage is sampled at the keyframe cadence, capped
|
||||
* here so long-GOP recordings still cannot slip a results screen (~10s) or
|
||||
* a match intro (~7s) between two samples.
|
||||
*/
|
||||
const MAX_SKIM_STRIDE_S = 2.5;
|
||||
const PROGRESS_POST_INTERVAL_MS = 400;
|
||||
const PREVIEW_POST_INTERVAL_MS = 600;
|
||||
const PREVIEW_WIDTH = 480;
|
||||
const PREVIEW_HEIGHT = 270;
|
||||
|
||||
function post(message: WorkerResponse): void {
|
||||
self.postMessage(message);
|
||||
let detectors: Detector<unknown>[] = [];
|
||||
let scheduler: DetectorScheduler | null = null;
|
||||
let telemetry = createScanTelemetry();
|
||||
let chunkAborted = false;
|
||||
/** last per-frame t, to reset telemetry when a new session rewinds the clock */
|
||||
let lastFrameT = Number.NEGATIVE_INFINITY;
|
||||
|
||||
function post(message: WorkerResponse, transfer: Transferable[] = []): void {
|
||||
self.postMessage(message, { transfer });
|
||||
}
|
||||
|
||||
async function init({
|
||||
@@ -27,20 +73,36 @@ async function init({
|
||||
await loadOpenCV();
|
||||
const resources = await fetchScoreboardResources(assetsBaseUrl);
|
||||
detectors = createAllDetectors(resources);
|
||||
suppressor = suppressSteadyFrames ? new ParseSuppressor() : null;
|
||||
throttle = new CheckThrottle();
|
||||
scheduler = new DetectorScheduler(detectors, {
|
||||
suppressSteadyFrames,
|
||||
matchOpeningTypes: [MAP_START_EVENT_TYPE],
|
||||
matchClosingTypes: SCOREBOARD_EVENT_TYPES,
|
||||
});
|
||||
telemetry = createScanTelemetry();
|
||||
post({ kind: "ready" });
|
||||
} catch (error) {
|
||||
post({ kind: "error", message: `init failed: ${String(error)}` });
|
||||
}
|
||||
}
|
||||
|
||||
async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
|
||||
/**
|
||||
* Run the due detectors over one frame; closes `bitmap`. When the scheduler
|
||||
* has no detector due, the canvas readback and normalize are skipped too.
|
||||
*/
|
||||
async function analyzeFrame(
|
||||
bitmap: ImageBitmap | VideoFrame,
|
||||
t: number,
|
||||
): Promise<void> {
|
||||
const due = scheduler!.dueDetectors(t);
|
||||
if (due.length === 0) {
|
||||
bitmap.close();
|
||||
return;
|
||||
}
|
||||
const width = "displayWidth" in bitmap ? bitmap.displayWidth : bitmap.width;
|
||||
const height =
|
||||
"displayHeight" in bitmap ? bitmap.displayHeight : bitmap.height;
|
||||
const canvas = new OffscreenCanvas(width, height);
|
||||
const ctx = canvas.getContext("2d")!;
|
||||
const ctx = canvas.getContext("2d", { willReadFrequently: true })!;
|
||||
ctx.drawImage(bitmap, 0, 0);
|
||||
bitmap.close();
|
||||
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||||
@@ -56,6 +118,7 @@ async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
|
||||
} finally {
|
||||
src.delete();
|
||||
}
|
||||
telemetry.analyzedFrames++;
|
||||
|
||||
// On detection, ship back the exact analyzed pixels (lossless, at capture
|
||||
// resolution) so the UI never has to re-grab a later frame — encoded at
|
||||
@@ -66,18 +129,24 @@ async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
|
||||
|
||||
try {
|
||||
for (const detector of detectors) {
|
||||
if (!throttle.shouldCheck(detector.id, t, detector.checkIntervalS))
|
||||
continue;
|
||||
if (!due.includes(detector.id)) continue;
|
||||
const counters = detectorTelemetry(telemetry, detector.id);
|
||||
counters.checks++;
|
||||
const gateStart = performance.now();
|
||||
const gate = detector.gate(frame);
|
||||
// interval-limited detectors watch changing content and opt out of
|
||||
// steady-frame suppression (see Detector.checkIntervalS)
|
||||
const suppressible =
|
||||
detector.checkIntervalS === undefined ? suppressor : null;
|
||||
const runParse = suppressible
|
||||
? suppressible.shouldParse(detector.id, gate.pass)
|
||||
: gate.pass;
|
||||
const events = runParse ? detector.parse(frame, t, gate) : [];
|
||||
if (runParse) suppressible?.recordParse(detector.id, events);
|
||||
counters.gateMs += performance.now() - gateStart;
|
||||
scheduler!.recordGate(detector.id, t, gate.pass);
|
||||
if (gate.pass) counters.gatePasses++;
|
||||
const runParse = gate.pass && scheduler!.shouldParse(detector.id, t);
|
||||
if (gate.pass && !runParse) counters.suppressedParses++;
|
||||
let events: ReturnType<typeof detector.parse> = [];
|
||||
if (runParse) {
|
||||
const parseStart = performance.now();
|
||||
events = detector.parse(frame, t, gate);
|
||||
counters.parses++;
|
||||
counters.parseMs += performance.now() - parseStart;
|
||||
scheduler!.recordParse(detector.id, t, events);
|
||||
}
|
||||
const blob =
|
||||
events.length > 0 && detector.attachFrame !== false
|
||||
? await frameBlob()
|
||||
@@ -91,16 +160,147 @@ async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
|
||||
frame: blob,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
post({ kind: "error", message: `analyze failed: ${String(error)}` });
|
||||
} finally {
|
||||
frame.delete();
|
||||
post({ kind: "done", t });
|
||||
}
|
||||
}
|
||||
|
||||
async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
|
||||
if (t + 5 < lastFrameT) telemetry = createScanTelemetry();
|
||||
lastFrameT = t;
|
||||
try {
|
||||
await analyzeFrame(bitmap, t);
|
||||
} catch (error) {
|
||||
post({ kind: "error", message: `analyze failed: ${String(error)}` });
|
||||
}
|
||||
post({ kind: "done", t, calm: scheduler!.calm(t), telemetry });
|
||||
}
|
||||
|
||||
async function scanChunk({
|
||||
file,
|
||||
chunkIndex,
|
||||
tStart,
|
||||
tEnd,
|
||||
}: ScanChunkRequest): Promise<void> {
|
||||
chunkAborted = false;
|
||||
scheduler!.reset(tStart);
|
||||
telemetry = createScanTelemetry();
|
||||
const wallStart = performance.now();
|
||||
let lastProgressAt = 0;
|
||||
let lastPreviewAt = 0;
|
||||
let cursor = tStart;
|
||||
let mode: "active" | "skim" = "active";
|
||||
|
||||
const input = new Input({
|
||||
formats: ALL_FORMATS,
|
||||
source: new BlobSource(file),
|
||||
});
|
||||
try {
|
||||
const track = await input.getPrimaryVideoTrack();
|
||||
if (!track || !(await track.canDecode())) {
|
||||
throw new Error("worker cannot decode this file");
|
||||
}
|
||||
const samples = new VideoSampleSink(track);
|
||||
const packets = new EncodedPacketSink(track);
|
||||
|
||||
const handleSample = async (sample: VideoSample): Promise<void> => {
|
||||
telemetry.decodedFrames++;
|
||||
const t = sample.timestamp;
|
||||
const span = Math.max(0, t - cursor);
|
||||
if (mode === "active") telemetry.activeVideoS += span;
|
||||
else telemetry.skimVideoS += span;
|
||||
cursor = Math.max(cursor, t);
|
||||
const frame = sample.toVideoFrame();
|
||||
sample.close();
|
||||
const now = performance.now();
|
||||
let preview: ImageBitmap | undefined;
|
||||
if (now - lastPreviewAt >= PREVIEW_POST_INTERVAL_MS) {
|
||||
lastPreviewAt = now;
|
||||
preview = await createImageBitmap(frame, {
|
||||
resizeWidth: PREVIEW_WIDTH,
|
||||
resizeHeight: PREVIEW_HEIGHT,
|
||||
});
|
||||
}
|
||||
if (t >= scheduler!.nextDueT()) {
|
||||
await analyzeFrame(frame, t);
|
||||
} else {
|
||||
frame.close();
|
||||
}
|
||||
if (preview || now - lastProgressAt >= PROGRESS_POST_INTERVAL_MS) {
|
||||
lastProgressAt = now;
|
||||
telemetry.wallMs = performance.now() - wallStart;
|
||||
post(
|
||||
{
|
||||
kind: "chunkProgress",
|
||||
chunkIndex,
|
||||
t: cursor,
|
||||
mode,
|
||||
telemetry,
|
||||
preview,
|
||||
},
|
||||
preview ? [preview] : [],
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
scan: while (!chunkAborted && cursor < tEnd) {
|
||||
if (mode === "active") {
|
||||
// dense sequential decode: every frame is seen, the scheduler
|
||||
// decides which are worth analyzing
|
||||
for await (const sample of samples.samples(cursor)) {
|
||||
if (!sample) continue;
|
||||
if (chunkAborted || sample.timestamp >= tEnd) {
|
||||
sample.close();
|
||||
break scan;
|
||||
}
|
||||
await handleSample(sample);
|
||||
if (scheduler!.calm(cursor)) {
|
||||
mode = "skim";
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (mode === "active") break; // media ended before tEnd
|
||||
} else {
|
||||
// skim: hop keyframe to keyframe (single-frame decodes) while
|
||||
// calm, capped so long GOPs cannot hide a short screen
|
||||
const key = await packets.getKeyPacket(cursor + MAX_SKIM_STRIDE_S, {
|
||||
verifyKeyPackets: true,
|
||||
});
|
||||
const target =
|
||||
key && key.timestamp > cursor
|
||||
? key.timestamp
|
||||
: cursor + MAX_SKIM_STRIDE_S;
|
||||
if (target >= tEnd) {
|
||||
cursor = tEnd;
|
||||
break;
|
||||
}
|
||||
const sample = await samples.getSample(target);
|
||||
if (!sample) {
|
||||
cursor = target;
|
||||
continue;
|
||||
}
|
||||
await handleSample(sample);
|
||||
cursor = Math.max(cursor, target);
|
||||
if (!scheduler!.calm(cursor)) mode = "active";
|
||||
}
|
||||
}
|
||||
|
||||
telemetry.wallMs = performance.now() - wallStart;
|
||||
post({ kind: "chunkDone", chunkIndex, telemetry });
|
||||
} catch (error) {
|
||||
post({
|
||||
kind: "error",
|
||||
message: `chunk ${chunkIndex} scan failed: ${String(error)}`,
|
||||
});
|
||||
} finally {
|
||||
input.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
self.onmessage = (e: MessageEvent) => {
|
||||
const msg = e.data as { kind: string } & Record<string, unknown>;
|
||||
if (msg.kind === "init") void init(msg as unknown as InitRequest);
|
||||
else if (msg.kind === "frame") void analyze(msg as unknown as AnalyzeRequest);
|
||||
const msg = e.data as WorkerRequest;
|
||||
if (msg.kind === "init") void init(msg);
|
||||
else if (msg.kind === "frame") void analyze(msg);
|
||||
else if (msg.kind === "scanChunk") void scanChunk(msg);
|
||||
else if (msg.kind === "abortChunk") chunkAborted = true;
|
||||
};
|
||||
|
||||
@@ -1,16 +1,38 @@
|
||||
/**
|
||||
* Main-thread wrapper around the AnalyzerWorker: init handshake, one
|
||||
* in-flight frame at a time (the sampler drops frames while busy). Each
|
||||
* frame yields one result per registered detector, then a single "done".
|
||||
* Main-thread wrapper around the AnalyzerWorker: init handshake, then either
|
||||
* one in-flight frame at a time (live capture / screenshot / seek fallback —
|
||||
* each frame yields one result per due detector, then a "done" carrying the
|
||||
* scheduler's calm signal and telemetry) or one in-flight chunk scan (the
|
||||
* worker decodes and analyzes a VoD time slice by itself, streaming results
|
||||
* and progress until "chunkDone").
|
||||
*/
|
||||
import { Config } from "../../../config";
|
||||
import type { ScanTelemetry } from "../core/detectors/telemetry";
|
||||
import type { WorkerResponse } from "./protocol";
|
||||
|
||||
export type ResultHandler = (
|
||||
result: Extract<WorkerResponse, { kind: "result" }>,
|
||||
) => void;
|
||||
export type ErrorHandler = (message: string) => void;
|
||||
export type DoneHandler = (t: number) => void;
|
||||
export interface DoneInfo {
|
||||
calm: boolean;
|
||||
telemetry: ScanTelemetry;
|
||||
}
|
||||
export type DoneHandler = (t: number, info: DoneInfo) => void;
|
||||
export type ChunkProgress = Extract<WorkerResponse, { kind: "chunkProgress" }>;
|
||||
export type ChunkProgressHandler = (progress: ChunkProgress) => void;
|
||||
|
||||
/** each chunk-scanning worker decodes and analyzes on its own; leave a core
|
||||
* for the main thread and one for the browser's media stack */
|
||||
export function defaultScanWorkerCount(): number {
|
||||
return Math.min(4, Math.max(1, (navigator.hardwareConcurrency || 4) - 2));
|
||||
}
|
||||
|
||||
interface PendingChunk {
|
||||
resolve(telemetry: ScanTelemetry): void;
|
||||
reject(error: Error): void;
|
||||
onProgress?: ChunkProgressHandler;
|
||||
}
|
||||
|
||||
export class AnalyzerClient {
|
||||
#worker: Worker;
|
||||
@@ -20,6 +42,8 @@ export class AnalyzerClient {
|
||||
#onError: ErrorHandler;
|
||||
#onDone: DoneHandler | undefined;
|
||||
#readyPromise: Promise<void>;
|
||||
#idleWaiters: (() => void)[] = [];
|
||||
#chunk: PendingChunk | null = null;
|
||||
|
||||
constructor(
|
||||
onResult: ResultHandler,
|
||||
@@ -49,23 +73,27 @@ export class AnalyzerClient {
|
||||
} else if (msg.kind === "result") {
|
||||
this.#onResult(msg);
|
||||
} else if (msg.kind === "done") {
|
||||
this.#busy = false;
|
||||
this.#onDone?.(msg.t);
|
||||
this.#settle();
|
||||
this.#onDone?.(msg.t, { calm: msg.calm, telemetry: msg.telemetry });
|
||||
} else if (msg.kind === "chunkProgress") {
|
||||
this.#chunk?.onProgress?.(msg);
|
||||
} else if (msg.kind === "chunkDone") {
|
||||
const chunk = this.#chunk;
|
||||
this.#chunk = null;
|
||||
this.#settle();
|
||||
chunk?.resolve(msg.telemetry);
|
||||
} else if (msg.kind === "error") {
|
||||
this.#busy = false;
|
||||
this.#onError(msg.message);
|
||||
this.#fail(msg.message);
|
||||
}
|
||||
};
|
||||
// A throw outside the worker's own try/catch posts neither "error" nor
|
||||
// "done"; without these handlers `busy` would stay true forever and the
|
||||
// sampler / VoD scan would silently freeze.
|
||||
this.#worker.onerror = (e: ErrorEvent) => {
|
||||
this.#busy = false;
|
||||
this.#onError(`worker error: ${e.message || String(e)}`);
|
||||
this.#fail(`worker error: ${e.message || String(e)}`);
|
||||
};
|
||||
this.#worker.onmessageerror = () => {
|
||||
this.#busy = false;
|
||||
this.#onError("worker message deserialization failed");
|
||||
this.#fail("worker message deserialization failed");
|
||||
};
|
||||
this.#worker.postMessage({
|
||||
kind: "init",
|
||||
@@ -82,6 +110,13 @@ export class AnalyzerClient {
|
||||
return this.#busy || !this.#ready;
|
||||
}
|
||||
|
||||
/** Resolves once no frame or chunk scan is in flight. Call whenReady() first. */
|
||||
async whenIdle(): Promise<void> {
|
||||
while (this.#busy) {
|
||||
await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns false (and closes the bitmap) if the worker is still busy. */
|
||||
analyze(bitmap: ImageBitmap | VideoFrame, t: number): boolean {
|
||||
if (this.busy) {
|
||||
@@ -93,7 +128,46 @@ export class AnalyzerClient {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan [tStart, tEnd) of `file` inside the worker. Results stream to the
|
||||
* shared result handler; resolves with the chunk's telemetry once done
|
||||
* (an aborted chunk resolves too — abort is not an error).
|
||||
*/
|
||||
scanChunk(
|
||||
request: { file: File; chunkIndex: number; tStart: number; tEnd: number },
|
||||
onProgress?: ChunkProgressHandler,
|
||||
): Promise<ScanTelemetry> {
|
||||
if (this.busy) {
|
||||
return Promise.reject(new Error("analyzer is busy"));
|
||||
}
|
||||
this.#busy = true;
|
||||
return new Promise((resolve, reject) => {
|
||||
this.#chunk = { resolve, reject, onProgress };
|
||||
this.#worker.postMessage({ kind: "scanChunk", ...request });
|
||||
});
|
||||
}
|
||||
|
||||
/** Ask a running chunk scan to stop; it resolves after the current frame. */
|
||||
abortChunk(): void {
|
||||
if (this.#chunk) this.#worker.postMessage({ kind: "abortChunk" });
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.#worker.terminate();
|
||||
}
|
||||
|
||||
#settle(): void {
|
||||
this.#busy = false;
|
||||
const waiters = this.#idleWaiters;
|
||||
this.#idleWaiters = [];
|
||||
for (const waiter of waiters) waiter();
|
||||
}
|
||||
|
||||
#fail(message: string): void {
|
||||
const chunk = this.#chunk;
|
||||
this.#chunk = null;
|
||||
this.#settle();
|
||||
if (chunk) chunk.reject(new Error(message));
|
||||
else this.#onError(message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
/**
|
||||
* AnalyzerPool: several AnalyzerClients so frame analysis parallelizes
|
||||
* across cores. The VoD scan hands each decoded frame to any idle worker
|
||||
* and never waits; when all workers are busy the frame is dropped — the
|
||||
* next one is milliseconds of video away, so coverage stays as dense as
|
||||
* the machine can analyze. Results arrive out of decode order, which the
|
||||
* consumer must tolerate (TimelineBuilder does: it merges on |Δt| and
|
||||
* keeps its list sorted, independent of arrival order).
|
||||
*/
|
||||
import {
|
||||
AnalyzerClient,
|
||||
type ErrorHandler,
|
||||
type ResultHandler,
|
||||
} from "./client";
|
||||
|
||||
/** leave cores for the main thread and the video decoder */
|
||||
export function defaultPoolSize(): number {
|
||||
return Math.min(4, Math.max(1, (navigator.hardwareConcurrency || 4) - 2));
|
||||
}
|
||||
|
||||
export class AnalyzerPool {
|
||||
#clients: AnalyzerClient[];
|
||||
#idleWaiters: (() => void)[] = [];
|
||||
|
||||
constructor(size: number, onResult: ResultHandler, onError: ErrorHandler) {
|
||||
const wake = () => {
|
||||
const waiters = this.#idleWaiters;
|
||||
this.#idleWaiters = [];
|
||||
for (const waiter of waiters) waiter();
|
||||
};
|
||||
this.#clients = Array.from(
|
||||
{ length: size },
|
||||
() =>
|
||||
new AnalyzerClient(
|
||||
onResult,
|
||||
(message) => {
|
||||
wake(); // an errored frame is also a finished frame — don't hang whenIdle
|
||||
onError(message);
|
||||
},
|
||||
wake,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
whenReady(): Promise<void> {
|
||||
return Promise.all(this.#clients.map((c) => c.whenReady())).then(() => {});
|
||||
}
|
||||
|
||||
hasIdle(): boolean {
|
||||
return this.#clients.some((c) => !c.busy);
|
||||
}
|
||||
|
||||
/** Resolves once at least one worker is free. Call whenReady() first. */
|
||||
async whenAnyIdle(): Promise<void> {
|
||||
while (!this.hasIdle()) {
|
||||
await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
/** Hand the frame to an idle worker; false (and the frame closed) when all are busy. */
|
||||
tryAnalyze(frame: ImageBitmap | VideoFrame, t: number): boolean {
|
||||
const idle = this.#clients.find((c) => !c.busy);
|
||||
if (!idle) {
|
||||
frame.close();
|
||||
return false;
|
||||
}
|
||||
return idle.analyze(frame, t);
|
||||
}
|
||||
|
||||
/** Resolves once no worker has a frame in flight. Call whenReady() first. */
|
||||
async whenIdle(): Promise<void> {
|
||||
while (this.#clients.some((c) => c.busy)) {
|
||||
await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
for (const client of this.#clients) client.dispose();
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { ScanTelemetry } from "../core/detectors/telemetry";
|
||||
import type { DetectedEvent, GateResult } from "../core/detectors/types";
|
||||
|
||||
export interface InitRequest {
|
||||
@@ -10,8 +11,9 @@ export interface InitRequest {
|
||||
assetsBaseUrl: string;
|
||||
/**
|
||||
* skip parse() for a detector whose gate keeps firing without confidence
|
||||
* improving (static screen); default true — one-shot consumers like the
|
||||
* screenshot harness turn it off
|
||||
* improving (static screen), and let the scheduler thin out checks;
|
||||
* default true — one-shot consumers like the screenshot harness turn it
|
||||
* off to get every detector on every frame
|
||||
*/
|
||||
suppressSteadyFrames?: boolean;
|
||||
}
|
||||
@@ -25,6 +27,32 @@ export interface AnalyzeRequest {
|
||||
t: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan a time slice of a VoD entirely inside the worker: demux + decode with
|
||||
* mediabunny, schedule detectors, post results as they fire. Decoding in the
|
||||
* worker removes the per-frame main-thread hop and lets each worker own a
|
||||
* contiguous slice, so scheduler state (cadence, suppression, calm) is exact
|
||||
* instead of split across a pool.
|
||||
*/
|
||||
export interface ScanChunkRequest {
|
||||
kind: "scanChunk";
|
||||
file: File;
|
||||
chunkIndex: number;
|
||||
/** seconds; the chunk scans [tStart, tEnd) */
|
||||
tStart: number;
|
||||
tEnd: number;
|
||||
}
|
||||
|
||||
export interface AbortChunkRequest {
|
||||
kind: "abortChunk";
|
||||
}
|
||||
|
||||
export type WorkerRequest =
|
||||
| InitRequest
|
||||
| AnalyzeRequest
|
||||
| ScanChunkRequest
|
||||
| AbortChunkRequest;
|
||||
|
||||
export type WorkerResponse =
|
||||
| { kind: "ready" }
|
||||
| {
|
||||
@@ -36,6 +64,23 @@ export type WorkerResponse =
|
||||
/** lossless PNG of the exact frame that was analyzed; present when events fired */
|
||||
frame?: Blob;
|
||||
}
|
||||
/** all detectors have reported for frame t */
|
||||
| { kind: "done"; t: number }
|
||||
/** all due detectors have reported for frame t (per-frame path only) */
|
||||
| {
|
||||
kind: "done";
|
||||
t: number;
|
||||
/** scheduler sees dead air — the caller may widen its sampling stride */
|
||||
calm: boolean;
|
||||
telemetry: ScanTelemetry;
|
||||
}
|
||||
| {
|
||||
kind: "chunkProgress";
|
||||
chunkIndex: number;
|
||||
/** seconds of video the chunk scan has reached */
|
||||
t: number;
|
||||
mode: "active" | "skim";
|
||||
telemetry: ScanTelemetry;
|
||||
/** small bitmap of the latest decoded frame, for the preview canvas */
|
||||
preview?: ImageBitmap;
|
||||
}
|
||||
| { kind: "chunkDone"; chunkIndex: number; telemetry: ScanTelemetry }
|
||||
| { kind: "error"; message: string };
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
"test:scanner": "vitest run --project scanner",
|
||||
"scanner:report": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/report.ts",
|
||||
"scanner:fixtures": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/run-fixtures.ts",
|
||||
"scanner:replay": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/replay-frames.ts",
|
||||
"scanner:bootstrap-atlas": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/bootstrap-atlas-from-fixture.ts",
|
||||
"scanner:build-glyph-atlas": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/build-glyph-atlas.ts",
|
||||
"scanner:build-localized-entries": "vite-node -c scripts/scanner/vite-node.config.ts scripts/scanner/build-localized-entries.ts",
|
||||
|
||||
82
scripts/scanner/replay-frames.ts
Normal file
82
scripts/scanner/replay-frames.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
/** biome-ignore-all lint/suspicious/noConsole: CLI script output */
|
||||
/**
|
||||
* CLI harness: replay a directory of extracted VoD frames through the full
|
||||
* detector registry driven by a DetectorScheduler, mirroring the analyzer
|
||||
* worker's chunk scan — to reproduce scheduling-dependent misses offline.
|
||||
* This is the tool for "the browser scan missed an event that a fixture
|
||||
* parses fine": extract the surrounding footage with
|
||||
* ffmpeg -ss <startT> -i vod.mkv -t 30 -vf fps=6 frames/f%04d.png
|
||||
* then replay it and watch which checks the scheduler ran and what they saw.
|
||||
*
|
||||
* Usage: pnpm scanner:replay <framesDir> <startT> <fps>
|
||||
* (frames are ffmpeg-numbered f0001.png..; t = startT + (n-1)/fps)
|
||||
*/
|
||||
import { readdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { loadOpenCV } from "../../app/features/scanner/core/cv";
|
||||
import { MAP_START_EVENT_TYPE } from "../../app/features/scanner/core/detectors/map-start/index";
|
||||
import {
|
||||
createAllDetectors,
|
||||
SCOREBOARD_EVENT_TYPES,
|
||||
} from "../../app/features/scanner/core/detectors/registry";
|
||||
import { DetectorScheduler } from "../../app/features/scanner/core/detectors/scheduler";
|
||||
import { normalizeFrame, toMat } from "../../app/features/scanner/core/image";
|
||||
import { readImage } from "../../app/features/scanner/node/image-io";
|
||||
import { loadScoreboardResources } from "../../app/features/scanner/node/resources";
|
||||
|
||||
const [framesDir, startTArg, fpsArg] = process.argv.slice(2);
|
||||
if (!framesDir || !startTArg || !fpsArg) {
|
||||
console.error("usage: pnpm scanner:replay <framesDir> <startT> <fps>");
|
||||
process.exit(1);
|
||||
}
|
||||
const startT = Number(startTArg);
|
||||
const fps = Number(fpsArg);
|
||||
|
||||
await loadOpenCV();
|
||||
const detectors = createAllDetectors(await loadScoreboardResources());
|
||||
const scheduler = new DetectorScheduler(detectors, {
|
||||
matchOpeningTypes: [MAP_START_EVENT_TYPE],
|
||||
matchClosingTypes: SCOREBOARD_EVENT_TYPES,
|
||||
});
|
||||
scheduler.reset(startT);
|
||||
|
||||
const files = readdirSync(framesDir)
|
||||
.filter((f) => f.endsWith(".png"))
|
||||
.sort();
|
||||
|
||||
for (const [i, file] of files.entries()) {
|
||||
const t = startT + i / fps;
|
||||
const due = scheduler.dueDetectors(t);
|
||||
if (due.length === 0) continue;
|
||||
const image = await readImage(join(framesDir, file));
|
||||
const src = toMat(image);
|
||||
const frame = normalizeFrame(src);
|
||||
src.delete();
|
||||
for (const detector of detectors) {
|
||||
if (!due.includes(detector.id)) continue;
|
||||
const gate = detector.gate(frame);
|
||||
scheduler.recordGate(detector.id, t, gate.pass);
|
||||
if (!gate.pass) {
|
||||
if (process.env.LOG_GATES?.includes(detector.id)) {
|
||||
console.log(
|
||||
`${t.toFixed(2)} ${file} ${detector.id} gate=${gate.score.toFixed(3)} FAIL`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!scheduler.shouldParse(detector.id, t)) {
|
||||
console.log(
|
||||
`${t.toFixed(2)} ${file} ${detector.id} gate=${gate.score.toFixed(3)} PARSE-SUPPRESSED`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const events = detector.parse(frame, t, gate);
|
||||
scheduler.recordParse(detector.id, t, events);
|
||||
console.log(
|
||||
`${t.toFixed(2)} ${file} ${detector.id} gate=${gate.score.toFixed(3)} events=[${events
|
||||
.map((e) => `${e.type}@${e.confidence.toFixed(3)}`)
|
||||
.join(", ")}]`,
|
||||
);
|
||||
}
|
||||
frame.delete();
|
||||
}
|
||||
Reference in New Issue
Block a user