Optimize Scanner live
Some checks are pending
E2E Tests / e2e (push) Waiting to run
Tests and checks on push / run-checks-and-tests (push) Waiting to run
Updates translation progress / update-translation-progress-issue (push) Waiting to run

This commit is contained in:
Kalle
2026-08-23 18:39:07 +03:00
parent ba4f47ffe4
commit 611aed4880
7 changed files with 408 additions and 62 deletions

View File

@@ -154,7 +154,12 @@ sequenceDiagram
are in each detector's module
header; accuracy-critical matching internals in `core/glyphs.ts` and
`core/detectors/scoreboard/weapons.ts` — read those before touching
recognition code.
recognition code. Parse cost matters live (a stalled worker drops
frames): a CJK splash-tag name once cost tens of seconds per death
parse, which is why the death detector memoizes tag reads on a
downscaled tag signature (same killer recurs pixel-identical) and
`classifySegment` prescreens oversized eligibility lists at half scale
— both tuned so `scanner:report` stays bit-identical.
- Scheduling (`core/detectors/scheduler.ts`): the per-session
DetectorScheduler decides which detectors see a frame. Failing gates are
re-checked every `searchIntervalS` (0.25s — produced VoDs cut screens to
@@ -173,6 +178,11 @@ sequenceDiagram
collection and the panel stays hidden. A match's objective reads render
as one step-line timeline
(`~/components/ObjectiveTimeline.tsx`, shared with the match page).
The Live tab buffers frames sampled while the worker is busy; past the
buffer limit the backlog is decimated toward even time-spacing
(`worker/frame-queue.ts`) rather than truncated oldest-first, so a
parse stall can no longer swallow a results screen whole (the exact
failure that cost a live match its scoreboard on 2026-08-22).
- VoD scans (`components/VodPage.tsx`): on the WebCodecs path each worker
demuxes + decodes its own contiguous slice (mediabunny in the worker — no
frames cross the main thread). When the scheduler reports calm (no gate

View File

@@ -65,10 +65,13 @@ import { thumbnailFromBlob } from "./thumbnail";
const SAMPLE_FPS = 2;
/**
* A battle-log parse streak can occupy the worker for seconds; buffering the
* frames sampled meanwhile (analyzed late, VoD-style) keeps quickly browsed
* entries from being missed. 24 frames = ~12s of backlog before the oldest
* frame is dropped.
* A slow parse (a browsed battle-log entry, a CJK splash-tag name) can
* occupy the worker for seconds to tens of seconds; buffering the frames
* sampled meanwhile (analyzed late, VoD-style) keeps what happened during
* the stall from being missed. 24 frames hold ~12s at full density; past
* that the backlog is decimated toward even spacing over the whole stall
* (worker/frame-queue.ts) instead of dropping its oldest frames, so a
* results screen mid-stall survives as a few frames.
*/
const FRAME_QUEUE_LIMIT = 24;

View File

@@ -141,6 +141,31 @@ const TAG_NAME_REFINE_MIN_INK = 200;
const TAG_SPLIT_MIN_FRACTION = 0.15;
const TAG_SPLIT_MIN_CHANNEL_DISTANCE = 40;
/**
* The splash-tag read dominates parse cost — a CJK name OCRs against the
* full name atlas for tens of seconds, 90%+ of a slow death parse — and
* the same killer's tag recurs pixel-identical, both across the frames of
* one death's parse streak and across their later kills. Reads are
* memoized on a small downscaled signature of the leveled tag band:
* VoD-measured mean abs diffs are ≤1 between reads of one killer's tag
* (even across different deaths) and ≥95 between different killers, so
* the threshold has a wide margin on both sides.
*/
const TAG_MEMO_WIDTH = 48;
const TAG_MEMO_HEIGHT = 12;
const TAG_MEMO_MAX_MEAN_DIFF = 12;
const TAG_MEMO_MAX_ENTRIES = 16;
/** A failed read is not worth pinning onto every later frame of its tag. */
const TAG_MEMO_MIN_CONFIDENCE = 0.5;
interface TagNameRead {
name: string | null;
confidence: number;
raw: string;
background: [number, number, number] | null;
textColor: [number, number, number] | null;
}
interface WeaponCandidate {
/** the full weapon line as this template renders it, e.g. "Durch Klecksroller" */
text: string;
@@ -274,6 +299,45 @@ export function createDeathDetector(
return inner;
}
const tagMemo: { signature: Uint8Array; read: TagNameRead }[] = [];
function tagSignature(inner: Mat): Uint8Array {
const small = new cv.Mat();
cv.resize(
inner,
small,
new cv.Size(TAG_MEMO_WIDTH, TAG_MEMO_HEIGHT),
0,
0,
cv.INTER_AREA,
);
const signature = new Uint8Array(small.data);
small.delete();
return signature;
}
/** Memoized read for a matching tag, freshened to the list's end. */
function tagMemoLookup(signature: Uint8Array): TagNameRead | null {
for (let i = 0; i < tagMemo.length; i++) {
const entry = tagMemo[i]!;
let sum = 0;
for (let k = 0; k < signature.length; k++) {
sum += Math.abs(signature[k]! - entry.signature[k]!);
}
if (sum / signature.length <= TAG_MEMO_MAX_MEAN_DIFF) {
tagMemo.splice(i, 1);
tagMemo.push(entry);
return entry.read;
}
}
return null;
}
function tagMemoStore(signature: Uint8Array, read: TagNameRead): void {
tagMemo.push({ signature, read });
if (tagMemo.length > TAG_MEMO_MAX_ENTRIES) tagMemo.shift();
}
/** Per-channel median color of `inner`, over pixels where mask(i) holds. */
function medianColor(
inner: Mat,
@@ -678,12 +742,16 @@ export function createDeathDetector(
let nameRaw = "";
let tagBackground: [number, number, number] | null = null;
let tagTextColor: [number, number, number] | null = null;
let nameMemoHit = false;
if (tagNameGlyphs) {
const spaceGap = Math.max(
7,
Math.round(tagNameGlyphs.medianWidth * 0.55),
);
const inner = levelTagInner(rgb);
const signature = tagSignature(inner);
const memoized = tagMemoLookup(signature);
nameMemoHit = memoized !== null;
const readWithBackground = (
backgrounds: readonly [number, number, number][],
) => {
@@ -718,52 +786,66 @@ export function createDeathDetector(
return { parsed, background: backgrounds[0]!, textColor };
};
const median = medianColor(inner);
const dominants = dominantColors(inner, 2);
const dominant = dominants[0]!.color;
const candidates: [number, number, number][][] = [[median]];
if (dominant.some((c, i) => Math.abs(c - median[i]!) > 8))
candidates.push([dominant]);
const second = dominants[1];
if (
second &&
second.fraction >= TAG_SPLIT_MIN_FRACTION &&
second.color.some(
(c, i) => Math.abs(c - dominant[i]!) > TAG_SPLIT_MIN_CHANNEL_DISTANCE,
)
) {
candidates.push([dominant, second.color]);
}
// an empty read never beats one with glyphs (an estimate landing on
// the text color blanks the band, and recognizeText scores a
// segment-less band confidence 1); near-tied confidences resolve to
// the longer read, since confidence is the *min* char score and
// erasing most of the name can still read the survivors immaculately
const NEAR_TIE = 0.03;
const beats = (
a: { parsed: { name: string; confidence: number } },
b: typeof a,
) => {
const aRead = a.parsed.name.length > 0 ? 1 : 0;
const bRead = b.parsed.name.length > 0 ? 1 : 0;
if (aRead !== bRead) return aRead - bRead;
if (Math.abs(a.parsed.confidence - b.parsed.confidence) <= NEAR_TIE) {
return a.parsed.name.length - b.parsed.name.length;
let read = memoized;
if (read === null) {
const median = medianColor(inner);
const dominants = dominantColors(inner, 2);
const dominant = dominants[0]!.color;
const candidates: [number, number, number][][] = [[median]];
if (dominant.some((c, i) => Math.abs(c - median[i]!) > 8))
candidates.push([dominant]);
const second = dominants[1];
if (
second &&
second.fraction >= TAG_SPLIT_MIN_FRACTION &&
second.color.some(
(c, i) =>
Math.abs(c - dominant[i]!) > TAG_SPLIT_MIN_CHANNEL_DISTANCE,
)
) {
candidates.push([dominant, second.color]);
}
// an empty read never beats one with glyphs (an estimate landing on
// the text color blanks the band, and recognizeText scores a
// segment-less band confidence 1); near-tied confidences resolve to
// the longer read, since confidence is the *min* char score and
// erasing most of the name can still read the survivors immaculately
const NEAR_TIE = 0.03;
const beats = (
a: { parsed: { name: string; confidence: number } },
b: typeof a,
) => {
const aRead = a.parsed.name.length > 0 ? 1 : 0;
const bRead = b.parsed.name.length > 0 ? 1 : 0;
if (aRead !== bRead) return aRead - bRead;
if (Math.abs(a.parsed.confidence - b.parsed.confidence) <= NEAR_TIE) {
return a.parsed.name.length - b.parsed.name.length;
}
return a.parsed.confidence - b.parsed.confidence;
};
let best = readWithBackground(candidates[0]!);
for (const backgrounds of candidates.slice(1)) {
const alt = readWithBackground(backgrounds);
if (beats(alt, best) > 0) best = alt;
}
read = {
name: best.parsed.name.length > 0 ? best.parsed.name : null,
confidence: best.parsed.confidence,
raw: best.parsed.raw.text,
background: best.background,
textColor: best.textColor,
};
if (read.confidence >= TAG_MEMO_MIN_CONFIDENCE && read.name !== null) {
tagMemoStore(signature, read);
}
return a.parsed.confidence - b.parsed.confidence;
};
let best = readWithBackground(candidates[0]!);
for (const backgrounds of candidates.slice(1)) {
const alt = readWithBackground(backgrounds);
if (beats(alt, best) > 0) best = alt;
}
inner.delete();
tagBackground = best.background;
tagTextColor = best.textColor;
nameRaw = best.parsed.raw.text;
if (best.parsed.name.length > 0) name = best.parsed.name;
nameConfidence = best.parsed.confidence;
tagBackground = read.background;
tagTextColor = read.textColor;
nameRaw = read.raw;
name = read.name;
nameConfidence = read.confidence;
confidences.push(nameConfidence);
}
@@ -806,6 +888,7 @@ export function createDeathDetector(
),
nameRaw,
nameScore: nameConfidence,
nameMemoHit,
tagBackground,
tagTextColor,
},
@@ -818,8 +901,10 @@ export function createDeathDetector(
// Death merge window, so every parse it skips would merge anyway.
// sufficientConfidence sits just under the measured clean-read floor
// (fixtures 0.750-0.825, confirmed scan events 0.751+); the refine and
// stagnation overrides cap what a ~1.4s parse can cost when a dirty
// read never reaches it
// stagnation overrides cap what a parse can cost when a dirty read
// never reaches it, and the tag memo keeps the streak's repeat parses
// off the expensive name read (a first-sight CJK name runs tens of
// seconds; repeats must not)
return {
id: "death",
refineIntervalS: 0.5,

View File

@@ -45,6 +45,8 @@ interface Glyph {
ink: number;
/** exact fixture crop vs font-rendered approximation */
source: "fixture" | "font";
/** lazily-built PRESCREEN_SCALE thumbnail for the eligibility prescreen */
small?: Mat;
}
export interface GlyphSet {
@@ -297,6 +299,30 @@ function measureSegment(binary: Mat, seg: Segment): SegmentInfo {
*/
const FIXTURE_TIEBREAK = 0.02;
/**
* A CJK-charset segment leaves thousands of templates eligible with barely
* differing bounds (similar ink coverage and heights across the charset),
* so the bound-sorted early break never fires and every template pays a
* full matchTemplate — tens of seconds per segment. Above this eligibility
* count a half-scale NCC pass ranks the templates first (matchTemplate
* work scales with region area × template area, so ~16x cheaper) and only
* glyphs whose estimated score lands within PRESCREEN_MARGIN of the
* front-runner advance to full matching. The margin absorbs the low-res
* estimate's error and sits far beyond FIXTURE_TIEBREAK, so the tie-break
* pool survives; the estimate only prunes, never scores. Tuning is
* accuracy-first, verified against the full scanner:report — at these
* values the report is bit-identical to no-prescreen while a JP
* splash-tag read drops from ~21s to ~1.3s. Quarter scale is too coarse
* (20px glyphs land at ~5px where the NCC ranking turns to noise), a
* tighter margin loses real reads ('R' at 0.12, and stragglers survive
* past 0.22), and the keep cap is only a runaway backstop — capping at
* 256 cut true glyphs that sat within the margin.
*/
const PRESCREEN_MIN_ELIGIBLE = 200;
const PRESCREEN_SCALE = 0.5;
const PRESCREEN_MARGIN = 0.3;
const PRESCREEN_MAX_KEEP = 1024;
function classifySegment(
masked: Mat,
seg: SegmentInfo,
@@ -336,14 +362,7 @@ function classifySegment(
// matching runs. Matching in descending-bound order lets the loop stop as
// soon as no remaining glyph could come within FIXTURE_TIEBREAK of the
// best — those can neither win nor take part in the fixture tie-break.
const eligible: {
glyph: Glyph;
tRows: number;
tCols: number;
r: number;
hr: number;
bound: number;
}[] = [];
const eligible: EligibleGlyph[] = [];
for (const glyph of set.glyphs) {
const t = glyph.mat;
const tRows = t.rows;
@@ -392,8 +411,25 @@ function classifySegment(
// so the loop can stop as soon as either answer is certain: no remaining
// bound reaches the floor, or a computed score already cleared it.
const probeMode = scoreFloor !== Number.NEGATIVE_INFINITY;
// a probe prunes against its own floor instead of the front-runner: its
// usual answer is "no glyph clears the floor", which otherwise costs a
// full match of every template whose loose bound exceeds it
const contenders =
eligible.length >= PRESCREEN_MIN_ELIGIBLE
? prescreen(
region,
eligible,
{
x0,
segX0: seg.x0,
segX1: seg.x1,
minOverlap,
},
probeMode ? scoreFloor : null,
)
: eligible;
let bestScore = scoreFloor;
for (const { glyph, tRows, tCols, r, hr, bound } of eligible) {
for (const { glyph, tRows, tCols, r, hr, bound } of contenders) {
if (bound < bestScore - FIXTURE_TIEBREAK) break;
if (probeMode && (bound <= scoreFloor || bestScore > scoreFloor)) break;
cv.matchTemplate(region, glyph.mat, result, cv.TM_CCOEFF_NORMED);
@@ -449,6 +485,131 @@ function classifySegment(
return candidates.slice(0, 5);
}
interface EligibleGlyph {
glyph: Glyph;
tRows: number;
tCols: number;
r: number;
hr: number;
bound: number;
}
/** Low-res ranking pass over an oversized eligibility list; see the
* PRESCREEN_* constants for why and how survivors are chosen. `geometry`
* carries the caller's placement window in full-res coordinates: without
* the same min-overlap restriction the estimates suffer exactly the
* failure the full loop guards against — a template scoring on the
* neighboring glyph inside the pad — which inflates the front-runner and
* prunes the true glyph. */
function prescreen(
region: Mat,
eligible: EligibleGlyph[],
geometry: { x0: number; segX0: number; segX1: number; minOverlap: number },
/** probe mode: prune against this floor instead of the front-runner */
probeFloor: number | null = null,
): EligibleGlyph[] {
const cv = getCV();
const smallRegion = new cv.Mat();
cv.resize(
region,
smallRegion,
scaledSize(region.cols, region.rows),
0,
0,
cv.INTER_AREA,
);
const x0 = geometry.x0 * PRESCREEN_SCALE;
const segX0 = geometry.segX0 * PRESCREEN_SCALE;
const segX1 = geometry.segX1 * PRESCREEN_SCALE;
// the slack pixel keeps quantized low-res placements from cutting a
// boundary placement the full-res window allows
const minOverlap = geometry.minOverlap * PRESCREEN_SCALE - 1;
const result = new cv.Mat();
// entries the low-res pass cannot estimate (template degenerate or no
// valid placement after scaling) are force-kept — but must stay out of
// the front-runner max, or their untightened bound (≈1) inflates the
// floor and prunes every genuinely estimated glyph
const kept: EligibleGlyph[] = [];
const scored: { entry: EligibleGlyph; est: number }[] = [];
for (const entry of eligible) {
const small = smallGlyph(entry.glyph);
if (
small.rows < 2 ||
small.cols < 2 ||
small.rows > smallRegion.rows ||
small.cols > smallRegion.cols
) {
kept.push(entry);
continue;
}
cv.matchTemplate(smallRegion, small, result, cv.TM_CCOEFF_NORMED);
const rCols = smallRegion.cols - small.cols + 1;
const rRows = smallRegion.rows - small.rows + 1;
const overlapAt = (rx: number) =>
Math.min(x0 + rx + small.cols, segX1) - Math.max(x0 + rx, segX0);
let lo = 0;
while (lo < rCols && overlapAt(lo) < minOverlap) lo++;
let hi = rCols - 1;
while (hi >= lo && overlapAt(hi) < minOverlap) hi--;
if (hi < lo) {
kept.push(entry);
continue;
}
let maxVal = Number.NEGATIVE_INFINITY;
const scores = result.data32F;
for (let ry = 0, rowBase = 0; ry < rRows; ry++, rowBase += rCols) {
for (let rx = lo; rx <= hi; rx++) {
const v = scores[rowBase + rx]!;
if (v > maxVal) maxVal = v;
}
}
scored.push({ entry, est: maxVal * entry.bound });
}
result.delete();
smallRegion.delete();
scored.sort((a, b) => b.est - a.est);
if (scored.length > 0) {
const floor =
probeFloor !== null
? probeFloor - PRESCREEN_MARGIN
: scored[0]!.est - PRESCREEN_MARGIN;
let taken = 0;
for (const { entry, est } of scored) {
if (est < floor || taken >= PRESCREEN_MAX_KEEP) break;
kept.push(entry);
taken++;
}
}
// the main matching loop's early break assumes descending bounds
kept.sort((a, b) => b.bound - a.bound);
return kept;
}
function smallGlyph(glyph: Glyph): Mat {
if (!glyph.small) {
const cv = getCV();
const small = new cv.Mat();
cv.resize(
glyph.mat,
small,
scaledSize(glyph.mat.cols, glyph.mat.rows),
0,
0,
cv.INTER_AREA,
);
glyph.small = small;
}
return glyph.small;
}
function scaledSize(cols: number, rows: number) {
const cv = getCV();
return new cv.Size(
Math.max(1, Math.round(cols * PRESCREEN_SCALE)),
Math.max(1, Math.round(rows * PRESCREEN_SCALE)),
);
}
interface ClassifiedSegment {
seg: SegmentInfo;
ranked: ReturnType<typeof classifySegment>;

View File

@@ -0,0 +1,52 @@
/**
* Unit tests for the live frame backlog's decimating eviction: which index
* gets dropped, and the property the policy exists for — a long parse
* stall keeps thinned coverage of its whole span instead of truncating to
* the newest limit/fps seconds.
*/
import assert from "node:assert/strict";
import { frameEvictionIndex } from "../../worker/frame-queue";
import test from "../node-test-compat";
test("evicts the oldest of a backlog too short to thin", () => {
assert.equal(frameEvictionIndex([]), 0);
assert.equal(frameEvictionIndex([1]), 0);
assert.equal(frameEvictionIndex([1, 2]), 0);
});
test("never evicts the endpoints", () => {
assert.equal(frameEvictionIndex([0, 100, 200]), 1);
});
test("evicts from the densest stretch", () => {
// 10..11 is dense; 0 and 30 anchor the span
assert.equal(frameEvictionIndex([0, 10, 10.5, 11, 30]), 2);
});
test("uniform spacing evicts the first interior frame", () => {
assert.equal(frameEvictionIndex([0, 1, 2, 3, 4]), 1);
});
test("a stall keeps thinned coverage of its whole span", () => {
const limit = 24;
const fps = 2;
const stallS = 50;
const queue: number[] = [];
for (let time = 0; time <= stallS; time += 1 / fps) {
queue.push(time);
if (queue.length > limit) {
queue.splice(frameEvictionIndex(queue), 1);
}
}
assert.equal(queue.length, limit);
assert.equal(queue[0], 0);
assert.equal(queue.at(-1), stallS);
let maxGap = 0;
for (let i = 1; i < queue.length; i++) {
maxGap = Math.max(maxGap, queue[i]! - queue[i - 1]!);
}
// drop-oldest would leave a 38.5s hole; decimation keeps every gap
// small enough that a ~10s results screen retains multiple frames
assert.ok(maxGap <= 4, `max gap ${maxGap}s`);
});

View File

@@ -12,6 +12,7 @@
*/
import { Config } from "../../../config";
import type { ScanTelemetry } from "../core/detectors/telemetry";
import { frameEvictionIndex } from "./frame-queue";
import type { WorkerResponse } from "./protocol";
export type ResultHandler = (
@@ -143,16 +144,19 @@ export class AnalyzerClient {
/**
* Analyze a frame now, or — with `frameQueueLimit` set — buffer it until
* the in-flight frame settles (past the limit the oldest buffered frame
* is dropped). Returns false (and closes the bitmap) only when the frame
* was dropped outright.
* the in-flight frame settles (past the limit the backlog is decimated:
* see frame-queue.ts). Returns false (and closes the bitmap) only when
* the frame was dropped outright.
*/
analyze(bitmap: ImageBitmap | VideoFrame, t: number): boolean {
if (this.busy) {
if (this.#frameQueueLimit > 0 && this.#ready) {
this.#frameQueue.push({ bitmap, t });
if (this.#frameQueue.length > this.#frameQueueLimit) {
this.#frameQueue.shift()?.bitmap.close();
const victim = frameEvictionIndex(
this.#frameQueue.map((frame) => frame.t),
);
this.#frameQueue.splice(victim, 1)[0]?.bitmap.close();
}
return true;
}

View File

@@ -0,0 +1,31 @@
/**
* Eviction policy for the live capture's frame backlog. Dropping the oldest
* frame truncates the buffered window to limit/fps seconds, so a parse
* stall longer than that (a CJK splash-tag name read runs tens of seconds)
* silently discards whole screens — including the scoreboard that closes a
* match. Instead the backlog is decimated: evict the frame whose two
* neighbors sit closest together in time. Repeated evictions thin the
* buffer toward evenly spaced coverage of the whole stall, so a screen that
* appeared mid-stall keeps a few frames rather than losing all of them
* (measured: a 50s stall at 2fps/24 slots keeps a mid-stall 10s screen at
* 4 frames with a 3s worst gap). The oldest and newest frames are never
* evicted, so the covered span itself always survives.
*/
/**
* Index of the frame to evict from a backlog of capture timestamps
* (ascending). Timestamps only — the caller owns the frames themselves.
*/
export function frameEvictionIndex(times: readonly number[]): number {
if (times.length < 3) return 0;
let best = 1;
let bestGap = Number.POSITIVE_INFINITY;
for (let i = 1; i < times.length - 1; i++) {
const gap = times[i + 1]! - times[i - 1]!;
if (gap < bestGap) {
bestGap = gap;
best = i;
}
}
return best;
}