diff --git a/app/features/scanner/README.md b/app/features/scanner/README.md index 49d3504a4..86ab5cf2a 100644 --- a/app/features/scanner/README.md +++ b/app/features/scanner/README.md @@ -154,7 +154,12 @@ sequenceDiagram are in each detector's module header; accuracy-critical matching internals in `core/glyphs.ts` and `core/detectors/scoreboard/weapons.ts` — read those before touching - recognition code. + recognition code. Parse cost matters live (a stalled worker drops + frames): a CJK splash-tag name once cost tens of seconds per death + parse, which is why the death detector memoizes tag reads on a + downscaled tag signature (same killer recurs pixel-identical) and + `classifySegment` prescreens oversized eligibility lists at half scale + — both tuned so `scanner:report` stays bit-identical. - Scheduling (`core/detectors/scheduler.ts`): the per-session DetectorScheduler decides which detectors see a frame. Failing gates are re-checked every `searchIntervalS` (0.25s — produced VoDs cut screens to @@ -173,6 +178,11 @@ sequenceDiagram collection and the panel stays hidden. A match's objective reads render as one step-line timeline (`~/components/ObjectiveTimeline.tsx`, shared with the match page). + The Live tab buffers frames sampled while the worker is busy; past the + buffer limit the backlog is decimated toward even time-spacing + (`worker/frame-queue.ts`) rather than truncated oldest-first, so a + parse stall can no longer swallow a results screen whole (the exact + failure that cost a live match its scoreboard on 2026-08-22). - VoD scans (`components/VodPage.tsx`): on the WebCodecs path each worker demuxes + decodes its own contiguous slice (mediabunny in the worker — no frames cross the main thread). When the scheduler reports calm (no gate diff --git a/app/features/scanner/components/LivePage.tsx b/app/features/scanner/components/LivePage.tsx index 04918308b..5e496b2cf 100644 --- a/app/features/scanner/components/LivePage.tsx +++ b/app/features/scanner/components/LivePage.tsx @@ -65,10 +65,13 @@ import { thumbnailFromBlob } from "./thumbnail"; const SAMPLE_FPS = 2; /** - * A battle-log parse streak can occupy the worker for seconds; buffering the - * frames sampled meanwhile (analyzed late, VoD-style) keeps quickly browsed - * entries from being missed. 24 frames = ~12s of backlog before the oldest - * frame is dropped. + * A slow parse (a browsed battle-log entry, a CJK splash-tag name) can + * occupy the worker for seconds to tens of seconds; buffering the frames + * sampled meanwhile (analyzed late, VoD-style) keeps what happened during + * the stall from being missed. 24 frames hold ~12s at full density; past + * that the backlog is decimated toward even spacing over the whole stall + * (worker/frame-queue.ts) instead of dropping its oldest frames, so a + * results screen mid-stall survives as a few frames. */ const FRAME_QUEUE_LIMIT = 24; diff --git a/app/features/scanner/core/detectors/death/index.ts b/app/features/scanner/core/detectors/death/index.ts index 0136d934f..ad7db86e8 100644 --- a/app/features/scanner/core/detectors/death/index.ts +++ b/app/features/scanner/core/detectors/death/index.ts @@ -141,6 +141,31 @@ const TAG_NAME_REFINE_MIN_INK = 200; const TAG_SPLIT_MIN_FRACTION = 0.15; const TAG_SPLIT_MIN_CHANNEL_DISTANCE = 40; +/** + * The splash-tag read dominates parse cost — a CJK name OCRs against the + * full name atlas for tens of seconds, 90%+ of a slow death parse — and + * the same killer's tag recurs pixel-identical, both across the frames of + * one death's parse streak and across their later kills. Reads are + * memoized on a small downscaled signature of the leveled tag band: + * VoD-measured mean abs diffs are ≤1 between reads of one killer's tag + * (even across different deaths) and ≥95 between different killers, so + * the threshold has a wide margin on both sides. + */ +const TAG_MEMO_WIDTH = 48; +const TAG_MEMO_HEIGHT = 12; +const TAG_MEMO_MAX_MEAN_DIFF = 12; +const TAG_MEMO_MAX_ENTRIES = 16; +/** A failed read is not worth pinning onto every later frame of its tag. */ +const TAG_MEMO_MIN_CONFIDENCE = 0.5; + +interface TagNameRead { + name: string | null; + confidence: number; + raw: string; + background: [number, number, number] | null; + textColor: [number, number, number] | null; +} + interface WeaponCandidate { /** the full weapon line as this template renders it, e.g. "Durch Klecksroller" */ text: string; @@ -274,6 +299,45 @@ export function createDeathDetector( return inner; } + const tagMemo: { signature: Uint8Array; read: TagNameRead }[] = []; + + function tagSignature(inner: Mat): Uint8Array { + const small = new cv.Mat(); + cv.resize( + inner, + small, + new cv.Size(TAG_MEMO_WIDTH, TAG_MEMO_HEIGHT), + 0, + 0, + cv.INTER_AREA, + ); + const signature = new Uint8Array(small.data); + small.delete(); + return signature; + } + + /** Memoized read for a matching tag, freshened to the list's end. */ + function tagMemoLookup(signature: Uint8Array): TagNameRead | null { + for (let i = 0; i < tagMemo.length; i++) { + const entry = tagMemo[i]!; + let sum = 0; + for (let k = 0; k < signature.length; k++) { + sum += Math.abs(signature[k]! - entry.signature[k]!); + } + if (sum / signature.length <= TAG_MEMO_MAX_MEAN_DIFF) { + tagMemo.splice(i, 1); + tagMemo.push(entry); + return entry.read; + } + } + return null; + } + + function tagMemoStore(signature: Uint8Array, read: TagNameRead): void { + tagMemo.push({ signature, read }); + if (tagMemo.length > TAG_MEMO_MAX_ENTRIES) tagMemo.shift(); + } + /** Per-channel median color of `inner`, over pixels where mask(i) holds. */ function medianColor( inner: Mat, @@ -678,12 +742,16 @@ export function createDeathDetector( let nameRaw = ""; let tagBackground: [number, number, number] | null = null; let tagTextColor: [number, number, number] | null = null; + let nameMemoHit = false; if (tagNameGlyphs) { const spaceGap = Math.max( 7, Math.round(tagNameGlyphs.medianWidth * 0.55), ); const inner = levelTagInner(rgb); + const signature = tagSignature(inner); + const memoized = tagMemoLookup(signature); + nameMemoHit = memoized !== null; const readWithBackground = ( backgrounds: readonly [number, number, number][], ) => { @@ -718,52 +786,66 @@ export function createDeathDetector( return { parsed, background: backgrounds[0]!, textColor }; }; - const median = medianColor(inner); - const dominants = dominantColors(inner, 2); - const dominant = dominants[0]!.color; - const candidates: [number, number, number][][] = [[median]]; - if (dominant.some((c, i) => Math.abs(c - median[i]!) > 8)) - candidates.push([dominant]); - const second = dominants[1]; - if ( - second && - second.fraction >= TAG_SPLIT_MIN_FRACTION && - second.color.some( - (c, i) => Math.abs(c - dominant[i]!) > TAG_SPLIT_MIN_CHANNEL_DISTANCE, - ) - ) { - candidates.push([dominant, second.color]); - } - // an empty read never beats one with glyphs (an estimate landing on - // the text color blanks the band, and recognizeText scores a - // segment-less band confidence 1); near-tied confidences resolve to - // the longer read, since confidence is the *min* char score and - // erasing most of the name can still read the survivors immaculately - const NEAR_TIE = 0.03; - const beats = ( - a: { parsed: { name: string; confidence: number } }, - b: typeof a, - ) => { - const aRead = a.parsed.name.length > 0 ? 1 : 0; - const bRead = b.parsed.name.length > 0 ? 1 : 0; - if (aRead !== bRead) return aRead - bRead; - if (Math.abs(a.parsed.confidence - b.parsed.confidence) <= NEAR_TIE) { - return a.parsed.name.length - b.parsed.name.length; + let read = memoized; + if (read === null) { + const median = medianColor(inner); + const dominants = dominantColors(inner, 2); + const dominant = dominants[0]!.color; + const candidates: [number, number, number][][] = [[median]]; + if (dominant.some((c, i) => Math.abs(c - median[i]!) > 8)) + candidates.push([dominant]); + const second = dominants[1]; + if ( + second && + second.fraction >= TAG_SPLIT_MIN_FRACTION && + second.color.some( + (c, i) => + Math.abs(c - dominant[i]!) > TAG_SPLIT_MIN_CHANNEL_DISTANCE, + ) + ) { + candidates.push([dominant, second.color]); + } + // an empty read never beats one with glyphs (an estimate landing on + // the text color blanks the band, and recognizeText scores a + // segment-less band confidence 1); near-tied confidences resolve to + // the longer read, since confidence is the *min* char score and + // erasing most of the name can still read the survivors immaculately + const NEAR_TIE = 0.03; + const beats = ( + a: { parsed: { name: string; confidence: number } }, + b: typeof a, + ) => { + const aRead = a.parsed.name.length > 0 ? 1 : 0; + const bRead = b.parsed.name.length > 0 ? 1 : 0; + if (aRead !== bRead) return aRead - bRead; + if (Math.abs(a.parsed.confidence - b.parsed.confidence) <= NEAR_TIE) { + return a.parsed.name.length - b.parsed.name.length; + } + return a.parsed.confidence - b.parsed.confidence; + }; + let best = readWithBackground(candidates[0]!); + for (const backgrounds of candidates.slice(1)) { + const alt = readWithBackground(backgrounds); + if (beats(alt, best) > 0) best = alt; + } + read = { + name: best.parsed.name.length > 0 ? best.parsed.name : null, + confidence: best.parsed.confidence, + raw: best.parsed.raw.text, + background: best.background, + textColor: best.textColor, + }; + if (read.confidence >= TAG_MEMO_MIN_CONFIDENCE && read.name !== null) { + tagMemoStore(signature, read); } - return a.parsed.confidence - b.parsed.confidence; - }; - let best = readWithBackground(candidates[0]!); - for (const backgrounds of candidates.slice(1)) { - const alt = readWithBackground(backgrounds); - if (beats(alt, best) > 0) best = alt; } inner.delete(); - tagBackground = best.background; - tagTextColor = best.textColor; - nameRaw = best.parsed.raw.text; - if (best.parsed.name.length > 0) name = best.parsed.name; - nameConfidence = best.parsed.confidence; + tagBackground = read.background; + tagTextColor = read.textColor; + nameRaw = read.raw; + name = read.name; + nameConfidence = read.confidence; confidences.push(nameConfidence); } @@ -806,6 +888,7 @@ export function createDeathDetector( ), nameRaw, nameScore: nameConfidence, + nameMemoHit, tagBackground, tagTextColor, }, @@ -818,8 +901,10 @@ export function createDeathDetector( // Death merge window, so every parse it skips would merge anyway. // sufficientConfidence sits just under the measured clean-read floor // (fixtures 0.750-0.825, confirmed scan events 0.751+); the refine and - // stagnation overrides cap what a ~1.4s parse can cost when a dirty - // read never reaches it + // stagnation overrides cap what a parse can cost when a dirty read + // never reaches it, and the tag memo keeps the streak's repeat parses + // off the expensive name read (a first-sight CJK name runs tens of + // seconds; repeats must not) return { id: "death", refineIntervalS: 0.5, diff --git a/app/features/scanner/core/glyphs.ts b/app/features/scanner/core/glyphs.ts index 9de5993ac..9dabd4e1f 100644 --- a/app/features/scanner/core/glyphs.ts +++ b/app/features/scanner/core/glyphs.ts @@ -45,6 +45,8 @@ interface Glyph { ink: number; /** exact fixture crop vs font-rendered approximation */ source: "fixture" | "font"; + /** lazily-built PRESCREEN_SCALE thumbnail for the eligibility prescreen */ + small?: Mat; } export interface GlyphSet { @@ -297,6 +299,30 @@ function measureSegment(binary: Mat, seg: Segment): SegmentInfo { */ const FIXTURE_TIEBREAK = 0.02; +/** + * A CJK-charset segment leaves thousands of templates eligible with barely + * differing bounds (similar ink coverage and heights across the charset), + * so the bound-sorted early break never fires and every template pays a + * full matchTemplate — tens of seconds per segment. Above this eligibility + * count a half-scale NCC pass ranks the templates first (matchTemplate + * work scales with region area × template area, so ~16x cheaper) and only + * glyphs whose estimated score lands within PRESCREEN_MARGIN of the + * front-runner advance to full matching. The margin absorbs the low-res + * estimate's error and sits far beyond FIXTURE_TIEBREAK, so the tie-break + * pool survives; the estimate only prunes, never scores. Tuning is + * accuracy-first, verified against the full scanner:report — at these + * values the report is bit-identical to no-prescreen while a JP + * splash-tag read drops from ~21s to ~1.3s. Quarter scale is too coarse + * (20px glyphs land at ~5px where the NCC ranking turns to noise), a + * tighter margin loses real reads ('R' at 0.12, and stragglers survive + * past 0.22), and the keep cap is only a runaway backstop — capping at + * 256 cut true glyphs that sat within the margin. + */ +const PRESCREEN_MIN_ELIGIBLE = 200; +const PRESCREEN_SCALE = 0.5; +const PRESCREEN_MARGIN = 0.3; +const PRESCREEN_MAX_KEEP = 1024; + function classifySegment( masked: Mat, seg: SegmentInfo, @@ -336,14 +362,7 @@ function classifySegment( // matching runs. Matching in descending-bound order lets the loop stop as // soon as no remaining glyph could come within FIXTURE_TIEBREAK of the // best — those can neither win nor take part in the fixture tie-break. - const eligible: { - glyph: Glyph; - tRows: number; - tCols: number; - r: number; - hr: number; - bound: number; - }[] = []; + const eligible: EligibleGlyph[] = []; for (const glyph of set.glyphs) { const t = glyph.mat; const tRows = t.rows; @@ -392,8 +411,25 @@ function classifySegment( // so the loop can stop as soon as either answer is certain: no remaining // bound reaches the floor, or a computed score already cleared it. const probeMode = scoreFloor !== Number.NEGATIVE_INFINITY; + // a probe prunes against its own floor instead of the front-runner: its + // usual answer is "no glyph clears the floor", which otherwise costs a + // full match of every template whose loose bound exceeds it + const contenders = + eligible.length >= PRESCREEN_MIN_ELIGIBLE + ? prescreen( + region, + eligible, + { + x0, + segX0: seg.x0, + segX1: seg.x1, + minOverlap, + }, + probeMode ? scoreFloor : null, + ) + : eligible; let bestScore = scoreFloor; - for (const { glyph, tRows, tCols, r, hr, bound } of eligible) { + for (const { glyph, tRows, tCols, r, hr, bound } of contenders) { if (bound < bestScore - FIXTURE_TIEBREAK) break; if (probeMode && (bound <= scoreFloor || bestScore > scoreFloor)) break; cv.matchTemplate(region, glyph.mat, result, cv.TM_CCOEFF_NORMED); @@ -449,6 +485,131 @@ function classifySegment( return candidates.slice(0, 5); } +interface EligibleGlyph { + glyph: Glyph; + tRows: number; + tCols: number; + r: number; + hr: number; + bound: number; +} + +/** Low-res ranking pass over an oversized eligibility list; see the + * PRESCREEN_* constants for why and how survivors are chosen. `geometry` + * carries the caller's placement window in full-res coordinates: without + * the same min-overlap restriction the estimates suffer exactly the + * failure the full loop guards against — a template scoring on the + * neighboring glyph inside the pad — which inflates the front-runner and + * prunes the true glyph. */ +function prescreen( + region: Mat, + eligible: EligibleGlyph[], + geometry: { x0: number; segX0: number; segX1: number; minOverlap: number }, + /** probe mode: prune against this floor instead of the front-runner */ + probeFloor: number | null = null, +): EligibleGlyph[] { + const cv = getCV(); + const smallRegion = new cv.Mat(); + cv.resize( + region, + smallRegion, + scaledSize(region.cols, region.rows), + 0, + 0, + cv.INTER_AREA, + ); + const x0 = geometry.x0 * PRESCREEN_SCALE; + const segX0 = geometry.segX0 * PRESCREEN_SCALE; + const segX1 = geometry.segX1 * PRESCREEN_SCALE; + // the slack pixel keeps quantized low-res placements from cutting a + // boundary placement the full-res window allows + const minOverlap = geometry.minOverlap * PRESCREEN_SCALE - 1; + const result = new cv.Mat(); + // entries the low-res pass cannot estimate (template degenerate or no + // valid placement after scaling) are force-kept — but must stay out of + // the front-runner max, or their untightened bound (≈1) inflates the + // floor and prunes every genuinely estimated glyph + const kept: EligibleGlyph[] = []; + const scored: { entry: EligibleGlyph; est: number }[] = []; + for (const entry of eligible) { + const small = smallGlyph(entry.glyph); + if ( + small.rows < 2 || + small.cols < 2 || + small.rows > smallRegion.rows || + small.cols > smallRegion.cols + ) { + kept.push(entry); + continue; + } + cv.matchTemplate(smallRegion, small, result, cv.TM_CCOEFF_NORMED); + const rCols = smallRegion.cols - small.cols + 1; + const rRows = smallRegion.rows - small.rows + 1; + const overlapAt = (rx: number) => + Math.min(x0 + rx + small.cols, segX1) - Math.max(x0 + rx, segX0); + let lo = 0; + while (lo < rCols && overlapAt(lo) < minOverlap) lo++; + let hi = rCols - 1; + while (hi >= lo && overlapAt(hi) < minOverlap) hi--; + if (hi < lo) { + kept.push(entry); + continue; + } + let maxVal = Number.NEGATIVE_INFINITY; + const scores = result.data32F; + for (let ry = 0, rowBase = 0; ry < rRows; ry++, rowBase += rCols) { + for (let rx = lo; rx <= hi; rx++) { + const v = scores[rowBase + rx]!; + if (v > maxVal) maxVal = v; + } + } + scored.push({ entry, est: maxVal * entry.bound }); + } + result.delete(); + smallRegion.delete(); + scored.sort((a, b) => b.est - a.est); + if (scored.length > 0) { + const floor = + probeFloor !== null + ? probeFloor - PRESCREEN_MARGIN + : scored[0]!.est - PRESCREEN_MARGIN; + let taken = 0; + for (const { entry, est } of scored) { + if (est < floor || taken >= PRESCREEN_MAX_KEEP) break; + kept.push(entry); + taken++; + } + } + // the main matching loop's early break assumes descending bounds + kept.sort((a, b) => b.bound - a.bound); + return kept; +} + +function smallGlyph(glyph: Glyph): Mat { + if (!glyph.small) { + const cv = getCV(); + const small = new cv.Mat(); + cv.resize( + glyph.mat, + small, + scaledSize(glyph.mat.cols, glyph.mat.rows), + 0, + 0, + cv.INTER_AREA, + ); + glyph.small = small; + } + return glyph.small; +} + +function scaledSize(cols: number, rows: number) { + const cv = getCV(); + return new cv.Size( + Math.max(1, Math.round(cols * PRESCREEN_SCALE)), + Math.max(1, Math.round(rows * PRESCREEN_SCALE)), + ); +} + interface ClassifiedSegment { seg: SegmentInfo; ranked: ReturnType; diff --git a/app/features/scanner/tests/logic/frame-queue.test.ts b/app/features/scanner/tests/logic/frame-queue.test.ts new file mode 100644 index 000000000..ddae9f830 --- /dev/null +++ b/app/features/scanner/tests/logic/frame-queue.test.ts @@ -0,0 +1,52 @@ +/** + * Unit tests for the live frame backlog's decimating eviction: which index + * gets dropped, and the property the policy exists for — a long parse + * stall keeps thinned coverage of its whole span instead of truncating to + * the newest limit/fps seconds. + */ + +import assert from "node:assert/strict"; +import { frameEvictionIndex } from "../../worker/frame-queue"; +import test from "../node-test-compat"; + +test("evicts the oldest of a backlog too short to thin", () => { + assert.equal(frameEvictionIndex([]), 0); + assert.equal(frameEvictionIndex([1]), 0); + assert.equal(frameEvictionIndex([1, 2]), 0); +}); + +test("never evicts the endpoints", () => { + assert.equal(frameEvictionIndex([0, 100, 200]), 1); +}); + +test("evicts from the densest stretch", () => { + // 10..11 is dense; 0 and 30 anchor the span + assert.equal(frameEvictionIndex([0, 10, 10.5, 11, 30]), 2); +}); + +test("uniform spacing evicts the first interior frame", () => { + assert.equal(frameEvictionIndex([0, 1, 2, 3, 4]), 1); +}); + +test("a stall keeps thinned coverage of its whole span", () => { + const limit = 24; + const fps = 2; + const stallS = 50; + const queue: number[] = []; + for (let time = 0; time <= stallS; time += 1 / fps) { + queue.push(time); + if (queue.length > limit) { + queue.splice(frameEvictionIndex(queue), 1); + } + } + assert.equal(queue.length, limit); + assert.equal(queue[0], 0); + assert.equal(queue.at(-1), stallS); + let maxGap = 0; + for (let i = 1; i < queue.length; i++) { + maxGap = Math.max(maxGap, queue[i]! - queue[i - 1]!); + } + // drop-oldest would leave a 38.5s hole; decimation keeps every gap + // small enough that a ~10s results screen retains multiple frames + assert.ok(maxGap <= 4, `max gap ${maxGap}s`); +}); diff --git a/app/features/scanner/worker/client.ts b/app/features/scanner/worker/client.ts index adf56a1c9..d05af4732 100644 --- a/app/features/scanner/worker/client.ts +++ b/app/features/scanner/worker/client.ts @@ -12,6 +12,7 @@ */ import { Config } from "../../../config"; import type { ScanTelemetry } from "../core/detectors/telemetry"; +import { frameEvictionIndex } from "./frame-queue"; import type { WorkerResponse } from "./protocol"; export type ResultHandler = ( @@ -143,16 +144,19 @@ export class AnalyzerClient { /** * Analyze a frame now, or — with `frameQueueLimit` set — buffer it until - * the in-flight frame settles (past the limit the oldest buffered frame - * is dropped). Returns false (and closes the bitmap) only when the frame - * was dropped outright. + * the in-flight frame settles (past the limit the backlog is decimated: + * see frame-queue.ts). Returns false (and closes the bitmap) only when + * the frame was dropped outright. */ analyze(bitmap: ImageBitmap | VideoFrame, t: number): boolean { if (this.busy) { if (this.#frameQueueLimit > 0 && this.#ready) { this.#frameQueue.push({ bitmap, t }); if (this.#frameQueue.length > this.#frameQueueLimit) { - this.#frameQueue.shift()?.bitmap.close(); + const victim = frameEvictionIndex( + this.#frameQueue.map((frame) => frame.t), + ); + this.#frameQueue.splice(victim, 1)[0]?.bitmap.close(); } return true; } diff --git a/app/features/scanner/worker/frame-queue.ts b/app/features/scanner/worker/frame-queue.ts new file mode 100644 index 000000000..c1625d891 --- /dev/null +++ b/app/features/scanner/worker/frame-queue.ts @@ -0,0 +1,31 @@ +/** + * Eviction policy for the live capture's frame backlog. Dropping the oldest + * frame truncates the buffered window to limit/fps seconds, so a parse + * stall longer than that (a CJK splash-tag name read runs tens of seconds) + * silently discards whole screens — including the scoreboard that closes a + * match. Instead the backlog is decimated: evict the frame whose two + * neighbors sit closest together in time. Repeated evictions thin the + * buffer toward evenly spaced coverage of the whole stall, so a screen that + * appeared mid-stall keeps a few frames rather than losing all of them + * (measured: a 50s stall at 2fps/24 slots keeps a mid-stall 10s screen at + * 4 frames with a 3s worst gap). The oldest and newest frames are never + * evicted, so the covered span itself always survives. + */ + +/** + * Index of the frame to evict from a backlog of capture timestamps + * (ascending). Timestamps only — the caller owns the frames themselves. + */ +export function frameEvictionIndex(times: readonly number[]): number { + if (times.length < 3) return 0; + let best = 1; + let bestGap = Number.POSITIVE_INFINITY; + for (let i = 1; i < times.length - 1; i++) { + const gap = times[i + 1]! - times[i - 1]!; + if (gap < bestGap) { + bestGap = gap; + best = i; + } + } + return best; +}