Leahnaya/sendou.ink
1
0
Fork 0
mirror of https://github.com/Sendouc/sendou.ink.git synced 2026-08-21 18:44:51 -05:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Import emberz CV parser core into app/features/cv

Browse Source 
Wholesale copy of core/worker/capture/store/node from the emberz repo with
relative imports stripped of .ts extensions. Atlases and template sets now
load from the assets repo CDN (assets/cv/v1): the worker receives the base
URL via its init message, Node tests/tools read the local assets checkout
(CV_ASSETS_DIR override).
This commit is contained in:
Kalle
2026-07-19 12:56:11 +03:00
parent a6bf55fe1a
commit fa519daffa
61 changed files with 21144 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
app/features/cv/capture/sampler.ts Normal file
View File

@@ -0,0 +1,71 @@
/**
* Capture layer: OBS Virtual Camera in via getUserMedia, frames out as
* ImageBitmaps at a low sample rate. The interface downstream is just
* (bitmap, t) — a WHIP/MediaMTX transport can replace this file later.
*/
export async function openVirtualCamera(deviceId?: string): Promise<MediaStream> {
return navigator.mediaDevices.getUserMedia({
video: {
deviceId: deviceId ? { exact: deviceId } : undefined,
width: { ideal: 1920 },
height: { ideal: 1080 },
},
audio: false,
});
}
export async function listVideoInputs(): Promise<MediaDeviceInfo[]> {
const devices = await navigator.mediaDevices.enumerateDevices();
return devices.filter((d) => d.kind === "videoinput");
}
export type FrameHandler = (bitmap: ImageBitmap, t: number) => void;
/**
* Sample frames from a playing video element at ~fps using
* requestVideoFrameCallback. Returns a stop function.
*/
export function startSampler(
video: HTMLVideoElement,
fps: number,
onFrame: FrameHandler,
): () => void {
const intervalMs = 1000 / fps;
let lastSample = -Infinity;
let lastMediaTime = -Infinity;
let stopped = false;
let handle = 0;
const tick = async (now: number, metadata: VideoFrameCallbackMetadata) => {
if (stopped) return;
// Throttle on the callback clock, not metadata.mediaTime: Firefox never
// advances mediaTime for MediaStream-backed videos, which would freeze
// sampling after the first frame.
if (now - lastSample >= intervalMs) {
lastSample = now;
try {
const bitmap = await createImageBitmap(video);
if (stopped) {
bitmap.close();
return;
}
// Same Firefox quirk for the frame timestamp: fall back to the clock
// when mediaTime isn't advancing so timestamps stay monotonic (the
// timeline's merge windows compare them).
const t = metadata.mediaTime > lastMediaTime ? metadata.mediaTime : now / 1000;
lastMediaTime = Math.max(lastMediaTime, metadata.mediaTime);
onFrame(bitmap, t);
} catch {
// video not ready / tab hidden — skip this frame
}
}
if (!stopped) handle = video.requestVideoFrameCallback(tick);
};
handle = video.requestVideoFrameCallback(tick);
return () => {
stopped = true;
video.cancelVideoFrameCallback(handle);
};
}

110
app/features/cv/capture/vod-frames.ts Normal file
View File

@@ -0,0 +1,110 @@
/**
* VoD frame extraction: step through a video file yielding (frame, t) as
* fast as decoding allows — no real-time playback. The primary path demuxes
* the file and decodes **every frame** sequentially with WebCodecs (via
* mediabunny), yielding the VideoFrames themselves (transferable to the
* analyzer workers with no main-thread conversion); when the container/codec
* can't be read that way, it falls back to seek-stepping a <video> element
* at a small fixed step, which handles anything the browser can play at the
* cost of per-seek latency and frame-exact coverage.
*/
import { ALL_FORMATS, BlobSource, Input, VideoSampleSink } from "mediabunny";
/**
* Seek fallback step: a <video> element can't enumerate frames, so seek in
* increments small enough that anything but blink-and-miss overlays is caught.
*/
const SEEK_STEP_SECONDS = 0.25;
interface VodFrame {
/** the consumer owns the frame and must close() it */
frame: ImageBitmap | VideoFrame;
/** seconds into the video */
t: number;
}
export interface VodScan {
method: "webcodecs" | "seek";
duration: number;
frames: AsyncGenerator<VodFrame>;
dispose(): void;
}
/**
* Open a scan over `file`, yielding every decoded frame (or, on the seek
* fallback, one frame every SEEK_STEP_SECONDS). `video` must already have
* the file loaded (metadata not required yet); it is only driven by the
* seek fallback.
*/
export async function openVodScan(file: File, video: HTMLVideoElement): Promise<VodScan> {
const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
try {
const track = await input.getPrimaryVideoTrack();
if (track && (await track.canDecode())) {
const duration = await input.computeDuration([track]);
return {
method: "webcodecs",
duration,
frames: webCodecsFrames(input, new VideoSampleSink(track)),
dispose: () => input.dispose(),
};
}
input.dispose();
} catch {
input.dispose();
}
await loadMetadata(video);
if (!Number.isFinite(video.duration)) {
throw new Error("video has no known duration — cannot scan by seeking");
}
return {
method: "seek",
duration: video.duration,
frames: seekFrames(video),
dispose: () => {},
};
}
async function* webCodecsFrames(input: Input, sink: VideoSampleSink): AsyncGenerator<VodFrame> {
try {
for await (const sample of sink.samples()) {
if (!sample) continue;
const t = sample.timestamp;
const frame = sample.toVideoFrame();
sample.close();
yield { frame, t };
}
} finally {
input.dispose();
}
}
async function* seekFrames(video: HTMLVideoElement): AsyncGenerator<VodFrame> {
for (let t = 0; t < video.duration; t += SEEK_STEP_SECONDS) {
await seekTo(video, t);
yield { frame: await createImageBitmap(video), t };
}
}
function loadMetadata(video: HTMLVideoElement): Promise<void> {
return new Promise((resolve, reject) => {
if (video.readyState >= HTMLMediaElement.HAVE_METADATA) return resolve();
video.addEventListener("loadedmetadata", () => resolve(), { once: true });
video.addEventListener(
"error",
() => reject(new Error(video.error?.message || "cannot decode this file as video")),
{ once: true },
);
});
}
function seekTo(video: HTMLVideoElement, t: number): Promise<void> {
return new Promise((resolve) => {
if (video.currentTime === t && video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
return resolve();
}
video.addEventListener("seeked", () => resolve(), { once: true });
video.currentTime = t;
});
}

80
app/features/cv/core/ability-harvest.ts Normal file
View File

@@ -0,0 +1,80 @@
/**
* Connect death events to scoreboard players: a death overlay shows the
* killer's splash-tag name, weapon, and full gear-ability grid, so every
* death in a match reveals one enemy player's build. Deaths are attributed
* to the next scoreboard-type event in the timeline (a match's deaths
* always precede its results screen), and matched to a player row by name
* and weapon id.
*/
import { DEATH_EVENT_TYPE, type DeathData } from "./detectors/death/index";
import { SCOREBOARD_EVENT_TYPES } from "./detectors/registry";
import type { ScoreboardData, ScoreboardPlayer } from "./detectors/scoreboard/index";
import type { DetectedEvent } from "./detectors/types";
/** player row index (0-7) → [head, clothes, shoes] ability-id rows */
export type PlayerAbilityMap = Map<number, string[][]>;
/**
* Match a death's killer to a scoreboard player row. Both signals are OCR
* output, so neither is trusted alone unless it is unambiguous: a combined
* name+weapon hit wins, then a unique name hit, then a unique weapon hit
* (two players on the same weapon with a misread name stay unattributed).
*/
function matchPlayer(players: ScoreboardPlayer[], death: DeathData): number | null {
const name = death.name?.trim().toLowerCase() || null;
const indices = players.map((_, i) => i);
const byName = name ? indices.filter((i) => players[i]!.name.trim().toLowerCase() === name) : [];
// scoreboard rows carry main-weapon ids; a sub/special credit says
// nothing about which main the killer holds
const byWeapon =
death.weaponId !== null && death.weaponType === "MAIN"
? indices.filter((i) => players[i]!.weaponId === death.weaponId)
: [];
const both = byName.filter((i) => byWeapon.includes(i));
if (both.length > 0) return both[0]!;
if (byName.length === 1) return byName[0]!;
if (byWeapon.length === 1) return byWeapon[0]!;
return null;
}
/**
* Harvest the builds one match's death events reveal: each death with a
* readable ability grid is attributed to a scoreboard player row.
*/
export function harvestAbilities(
players: ScoreboardPlayer[],
deaths: readonly DeathData[],
): PlayerAbilityMap {
const abilities: PlayerAbilityMap = new Map();
for (const death of deaths) {
if (death.abilities.length === 0) continue;
const index = matchPlayer(players, death);
if (index !== null) abilities.set(index, death.abilities);
}
return abilities;
}
/**
* For each scoreboard/replay event, harvest abilities from the death events
* since the previous scoreboard. Keyed by event object identity; events
* without any attributed death are absent from the result.
*/
export function connectAbilities(
events: readonly DetectedEvent[],
): Map<DetectedEvent, PlayerAbilityMap> {
const sorted = [...events].sort((a, b) => a.t - b.t);
const result = new Map<DetectedEvent, PlayerAbilityMap>();
let pendingDeaths: DeathData[] = [];
for (const event of sorted) {
if (event.type === DEATH_EVENT_TYPE) {
pendingDeaths.push(event.data as DeathData);
} else if (SCOREBOARD_EVENT_TYPES.includes(event.type)) {
const players = (event.data as ScoreboardData).players;
const abilities = harvestAbilities(players, pendingDeaths);
if (abilities.size > 0) result.set(event, abilities);
pendingDeaths = [];
}
}
return result;
}

116
app/features/cv/core/batches.ts Normal file
View File

@@ -0,0 +1,116 @@
/**
* Group a detected-event timeline into per-match batches for sendou.ink's
* /ingest endpoint: a batch starts at a MapStart event and ends at the next
* scoreboard-type event, carrying the match's death events in between. When
* a scoreboard arrives with no preceding MapStart (the intro was missed),
* the deaths since the previous scoreboard that fall within the last 10
* minutes are taken as its match instead — anything older belongs to no
* known match and is dropped, as is a match whose results screen was never
* detected. Scoreboards whose lobby is readable and not "Private Battle"
* are dropped together with their batch — only tournament lobbies are worth
* sending. The batch's death events reveal enemy builds; they are attached
* to the terminating scoreboard's player rows as `abilities` before
* sending.
*/
import { harvestAbilities } from "./ability-harvest";
import { DEATH_EVENT_TYPE, type DeathData } from "./detectors/death/index";
import { MAP_START_EVENT_TYPE } from "./detectors/map-start/index";
import { SCOREBOARD_EVENT_TYPES } from "./detectors/registry";
import type { ScoreboardData, ScoreboardPlayer } from "./detectors/scoreboard/index";
import type { DetectedEvent } from "./detectors/types";
/** The lobby header value private battles (tournament games) carry. */
const TOURNAMENT_LOBBY = "Private Battle";
/**
* How far back a scoreboard with no preceding MapStart claims deaths as its
* match — matches run well under 10 minutes, so anything older is another
* (undelimited) match's.
*/
const FALLBACK_WINDOW_SECONDS = 600;
export interface IngestScoreboardPlayer extends ScoreboardPlayer {
/** [head, clothes, shoes] ability rows harvested from this match's death screens */
abilities?: string[][];
}
/**
* Splits a timeline into ingest batches. Only event types the /ingest
* endpoint accepts are included (MapStart, Death, Scoreboard,
* ScoreboardReplay); each batch's scoreboard players carry the abilities
* harvested from that batch's deaths.
*
* Generic so callers with richer event records (the UI's StoredEvent) keep
* their extra fields — batch members are the input objects themselves,
* except the terminating scoreboard, which is shallow-copied for
* enrichment.
*/
export function buildIngestBatches<E extends DetectedEvent>(events: readonly E[]): E[][] {
const sorted = [...events].sort((a, b) => a.t - b.t);
const batches: E[][] = [];
let open: E[] | null = null;
// deaths since the last boundary with no MapStart to anchor them yet
let orphans: E[] = [];
for (const event of sorted) {
if (event.type === MAP_START_EVENT_TYPE) {
// a new match intro abandons any match whose scoreboard was missed
open = [event];
orphans = [];
} else if (SCOREBOARD_EVENT_TYPES.includes(event.type)) {
const data = event.data as ScoreboardData;
if (!data.lobby || data.lobby === TOURNAMENT_LOBBY) {
const matchEvents = open ?? orphans.filter((e) => event.t - e.t <= FALLBACK_WINDOW_SECONDS);
batches.push([...matchEvents, enrichScoreboard(event, matchEvents)]);
}
open = null;
orphans = [];
} else if (event.type === DEATH_EVENT_TYPE) {
(open ?? orphans).push(event);
}
}
return batches;
}
/**
* Groups whole batches into request-sized chunks of at most `maxEvents`
* events. Sending as many batches as fit in one request lets sendou.ink's
* content-based tournament resolution see the scoreboard *sequence* — a
* single match batch (one scoreboard) can't resolve by content. A batch is
* never split across chunks; an oversized lone batch gets its own chunk.
*/
export function chunkIngestBatches<E extends DetectedEvent>(
batches: readonly E[][],
maxEvents: number,
): E[][][] {
const chunks: E[][][] = [];
let current: E[][] = [];
let eventCount = 0;
for (const batch of batches) {
if (current.length > 0 && eventCount + batch.length > maxEvents) {
chunks.push(current);
current = [];
eventCount = 0;
}
current.push(batch);
eventCount += batch.length;
}
if (current.length > 0) chunks.push(current);
return chunks;
}
function enrichScoreboard<E extends DetectedEvent>(scoreboard: E, matchEvents: readonly E[]): E {
const deaths = matchEvents
.filter((e) => e.type === DEATH_EVENT_TYPE)
.map((e) => e.data as DeathData);
const data = scoreboard.data as ScoreboardData;
const abilities = harvestAbilities(data.players, deaths);
if (abilities.size === 0) return scoreboard;
const players: IngestScoreboardPlayer[] = data.players.map((player, i) => {
const build = abilities.get(i);
return build ? { ...player, abilities: build } : player;
});
return { ...scoreboard, data: { ...data, players } };
}

14
app/features/cv/core/canonical.ts Normal file
View File

@@ -0,0 +1,14 @@
/**
* Pure constants/types shared by UI and pipeline. No OpenCV dependency —
* the main-thread bundle must not pull in the WASM module (that lives in
* the worker).
*/
export const CANONICAL_WIDTH = 1920;
export const CANONICAL_HEIGHT = 1080;
export interface Roi {
x: number;
y: number;
w: number;
h: number;
}

66
app/features/cv/core/cv.ts Normal file
View File

@@ -0,0 +1,66 @@
/**
* OpenCV.js singleton loader. Works in Node, browser main thread, and workers —
* the UMD bundle embeds its WASM, so no asset paths are involved.
*
* Everything in core/ obtains the cv namespace through getCV(); callers must
* await loadOpenCV() once at startup (worker bootstrap, test setup, tool entry).
*/
import cvModule from "@techstark/opencv-js";
export type CV = typeof cvModule;
export type Mat = InstanceType<CV["Mat"]>;
let cvInstance: CV | null = null;
let loading: Promise<CV> | null = null;
export function loadOpenCV(): Promise<CV> {
if (cvInstance) return Promise.resolve(cvInstance);
if (loading) return loading;
const attempt = (async () => {
const mod = cvModule as unknown;
let cv: CV;
if (mod instanceof Promise) {
cv = await mod;
} else if ((mod as CV).Mat) {
cv = mod as CV;
} else {
await new Promise<void>((resolve) => {
(mod as { onRuntimeInitialized?: () => void }).onRuntimeInitialized = resolve;
});
cv = mod as CV;
}
cvInstance = cv;
return cv;
})();
// a failed load must not poison the singleton — clear it so callers can retry
loading = attempt.catch((error) => {
loading = null;
throw error;
});
return loading;
}
export function getCV(): CV {
if (!cvInstance) {
throw new Error("OpenCV not loaded — await loadOpenCV() before using core/");
}
return cvInstance;
}
// The bundled type definitions mark the optional mask argument of these as
// required; thin wrappers restore the real (mask-less) signatures.
export interface MinMaxResult {
minVal: number;
maxVal: number;
minLoc: { x: number; y: number };
maxLoc: { x: number; y: number };
}
export function minMaxLoc(mat: Mat): MinMaxResult {
return (getCV() as unknown as { minMaxLoc(m: Mat): MinMaxResult }).minMaxLoc(mat);
}
export function meanOf(mat: Mat): number[] {
return (getCV() as unknown as { mean(m: Mat): number[] }).mean(mat);
}

85
app/features/cv/core/detectors/death/abilities.ts Normal file
View File

@@ -0,0 +1,85 @@
/**
* Ability badge templates for the death-screen gear panel.
*
* The committed icon assets (assets/cv/abilities/) are the bare icon
* art with alpha; on screen each sits centered on a near-black circular
* badge that fills a known fraction of the slot. Templates are built by
* compositing the art onto a black square at that fraction, then resizing
* the square to the candidate badge sizes — after which they are shaped
* exactly like weapon templates and reuse matchWeapon for the NCC +
* ink-coverage scoring (see weapons.ts for why raw NCC is not enough).
*
* Mains (⌀~68) and subs (⌀~48) render the art at different fractions of
* the badge, so each role gets its own template set; the role ROIs' heights
* keep the other role's sizes out of the match (matchTemplate skips
* templates taller than the region).
*/
import { getCV } from "../../cv";
import type { FrameData } from "../../image";
import { buildTemplateSizes, type WeaponTemplate } from "../scoreboard/weapons";
import {
ABILITY_INK_THRESHOLD,
ABILITY_MAIN_ART_RATIO,
ABILITY_MAIN_SIZES,
ABILITY_SUB_ART_RATIO,
ABILITY_SUB_SIZES,
} from "./rois";
/** Badge interior brightness (near-black circle on the dark panel). */
const BADGE_BACKGROUND = 10;
export interface AbilityTemplates {
mains: WeaponTemplate[];
subs: WeaponTemplate[];
}
/**
* Exported for the calibration tooling's art-ratio sweeps and for other
* screens' badge sets (scoreboard-own builds its sizes/threshold here).
* inkThreshold must match the one passed to matchWeapon, or the coverage
* ratio compares mismatched ink counts.
*/
export function buildAbilityRole(
icons: { id: string; image: FrameData }[],
sizes: readonly number[],
artRatio: number,
inkThreshold: number = ABILITY_INK_THRESHOLD,
): WeaponTemplate[] {
const cv = getCV();
return icons.map(({ id, image }) => {
// composite the icon art over the badge black, padded so the art
// occupies artRatio of the square (as it does of the badge on screen)
const side = Math.round(image.width / artRatio);
const offset = Math.floor((side - image.width) / 2);
const padded = new cv.Mat(
side,
side,
cv.CV_8UC3,
new cv.Scalar(BADGE_BACKGROUND, BADGE_BACKGROUND, BADGE_BACKGROUND),
);
const dst = padded.data;
const src = image.data;
for (let y = 0; y < image.height; y++) {
for (let x = 0; x < image.width; x++) {
const si = (y * image.width + x) * 4;
const a = src[si + 3]! / 255;
const di = ((y + offset) * side + x + offset) * 3;
dst[di] = Math.round(src[si]! * a + BADGE_BACKGROUND * (1 - a));
dst[di + 1] = Math.round(src[si + 1]! * a + BADGE_BACKGROUND * (1 - a));
dst[di + 2] = Math.round(src[si + 2]! * a + BADGE_BACKGROUND * (1 - a));
}
}
const templateSizes = buildTemplateSizes(padded, sizes, inkThreshold);
padded.delete();
return { id, sizes: templateSizes };
});
}
export function prepareAbilityTemplates(
icons: { id: string; image: FrameData }[],
): AbilityTemplates {
return {
mains: buildAbilityRole(icons, ABILITY_MAIN_SIZES, ABILITY_MAIN_ART_RATIO),
subs: buildAbilityRole(icons, ABILITY_SUB_SIZES, ABILITY_SUB_ART_RATIO),
};
}

714
app/features/cv/core/detectors/death/index.ts Normal file
View File

@@ -0,0 +1,714 @@
/**
* DeathDetector: parses the death cam overlay shown while waiting to
* respawn — "Splatted by <weapon>!" in the top-center burst, the killer's
* gear abilities (3 rows x [main, sub, sub, sub]) in the bottom-left
* panel, and the killer's name from the tilted splash tag bottom-right.
*
* The weapon arrives primarily as text: both burst lines are OCR'd with
* the death-weapon atlas and matched against the per-language message
* templates (localized-messages.ts) — the weapon name sits on line 1 or 2
* depending on language ("Splatted by\n<weapon>!" vs "Durch <weapon>\n
* erledigt!") — then the weapon line is snapped to that language's weapon
* names and reported under its canonical English name. The constant line
* doubles as a parse-time confirmation: if it does not read back as any
* language's template, the gate hit was a lookalike frame and no event is
* emitted. When the weapon line is unreadable (the WIPEOUT banner covers
* it), the killer's weapon icon at the top of the burst is template-matched
* against the main-weapon set at burst size instead. Low-fidelity captures
* (720p upscaled) garble the read below the snap threshold while staying
* recoverable: a candidate-lattice re-rank (rankByRead) accepts on a
* decisive margin, and below that the burst icon and the text ranking
* corroborate each other (steps 2c/2d in parse).
*/
import { getCV, type Mat, minMaxLoc } from "../../cv";
import { type GlyphSet, type RecognizedText, recognizeText, scaleGlyphSet } from "../../glyphs";
import { copyRoi, cropRoi, meanBrightness, type Roi } from "../../image";
import { closestEntry, matchKey, rankBy, rankByRead } from "../../text";
import type { ScoreboardResources } from "../scoreboard/index";
import { parseName } from "../scoreboard/names";
import { matchWeapon, type WeaponMatch } from "../scoreboard/weapons";
import type { DetectedEvent, Detector, GateResult } from "../types";
import {
DEATH_MESSAGE_TEMPLATES,
type DeathMessageTemplate,
LOCALIZED_WEAPON_NAMES,
} from "./localized-messages";
import {
ABILITY_INK_THRESHOLD,
ABILITY_ROWS,
ABILITY_SLOT_MIN_INK,
ABILITY_SUB_XS,
abilityMainRoi,
abilitySubRoi,
BURST_ICON_ROI,
GATE_BURST_PROBES,
GATE_DARK_MAX_MEAN,
GATE_ICON_MIN_MAX,
GATE_PANEL_PROBES,
GATE_TEXT_MAX_FRACTION,
GATE_TEXT_MIN_MAX,
gateAbilityProbe,
JA_CONST_LINE_ROI,
JA_WEAPON_LINE_ROI,
SPLAT_LINE1_ROI,
SPLAT_TEXT_BIN_THRESHOLD,
TAG_NAME_INNER,
TAG_NAME_OUTER,
TAG_NAME_TEXT_HEIGHT,
TAG_TILT_DEG,
WEAPON_LINE_ROI,
WEAPON_TEXT_HEIGHT,
} from "./rois";
import { ALL_WEAPON_ENTRIES, type WeaponEntry, type WeaponType } from "./weapon-names";
export interface DeathData {
/** killer's weapon (English name, e.g. "Splattershot"); null if unreadable */
weapon: string | null;
/**
* the weapon's in-game id, unique only within its kind (MAIN ids are the
* assets/cv/main-weapons id space; SUB/SPECIAL ids are sendou.ink's)
*/
weaponId: number | null;
/** which kind of weapon got the splat; null when the weapon is unreadable */
weaponType: WeaponType | null;
/**
* killer's gear abilities, [head, clothes, shoes] rows of
* [main, sub...] ability ids (assets/cv/abilities id space); rows
* carry as many sub entries as the gear has slots (1-3)
*/
abilities: string[][];
/** killer's splash-tag name; null if unreadable */
name: string | null;
}
export const DEATH_EVENT_TYPE = "Death";
/** The constant message line must read back at least this well to emit. */
const LINE1_MIN_SCORE = 0.5;
/** Snapped weapon reading below this is reported as null (kept in debug). */
const WEAPON_MIN_SCORE = 0.55;
/** Burst-icon fallback match below this is ignored (kept in debug). */
const BURST_ICON_MIN_SCORE = 0.52;
/**
* Candidate-lattice re-rank acceptance (rankByRead — its scores sit well
* below the plain-snap scale; see text.ts). On the 720p-upscaled JP
* frames that motivated it, correct picks score 0.25-0.47 with a margin
* of 0.056+ over the nearest other weapon, while wrong picks margin
* <= 0.03 — the margin, not the score, is the discriminator.
*/
const LATTICE_MIN_SCORE = 0.22;
const LATTICE_MIN_MARGIN = 0.05;
/**
* Burst-icon corroboration: below the decisive threshold the icon alone
* can't be trusted, but when the garbled text *independently* ranks the
* icon's weapon at (or within EPS of) its own top, the two weak signals
* agree out of ~350 candidates and the weapon is accepted. Floor sits at
* the weakest corroborated fixture positive (Splat Dualies at 0.33).
*/
const BURST_ICON_CORROBORATE_MIN_SCORE = 0.3;
const CORROBORATE_EPS = 0.02;
const TAG_NAME_BIN_THRESHOLD = 160;
/**
* The text-color refinement band is absolute closeness (255 - distance) to
* the estimated text color, so its threshold is tight by construction:
* 215 keeps pixels within 40 of the text color — glyph cores — while art
* highlights that leak past the banner-median band sit further away.
*/
const TAG_NAME_REFINE_BIN_THRESHOLD = 215;
/** Don't trust a text-color estimate taken from fewer ink pixels. */
const TAG_NAME_REFINE_MIN_INK = 200;
/**
* Split-banner detection: some banners paint the name band in two flat
* hues (diagonal splits). Any single background estimate turns the other
* half into one huge "ink" blob that border-clearing deletes together
* with the glyphs standing on it, so when a second quantized color bin
* both covers a real share of the band and sits far from the first, a
* third read candidate measures distance from the *nearest* of the two.
* The share floor keeps the text color itself (or sparse art) from being
* mistaken for a second background, which would erase the glyphs.
*/
const TAG_SPLIT_MIN_FRACTION = 0.15;
const TAG_SPLIT_MIN_CHANNEL_DISTANCE = 40;
interface WeaponCandidate {
/** the full weapon line as this template renders it, e.g. "Durch Klecksroller" */
text: string;
entry: WeaponEntry;
}
/** JA templates read through the JA atlas and the swapped-width line ROIs. */
function isJaTemplate(t: DeathMessageTemplate): boolean {
return t.langs.some((lang) => lang.endsWith("ja"));
}
/**
* The weapon-line strings a template can show: that language's localized
* names plus every canonical English name (localized-messages omits names
* identical to English), wrapped in the template's constant pre/post text.
*/
const templateCandidates = new Map<DeathMessageTemplate, WeaponCandidate[]>();
function candidatesFor(template: DeathMessageTemplate): WeaponCandidate[] {
let candidates = templateCandidates.get(template);
if (candidates) return candidates;
const byName = new Map(ALL_WEAPON_ENTRIES.map((e) => [e.name, e]));
const seen = new Set<string>();
candidates = [];
const push = (text: string, entry: WeaponEntry | undefined) => {
const k = matchKey(text);
if (!entry || seen.has(k)) return;
seen.add(k);
candidates!.push({ text: template.weaponPre + text + template.weaponPost, entry });
};
for (const lang of template.langs) {
for (const { text, name } of LOCALIZED_WEAPON_NAMES[lang] ?? []) push(text, byName.get(name));
}
for (const entry of ALL_WEAPON_ENTRIES) push(entry.name, entry);
templateCandidates.set(template, candidates);
return candidates;
}
export function createDeathDetector(resources: ScoreboardResources): Detector<DeathData> {
const cv = getCV();
const scaled = (set: GlyphSet | null | undefined, height: number): GlyphSet | null =>
set ? scaleGlyphSet(set, height / set.height) : null;
const weaponGlyphs = scaled(resources.deathWeaponGlyphs, WEAPON_TEXT_HEIGHT);
// JA glyphs match at native scale: the atlas mixes fixture crops with
// per-face renders already sized to the on-screen condensed text
const jaGlyphs = resources.deathWeaponJaGlyphs ?? null;
const tagNameGlyphs = scaled(resources.deathTagNameGlyphs, TAG_NAME_TEXT_HEIGHT);
const abilities = resources.abilities ?? null;
const burstWeapons = resources.deathBurstWeapons ?? null;
const mainById = new Map(
ALL_WEAPON_ENTRIES.filter((e) => e.type === "MAIN").map((e) => [e.id, e]),
);
function gate(frame: Mat): GateResult {
let darkOk = 0;
const darkProbes = [...GATE_BURST_PROBES, ...GATE_PANEL_PROBES];
for (const roi of darkProbes) {
if (meanBrightness(frame, roi) < GATE_DARK_MAX_MEAN) darkOk++;
}
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const line1 = copyRoi(gray, SPLAT_LINE1_ROI);
const { maxVal } = minMaxLoc(line1);
const bin = new cv.Mat();
cv.threshold(line1, bin, GATE_TEXT_MIN_MAX, 255, cv.THRESH_BINARY);
line1.delete();
const whiteFraction = cv.countNonZero(bin) / (bin.rows * bin.cols);
bin.delete();
const textOk =
maxVal > GATE_TEXT_MIN_MAX && whiteFraction > 0.01 && whiteFraction < GATE_TEXT_MAX_FRACTION;
// max RGB channel, not gray: saturated icon art can be gray-dark (rois.ts)
let iconOk = 0;
for (const row of [0, 1, 2]) {
const probe = copyRoi(frame, gateAbilityProbe(row));
const d = probe.data;
const ch = probe.channels();
const n = probe.rows * probe.cols;
let maxCh = 0;
for (let i = 0; i < n; i++) {
const v = Math.max(d[i * ch]!, d[i * ch + 1]!, d[i * ch + 2]!);
if (v > maxCh) maxCh = v;
}
probe.delete();
if (maxCh > GATE_ICON_MIN_MAX) iconOk++;
}
gray.delete();
const score = (darkOk / darkProbes.length + (textOk ? 1 : 0) + iconOk / 3) / 3;
return { pass: darkOk === darkProbes.length && textOk && iconOk === 3, score };
}
/** Crop the tilted tag, rotate it level, and return the name band crop. */
function levelTagInner(rgb: Mat): Mat {
const outer = copyRoi(rgb, TAG_NAME_OUTER);
const center = new cv.Point(outer.cols / 2, outer.rows / 2);
const m = cv.getRotationMatrix2D(center, -TAG_TILT_DEG, 1);
const rotated = new cv.Mat();
cv.warpAffine(
outer,
rotated,
m,
new cv.Size(outer.cols, outer.rows),
cv.INTER_LINEAR,
cv.BORDER_REPLICATE,
new cv.Scalar(),
);
m.delete();
outer.delete();
const inner = copyRoi(rotated, TAG_NAME_INNER);
rotated.delete();
return inner;
}
/** Per-channel median color of `inner`, over pixels where mask(i) holds. */
function medianColor(inner: Mat, mask?: (i: number) => boolean): [number, number, number] {
const n = inner.rows * inner.cols;
const px = inner.data;
const color: [number, number, number] = [0, 0, 0];
for (let c = 0; c < 3; c++) {
const hist = new Array<number>(256).fill(0);
let total = 0;
for (let i = 0; i < n; i++) {
if (mask && !mask(i)) continue;
hist[px[i * 3 + c]!]!++;
total++;
}
let acc = 0;
let v = 0;
for (; v < 255; v++) {
acc += hist[v]!;
if (acc >= total / 2) break;
}
color[c] = v;
}
return color;
}
/**
* Dominant colors of `inner`: the most frequent quantized colors (5 bits/
* channel), each refined to the per-channel median within its bin, with
* the bin's share of the band. A second background estimate besides
* medianColor: independent whole-image channel medians blend distinct
* populations into a color nobody has (a black banner half-covered by
* green art medians to green-ish, turning the banner base itself into
* "ink" that swallows the name), while the bin vote fails the other way
* on textured banner bases, where a flat art blob out-votes any single
* shade of the texture. Neither estimator wins everywhere, so the parse
* tries both and keeps the better read. The runner-up cluster feeds the
* split-banner candidate (see TAG_SPLIT_MIN_FRACTION).
*
* Bins are clustered before ranking: a flat hue that straddles a
* quantization boundary splits into neighbor bins (a split banner's
* yellow half measured 0.090+0.084 as two bins), and unclustered each
* fragment under-reports the hue's real share of the band.
*/
function dominantColors(
inner: Mat,
count: number,
): { color: [number, number, number]; fraction: number }[] {
const n = inner.rows * inner.cols;
const px = inner.data;
const bins = new Map<number, number>();
for (let i = 0; i < n; i++) {
const key = ((px[i * 3]! >> 3) << 10) | ((px[i * 3 + 1]! >> 3) << 5) | (px[i * 3 + 2]! >> 3);
bins.set(key, (bins.get(key) ?? 0) + 1);
}
// greedy cluster of the top bins by quantized-center proximity
const CLUSTER_MAX_CHANNEL_DISTANCE = 24;
const top = [...bins.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8);
const centerOf = (key: number): [number, number, number] => [
((key >> 10) << 3) + 4,
(((key >> 5) & 31) << 3) + 4,
((key & 31) << 3) + 4,
];
const clusters: { seed: [number, number, number]; keys: Set<number>; count: number }[] = [];
for (const [key, binCount] of top) {
const c = centerOf(key);
const home = clusters.find((cl) =>
cl.seed.every((s, i) => Math.abs(s - c[i]!) <= CLUSTER_MAX_CHANNEL_DISTANCE),
);
if (home) {
home.keys.add(key);
home.count += binCount;
} else {
clusters.push({ seed: c, keys: new Set([key]), count: binCount });
}
}
return clusters
.sort((a, b) => b.count - a.count)
.slice(0, count)
.map(({ keys, count: clusterCount }) => {
const inCluster = (i: number) =>
keys.has(
((px[i * 3]! >> 3) << 10) | ((px[i * 3 + 1]! >> 3) << 5) | (px[i * 3 + 2]! >> 3),
);
return { color: medianColor(inner, inCluster), fraction: clusterCount / n };
});
}
/**
* Text-ness map of the name band: per-pixel max-channel distance from
* the nearest of `colors` (one color for solid banners; the two hues of
* a split banner). Banner art and text color vary per player — including
* pairs like pink text on a light-blue banner with almost no luminance
* contrast — so "differs from the dominant banner color" is the primary
* signal, not brightness in any fixed channel or polarity. `invert`
* flips the polarity to *closeness* for the text-color refinement pass.
*/
function distanceBand(
inner: Mat,
colors: readonly [number, number, number][],
invert: boolean,
): Mat {
const n = inner.rows * inner.cols;
const px = inner.data;
const band = new cv.Mat(inner.rows, inner.cols, cv.CV_8UC1);
const out = band.data;
for (let i = 0; i < n; i++) {
let d = 255;
for (const color of colors) {
const dc = Math.max(
Math.abs(px[i * 3]! - color[0]),
Math.abs(px[i * 3 + 1]! - color[1]),
Math.abs(px[i * 3 + 2]! - color[2]),
);
if (dc < d) d = dc;
}
out[i] = invert ? 255 - d : d;
}
return band;
}
/**
* Zero out ink components that touch the band border. Busy banner art
* (collages, prints) also differs from the median banner color, but the
* art always continues past the name band and so touches its edges,
* while the name is laid out inside it (fixture extremes: dakuten at
* y=3, a 'y' descender ending 2px above the bottom). Left in place, an
* edge blob merges into a glyph's column segment and corrupts the read.
*/
function clearBorderBlobs(band: Mat, threshold: number): void {
const bin = new cv.Mat();
cv.threshold(band, bin, threshold, 255, cv.THRESH_BINARY);
const labels = new cv.Mat();
const stats = new cv.Mat();
const centroids = new cv.Mat();
const count = cv.connectedComponentsWithStats(bin, labels, stats, centroids, 8);
bin.delete();
centroids.delete();
const s = stats.data32S;
const touchesBorder = new Uint8Array(count);
for (let i = 1; i < count; i++) {
const left = s[i * 5 + cv.CC_STAT_LEFT]!;
const top = s[i * 5 + cv.CC_STAT_TOP]!;
const right = left + s[i * 5 + cv.CC_STAT_WIDTH]!;
const bottom = top + s[i * 5 + cv.CC_STAT_HEIGHT]!;
touchesBorder[i] =
left === 0 || top === 0 || right === band.cols || bottom === band.rows ? 1 : 0;
}
stats.delete();
const lab = labels.data32S;
const out = band.data;
for (let i = 0; i < out.length; i++) {
if (touchesBorder[lab[i]!]!) out[i] = 0;
}
labels.delete();
}
function parse(frame: Mat, t: number): DetectedEvent<DeathData>[] {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const rgb = new cv.Mat();
cv.cvtColor(frame, rgb, cv.COLOR_RGBA2RGB);
const confidences: number[] = [];
// 1. read both burst lines and find the language template whose constant
// line reads back best — a gate hit matching none is a lookalike.
// Latin templates read the standard line boxes with the Latin atlas;
// JA templates read the swapped-width JA boxes (weapon line 1 wide,
// constant line 2 narrow — see rois.ts) with the JA atlas, so the two
// scripts never compete inside one glyph set.
let line1: RecognizedText | null = null;
let line2: RecognizedText | null = null;
let jaWeaponLine: RecognizedText | null = null;
let jaConstLine: RecognizedText | null = null;
let template: DeathMessageTemplate | null = null;
let line1Score = 0;
if (weaponGlyphs) {
const readLine = (roi: Roi, glyphs: GlyphSet) => {
const crop = cropRoi(gray, roi);
const read = recognizeText(crop, glyphs, {
binThreshold: SPLAT_TEXT_BIN_THRESHOLD,
minCharScore: 0.3,
});
crop.delete();
return read;
};
line1 = readLine(SPLAT_LINE1_ROI, weaponGlyphs);
line2 = readLine(WEAPON_LINE_ROI, weaponGlyphs);
if (jaGlyphs) {
jaWeaponLine = readLine(JA_WEAPON_LINE_ROI, jaGlyphs);
jaConstLine = readLine(JA_CONST_LINE_ROI, jaGlyphs);
}
for (const t of DEATH_MESSAGE_TEMPLATES) {
let constReading: string;
if (isJaTemplate(t)) {
if (!jaConstLine) continue;
constReading = jaConstLine.text;
} else {
constReading = t.weaponLine === 1 ? line2.text : line1.text;
}
const score = closestEntry(constReading, [t.constText])?.score ?? 0;
if (score > line1Score) {
line1Score = score;
template = t;
}
}
if (!template || line1Score < LINE1_MIN_SCORE) {
gray.delete();
rgb.delete();
return [];
}
}
// 2. the other line carries the weapon name; snap it to the template
// language's names (localized + canonical English)
let weapon: string | null = null;
let weaponId: number | null = null;
let weaponType: WeaponType | null = null;
let weaponScore = 0;
let weaponRaw: RecognizedText | null = null;
let plainRanked: { entry: WeaponCandidate; score: number }[] = [];
const accept = (entry: WeaponEntry, score: number) => {
weapon = entry.name;
weaponId = Number(entry.id);
weaponType = entry.type;
weaponScore = score;
};
if (weaponGlyphs && template) {
weaponRaw = isJaTemplate(template) ? jaWeaponLine : template.weaponLine === 1 ? line1 : line2;
const reading = weaponRaw!.text;
if (reading) plainRanked = rankBy(reading, candidatesFor(template), (c) => c.text);
const match = plainRanked[0];
if (match) {
weaponScore = match.score;
if (match.score >= WEAPON_MIN_SCORE) accept(match.entry.entry, match.score);
}
}
// 2b. text can be unreadable while the burst's weapon icon is intact
// (the WIPEOUT banner covers the weapon name line), so fall back to
// matching the icon against the main-weapon set at burst size. Only a
// decisive match is trusted: fixture positives score 0.55+ while the
// best off-target frame (icon displaced by a rainmaker line) hits 0.48.
let burstIcon: WeaponMatch | null = null;
if (weapon === null && burstWeapons) {
const crop = cropRoi(rgb, BURST_ICON_ROI);
burstIcon = matchWeapon(crop, burstWeapons);
crop.delete();
const entry =
burstIcon.score >= BURST_ICON_MIN_SCORE ? mainById.get(burstIcon.id) : undefined;
if (entry) accept(entry, burstIcon.score);
}
// 2c. low-fidelity captures (720p upscaled to canonical) garble the
// per-segment top-1 read enough that the plain snap stays under
// WEAPON_MIN_SCORE, while the correct glyphs sit at rank 2-3 of the
// segments' candidate lists. Re-rank through those lists (rankByRead)
// and accept the top weapon when it clears the field decisively.
let latticeTop: { entry: WeaponEntry; score: number; margin: number } | null = null;
let latticeRanked: { entry: WeaponCandidate; score: number }[] = [];
if (weapon === null && weaponGlyphs && template && weaponRaw!.chars.length > 0) {
latticeRanked = rankByRead(weaponRaw!.chars, candidatesFor(template), (c) => c.text);
const top = latticeRanked[0]!;
// margin vs the nearest *other* weapon: the same weapon rides both
// its localized and English candidate lines
const runner = latticeRanked.find((r) => r.entry.entry !== top.entry.entry);
latticeTop = {
entry: top.entry.entry,
score: top.score,
margin: top.score - (runner?.score ?? 0),
};
if (latticeTop.score >= LATTICE_MIN_SCORE && latticeTop.margin >= LATTICE_MIN_MARGIN) {
accept(latticeTop.entry, latticeTop.score);
}
}
// 2d. neither signal is decisive alone, but if the burst icon's main
// weapon is also the text's best guess (plain or lattice ranking,
// within EPS of that ranking's top), the independent agreement is
// decisive together.
if (weapon === null && burstIcon && burstIcon.score >= BURST_ICON_CORROBORATE_MIN_SCORE) {
const entry = mainById.get(burstIcon.id);
if (entry) {
const bestFor = (ranked: { entry: WeaponCandidate; score: number }[]) =>
ranked.reduce((s, r) => (r.entry.entry === entry ? Math.max(s, r.score) : s), 0);
const supported =
(plainRanked.length > 0 &&
bestFor(plainRanked) >= plainRanked[0]!.score - CORROBORATE_EPS) ||
(latticeRanked.length > 0 &&
bestFor(latticeRanked) >= latticeRanked[0]!.score - CORROBORATE_EPS);
if (supported) accept(entry, burstIcon.score);
}
}
if (weaponGlyphs && template) confidences.push(weaponScore);
// 3. ability grid; rows carry 1-3 sub circles (left-aligned, as many
// as the gear has slots), so a sub box without badge ink ends the row
const abilityRows: string[][] = [];
const abilityDebug: (WeaponMatch | null)[][] = [];
if (abilities) {
for (let row = 0; row < ABILITY_ROWS; row++) {
const ids: string[] = [];
const debug: (WeaponMatch | null)[] = [];
const mainCrop = cropRoi(rgb, abilityMainRoi(row));
const main = matchWeapon(mainCrop, abilities.mains, {
inkThreshold: ABILITY_INK_THRESHOLD,
});
mainCrop.delete();
ids.push(main.id);
debug.push(main);
confidences.push(Math.max(0, main.score));
for (let slot = 0; slot < ABILITY_SUB_XS.length; slot++) {
const crop = copyRoi(rgb, abilitySubRoi(row, slot));
const d = crop.data;
const n = crop.rows * crop.cols;
let ink = 0;
for (let i = 0; i < n; i++) {
const v = Math.max(d[i * 3]!, d[i * 3 + 1]!, d[i * 3 + 2]!);
if (v > ABILITY_INK_THRESHOLD) ink++;
}
if (ink < ABILITY_SLOT_MIN_INK) {
crop.delete();
break;
}
const sub = matchWeapon(crop, abilities.subs, {
inkThreshold: ABILITY_INK_THRESHOLD,
});
crop.delete();
ids.push(sub.id);
debug.push(sub);
confidences.push(Math.max(0, sub.score));
}
abilityRows.push(ids);
abilityDebug.push(debug);
}
}
// 4. splash-tag name, two passes per background candidate: distance
// from the estimated banner color, then — because busy banner art
// (collages, prints) also differs from that estimate, merging with
// glyphs and surviving as fake ones — closeness to the text color
// estimated from pass 1's ink. Whichever pass reads back more
// confidently wins: solid banners stay on pass 1, art-heavy banners
// recover on pass 2. Both background estimators run (see
// dominantColor) and the better-reading candidate wins the same way.
let name: string | null = null;
let nameConfidence = 0;
let nameRaw = "";
let tagBackground: [number, number, number] | null = null;
let tagTextColor: [number, number, number] | null = null;
if (tagNameGlyphs) {
const spaceGap = Math.max(7, Math.round(tagNameGlyphs.medianWidth * 0.55));
const inner = levelTagInner(rgb);
const readWithBackground = (backgrounds: readonly [number, number, number][]) => {
const band = distanceBand(inner, backgrounds, false);
cv.normalize(band, band, 0, 255, cv.NORM_MINMAX);
clearBorderBlobs(band, TAG_NAME_BIN_THRESHOLD);
let parsed = parseName(band, tagNameGlyphs, {
spaceGap,
binThreshold: TAG_NAME_BIN_THRESHOLD,
});
let textColor: [number, number, number] | null = null;
const ink = band.data;
let inkCount = 0;
for (let i = 0; i < ink.length; i++) if (ink[i]! > TAG_NAME_BIN_THRESHOLD) inkCount++;
if (inkCount >= TAG_NAME_REFINE_MIN_INK) {
textColor = medianColor(inner, (i) => ink[i]! > TAG_NAME_BIN_THRESHOLD);
const refined = distanceBand(inner, [textColor], true);
clearBorderBlobs(refined, TAG_NAME_REFINE_BIN_THRESHOLD);
const reparsed = parseName(refined, tagNameGlyphs, {
spaceGap,
binThreshold: TAG_NAME_REFINE_BIN_THRESHOLD,
});
refined.delete();
if (reparsed.confidence > parsed.confidence) parsed = reparsed;
}
band.delete();
return { parsed, background: backgrounds[0]!, textColor };
};
const median = medianColor(inner);
const dominants = dominantColors(inner, 2);
const dominant = dominants[0]!.color;
const candidates: [number, number, number][][] = [[median]];
if (dominant.some((c, i) => Math.abs(c - median[i]!) > 8)) candidates.push([dominant]);
const second = dominants[1];
if (
second &&
second.fraction >= TAG_SPLIT_MIN_FRACTION &&
second.color.some((c, i) => Math.abs(c - dominant[i]!) > TAG_SPLIT_MIN_CHANNEL_DISTANCE)
) {
candidates.push([dominant, second.color]);
}
// an empty read never beats one that produced glyphs (an estimate
// landing on the text color blanks the whole band, and recognizeText
// reports a segment-less band as confidence 1), and near-tied
// confidences resolve to the longer read: confidence is the *min*
// char score, so a background estimate that erases most of the name
// can still read its two surviving glyphs immaculately — more
// recognized glyphs is the better read when neither is clearly worse
const NEAR_TIE = 0.03;
const beats = (a: { parsed: { name: string; confidence: number } }, b: typeof a) => {
const aRead = a.parsed.name.length > 0 ? 1 : 0;
const bRead = b.parsed.name.length > 0 ? 1 : 0;
if (aRead !== bRead) return aRead - bRead;
if (Math.abs(a.parsed.confidence - b.parsed.confidence) <= NEAR_TIE) {
return a.parsed.name.length - b.parsed.name.length;
}
return a.parsed.confidence - b.parsed.confidence;
};
let best = readWithBackground(candidates[0]!);
for (const backgrounds of candidates.slice(1)) {
const alt = readWithBackground(backgrounds);
if (beats(alt, best) > 0) best = alt;
}
inner.delete();
tagBackground = best.background;
tagTextColor = best.textColor;
nameRaw = best.parsed.raw.text;
if (best.parsed.name.length > 0) name = best.parsed.name;
nameConfidence = best.parsed.confidence;
confidences.push(nameConfidence);
}
gray.delete();
rgb.delete();
const confidence =
confidences.length > 0 ? confidences.reduce((a, b) => a + b, 0) / confidences.length : 0;
return [
{
type: DEATH_EVENT_TYPE,
t,
confidence,
data: { weapon, weaponId, weaponType, abilities: abilityRows, name },
debug: {
line1: line1?.text,
line2: line2?.text,
jaWeaponLine: jaWeaponLine?.text,
jaConstLine: jaConstLine?.text,
line1Score,
messageLangs: template?.langs,
weaponRaw: weaponRaw?.text,
weaponScore,
weaponLattice: latticeTop && {
name: latticeTop.entry.name,
score: latticeTop.score,
margin: latticeTop.margin,
},
burstIcon: burstIcon && { id: burstIcon.id, score: burstIcon.score, top: burstIcon.top },
abilityRows: abilityDebug.map((row) =>
row.map((m) => m && { top: m.top, score: m.score }),
),
nameRaw,
nameScore: nameConfidence,
tagBackground,
tagTextColor,
},
},
];
}
return { id: "death", gate, parse };
}

9914
app/features/cv/core/detectors/death/localized-messages.ts Normal file
View File

File diff suppressed because it is too large Load Diff

163
app/features/cv/core/detectors/death/rois.ts Normal file
View File

@@ -0,0 +1,163 @@
/**
* ALL death-screen ROI coordinates, in canonical 1920x1080 space.
* Calibrated against the death/ fixtures via tools/dump-crops.ts,
* HoughCircles measurement, and bright-row profiling.
*
* The death cam overlays three fixed elements on live gameplay:
* - a dark camo "splat burst" in the top-center with the two-line white
* message "Splatted by" / "<weapon name>!";
* - the killer's gear panel bottom-left: a dark rounded rect with three
* gear rows (head/clothes/shoes), each one large main-ability circle
* (⌀~68) plus three small sub-ability circles (⌀~48), rows divided by
* white dashed lines;
* - the killer's splash tag bottom-right, tilted a few degrees, with the
* name in large type (banner art and text color vary per player).
* Everything except the tag banner sits over the live scene, so probe
* regions were chosen inside the opaque-dark parts of the overlays.
*/
import type { Roi } from "../../canonical";
/** The constant "Splatted by" line (white text centered at x=960). */
export const SPLAT_LINE1_ROI: Roi = { x: 790, y: 362, w: 340, h: 52 };
/** The weapon name line ("Rapid Blaster Deco!"), centered, length varies. */
export const WEAPON_LINE_ROI: Roi = { x: 640, y: 414, w: 640, h: 48 };
/** White message text on the dark burst binarizes cleanly and high. */
export const SPLAT_TEXT_BIN_THRESHOLD = 190;
/** Tight cap height of the message text (atlas nominal height). */
export const WEAPON_TEXT_HEIGHT = 34;
/**
* Non-Latin weaponLine=1 languages (JA: "<weapon> で" over "やられた！")
* read the two lines with swapped widths: the variable-length weapon name
* sits on line 1, so it gets the full-width box, while the constant line
* below is short and centered, so a narrow box keeps scene ink that shows
* past the burst's edge out of the read. Both boxes are taller than the
* Latin ones: kana overshoot the Latin cap band on both sides (dakuten
* above, full-depth bodies below — the JP line spans y=359..401 where the
* Latin crop starts at 362).
*/
export const JA_WEAPON_LINE_ROI: Roi = { x: 640, y: 354, w: 640, h: 62 };
export const JA_CONST_LINE_ROI: Roi = { x: 790, y: 412, w: 340, h: 54 };
/**
* The killer's weapon icon, drawn upright at the top of the burst above the
* message text (main-weapon 2D icon art, ~110px; specials appear team-color
* tinted instead and are not matched). Templates competing in this box are
* built at BURST_ICON_TEMPLATE_SIZES — score peaks vary 124-132 per capture,
* so several sizes are tried. When a "Lost the Rainmaker!" style line is
* present the icon shifts up out of this box; the text read handles those.
*/
export const BURST_ICON_ROI: Roi = { x: 785, y: 230, w: 190, h: 140 };
export const BURST_ICON_TEMPLATE_SIZES = [116, 124, 132] as const;
/** Gear panel rows: [head, clothes, shoes]. */
export const ABILITY_ROWS = 3;
/** Main-ability circle centers (⌀~68). */
const ABILITY_MAIN_X = 515;
const ABILITY_MAIN_YS = [696, 795, 887] as const;
/** Sub-ability circle centers (⌀~48), slightly below the main's center. */
export const ABILITY_SUB_XS = [578, 630, 682] as const;
const ABILITY_SUB_YS = [702, 797, 888] as const;
/**
* Search boxes around each circle. Heights double as the size filter:
* matchTemplate silently skips templates taller than the ROI, so the
* 56px sub box excludes the main-size templates.
*/
export function abilityMainRoi(row: number): Roi {
const cy = ABILITY_MAIN_YS[row]!;
return { x: ABILITY_MAIN_X - 38, y: cy - 38, w: 76, h: 76 };
}
export function abilitySubRoi(row: number, slot: number): Roi {
const cx = ABILITY_SUB_XS[slot]!;
const cy = ABILITY_SUB_YS[row]!;
return { x: cx - 28, y: cy - 28, w: 56, h: 56 };
}
/** Template heights (px at 1080p) per circle role. */
export const ABILITY_MAIN_SIZES = [64, 68, 72] as const;
export const ABILITY_SUB_SIZES = [44, 48, 52] as const;
/**
* Icon art diameter as a fraction of the circle box (art-ratio sweep over
* the fixtures: mains peak at 1.0, subs at 0.92 — the badges draw the art
* nearly edge-to-edge, so the template sizes above are effectively the art
* sizes and the black ring contributes almost nothing).
*/
export const ABILITY_MAIN_ART_RATIO = 1.0;
export const ABILITY_SUB_ART_RATIO = 0.92;
/**
* Ink threshold inside a circle box: the badge is near-black, icon art is
* saturated-bright. The panel is slightly translucent, so a bright scene
* can ghost through at low intensity — kept above that.
*/
export const ABILITY_INK_THRESHOLD = 90;
/**
* A gear row only carries as many sub circles as the gear has slots
* (1-3, left-aligned); an absent slot shows the bare translucent panel.
* Bright pixels (max channel > ABILITY_INK_THRESHOLD) inside the sub box
* separate the cases cleanly: absent slots measure 0 across the fixtures
* while every real badge — ability art or the white "?" of an unrevealed
* slot, down to the dimmest 720p capture — measures 403+.
*/
export const ABILITY_SLOT_MIN_INK = 200;
/**
* Splash tag name band. The tag renders tilted (text baseline rises to the
* right by TAG_TILT_DEG); crop TAG_NAME_OUTER, rotate level around its
* center, then read TAG_NAME_INNER relative to the rotated crop.
*/
export const TAG_TILT_DEG = 3.0;
export const TAG_NAME_OUTER: Roi = { x: 1130, y: 770, w: 650, h: 140 };
/**
* Name band inside the rotated outer crop. The top edge must clear the
* kana dakuten, which rise well past the cap band (ご's marks start at
* outer y=24; the old y=34 top clipped them, and the surviving sliver
* touched the band border, where clearBorderBlobs deleted it — reading こ
* for ご). The title line above ends by outer y=17 (descenders included),
* so y=21 splits the two with margin on both sides.
*/
export const TAG_NAME_INNER: Roi = { x: 20, y: 21, w: 610, h: 87 };
/** Tight cap height of the tag name text (atlas nominal height). */
export const TAG_NAME_TEXT_HEIGHT = 46;
/**
* Gate probes. The burst camo is dark (~25-70) left/right of the constant
* text line and below the weapon line; the gear panel is dark in the strip
* right of the sub circles. The white message text gives the bright anchor.
*/
export const GATE_BURST_PROBES: readonly Roi[] = [
{ x: 750, y: 376, w: 36, h: 22 },
{ x: 1134, y: 376, w: 36, h: 22 },
{ x: 930, y: 466, w: 60, h: 18 },
];
export const GATE_PANEL_PROBES: readonly Roi[] = [
{ x: 720, y: 686, w: 30, h: 20 },
{ x: 720, y: 785, w: 30, h: 20 },
];
export const GATE_DARK_MAX_MEAN = 95;
/** SPLAT_LINE1_ROI must contain near-white pixels... */
export const GATE_TEXT_MIN_MAX = 210;
/** ...but not too many: it is a short text line, not a white panel. */
export const GATE_TEXT_MAX_FRACTION = 0.35;
/**
* The dark probes plus a white text line also describe the scoreboard
* screens, so the gate additionally requires bright icon art at all three
* main-ability circle centers. Brightness is the per-pixel max RGB channel,
* not grayscale: saturated ability art can be gray-dark everywhere in the
* circle (z-f-splatterscope-vod's middle-row orange/purple flame peaks at
* 134 in gray but 248 in max-channel). Death fixtures measure 244+ in
* max-channel while the closest non-death fixture row is 184.
*/
export function gateAbilityProbe(row: number): Roi {
return { x: ABILITY_MAIN_X - 14, y: ABILITY_MAIN_YS[row]! - 14, w: 28, h: 28 };
}
export const GATE_ICON_MIN_MAX = 200;

242
app/features/cv/core/detectors/death/weapon-names.ts Normal file
View File

@@ -0,0 +1,242 @@
/**
* English weapon names keyed by in-game weapon id, generated from
* sendou.ink locales/en/weapons.json (MAIN_/SUB_/SPECIAL_<id> entries;
* mains filtered to the assets/cv/main-weapons icon manifest).
* The death screen shows the killer's weapon as text ("Splatted by
* <name>!") and can credit a main, sub, or special, so the closed set
* that OCR output snaps to spans all three kinds. Ids are only unique
* within a kind (sub 0 = Splat Bomb, main 0 = Sploosh-o-matic).
*/
export type WeaponType = "MAIN" | "SUB" | "SPECIAL";
export interface WeaponEntry {
id: string;
name: string;
type: WeaponType;
}
export const WEAPON_NAMES: ReadonlyMap<string, string> = new Map([
["0", "Sploosh-o-matic"],
["1", "Neo Sploosh-o-matic"],
["10", "Splattershot Jr."],
["11", "Custom Splattershot Jr."],
["20", "Splash-o-matic"],
["21", "Neo Splash-o-matic"],
["22", "Splash-o-matic GCK-O"],
["30", "Aerospray MG"],
["31", "Aerospray RG"],
["32", "Colorz Aerospray"],
["40", "Splattershot"],
["41", "Tentatek Splattershot"],
["42", "Glamorz Splattershot"],
["45", "Hero Shot Replica"],
["46", "Octo Shot Replica"],
["47", "Order Shot Replica"],
["50", ".52 Gal"],
["51", ".52 Gal Deco"],
["60", "N-ZAP '85"],
["61", "N-ZAP '89"],
["70", "Splattershot Pro"],
["71", "Forge Splattershot Pro"],
["72", "Splattershot Pro FRZ-N"],
["80", ".96 Gal"],
["81", ".96 Gal Deco"],
["82", "Clawz .96 Gal"],
["90", "Jet Squelcher"],
["91", "Custom Jet Squelcher"],
["92", "Jet Squelcher COB-R"],
["100", "Splattershot Nova"],
["101", "Annaki Splattershot Nova"],
["200", "Luna Blaster"],
["201", "Luna Blaster Neo"],
["205", "Order Blaster Replica"],
["210", "Blaster"],
["211", "Custom Blaster"],
["212", "Gleamz Blaster"],
["220", "Range Blaster"],
["221", "Custom Range Blaster"],
["230", "Clash Blaster"],
["231", "Clash Blaster Neo"],
["240", "Rapid Blaster"],
["241", "Rapid Blaster Deco"],
["250", "Rapid Blaster Pro"],
["251", "Rapid Blaster Pro Deco"],
["252", "Rapid Blaster Pro WNT-R"],
["260", "S-BLAST '92"],
["261", "S-BLAST '91"],
["300", "L-3 Nozzlenose"],
["301", "L-3 Nozzlenose D"],
["302", "Glitterz L-3 Nozzlenose"],
["310", "H-3 Nozzlenose"],
["311", "H-3 Nozzlenose D"],
["312", "H-3 Nozzlenose VIP-R"],
["400", "Squeezer"],
["401", "Foil Squeezer"],
["1000", "Carbon Roller"],
["1001", "Carbon Roller Deco"],
["1002", "Carbon Roller ANG-L"],
["1010", "Splat Roller"],
["1011", "Krak-On Splat Roller"],
["1015", "Order Roller Replica"],
["1020", "Dynamo Roller"],
["1021", "Gold Dynamo Roller"],
["1022", "Starz Dynamo Roller"],
["1030", "Flingza Roller"],
["1031", "Foil Flingza Roller"],
["1040", "Big Swig Roller"],
["1041", "Big Swig Roller Express"],
["1042", "Planetz Big Swig Roller"],
["1100", "Inkbrush"],
["1101", "Inkbrush Nouveau"],
["1110", "Octobrush"],
["1111", "Octobrush Nouveau"],
["1112", "Cometz Octobrush"],
["1115", "Orderbrush Replica"],
["1120", "Painbrush"],
["1121", "Painbrush Nouveau"],
["1122", "Painbrush BRN-Z"],
["2000", "Classic Squiffer"],
["2001", "New Squiffer"],
["2010", "Splat Charger"],
["2011", "Z+F Splat Charger"],
["2012", "Splat Charger CAM-O"],
["2015", "Order Charger Replica"],
["2020", "Splatterscope"],
["2021", "Z+F Splatterscope"],
["2022", "Splatterscope CAM-O"],
["2030", "E-liter 4K"],
["2031", "Custom E-liter 4K"],
["2040", "E-liter 4K Scope"],
["2041", "Custom E-liter 4K Scope"],
["2050", "Bamboozler 14 Mk I"],
["2051", "Bamboozler 14 Mk II"],
["2060", "Goo Tuber"],
["2061", "Custom Goo Tuber"],
["2070", "Snipewriter 5H"],
["2071", "Snipewriter 5B"],
["3000", "Slosher"],
["3001", "Slosher Deco"],
["3005", "Order Slosher Replica"],
["3010", "Tri-Slosher"],
["3011", "Tri-Slosher Nouveau"],
["3012", "Tri-Slosher ASH-N"],
["3020", "Sloshing Machine"],
["3021", "Sloshing Machine Neo"],
["3030", "Bloblobber"],
["3031", "Bloblobber Deco"],
["3040", "Explosher"],
["3041", "Custom Explosher"],
["3050", "Dread Wringer"],
["3051", "Dread Wringer D"],
["3052", "Hornz Dread Wringer"],
["4000", "Mini Splatling"],
["4001", "Zink Mini Splatling"],
["4002", "Mini Splatling RTL-R"],
["4010", "Heavy Splatling"],
["4011", "Heavy Splatling Deco"],
["4015", "Order Splatling Replica"],
["4020", "Hydra Splatling"],
["4021", "Custom Hydra Splatling"],
["4022", "Torrentz Hydra Splatling"],
["4030", "Ballpoint Splatling"],
["4031", "Ballpoint Splatling Nouveau"],
["4040", "Nautilus 47"],
["4041", "Nautilus 79"],
["4050", "Heavy Edit Splatling"],
["4051", "Heavy Edit Splatling Nouveau"],
["5000", "Dapple Dualies"],
["5001", "Dapple Dualies Nouveau"],
["5002", "Dapple Dualies NOC-T"],
["5010", "Splat Dualies"],
["5011", "Enperry Splat Dualies"],
["5012", "Twinklez Splat Dualies"],
["5015", "Order Dualie Replicas"],
["5020", "Glooga Dualies"],
["5021", "Glooga Dualies Deco"],
["5030", "Dualie Squelchers"],
["5031", "Custom Dualie Squelchers"],
["5032", "Hoofz Dualie Squelchers"],
["5040", "Dark Tetra Dualies"],
["5041", "Light Tetra Dualies"],
["5050", "Douser Dualies FF"],
["5051", "Custom Douser Dualies FF"],
["6000", "Splat Brella"],
["6001", "Sorella Brella"],
["6005", "Order Brella Replica"],
["6010", "Tenta Brella"],
["6011", "Tenta Sorella Brella"],
["6012", "Tenta Brella CRE-M"],
["6020", "Undercover Brella"],
["6021", "Undercover Sorella Brella"],
["6022", "Patternz Undercover Brella"],
["6030", "Recycled Brella 24 Mk I"],
["6031", "Recycled Brella 24 Mk II"],
["7010", "Tri-Stringer"],
["7011", "Inkline Tri-Stringer"],
["7012", "Bulbz Tri-Stringer"],
["7015", "Order Stringer Replica"],
["7020", "REEF-LUX 450"],
["7021", "REEF-LUX 450 Deco"],
["7022", "REEF-LUX 450 MIL-K"],
["7030", "Wellstring V"],
["7031", "Custom Wellstring V"],
["8000", "Splatana Stamper"],
["8001", "Splatana Stamper Nouveau"],
["8002", "Stickerz Splatana Stamper"],
["8005", "Order Splatana Replica"],
["8010", "Splatana Wiper"],
["8011", "Splatana Wiper Deco"],
["8012", "Splatana Wiper RUS-T"],
["8020", "Mint Decavitator"],
["8021", "Charcoal Decavitator"],
]);
const SUB_WEAPON_NAMES: ReadonlyMap<string, string> = new Map([
["0", "Splat Bomb"],
["1", "Suction Bomb"],
["2", "Burst Bomb"],
["3", "Sprinkler"],
["4", "Splash Wall"],
["5", "Fizzy Bomb"],
["6", "Curling Bomb"],
["7", "Autobomb"],
["8", "Squid Beakon"],
["9", "Point Sensor"],
["10", "Ink Mine"],
["11", "Toxic Mist"],
["12", "Angle Shooter"],
["13", "Torpedo"],
]);
const SPECIAL_WEAPON_NAMES: ReadonlyMap<string, string> = new Map([
["1", "Trizooka"],
["2", "Big Bubbler"],
["3", "Zipcaster"],
["4", "Tenta Missiles"],
["5", "Ink Storm"],
["6", "Booyah Bomb"],
["7", "Wave Breaker"],
["8", "Ink Vac"],
["9", "Killer Wail 5.1"],
["10", "Inkjet"],
["11", "Ultra Stamp"],
["12", "Crab Tank"],
["13", "Reefslider"],
["14", "Triple Inkstrike"],
["15", "Tacticooler"],
["16", "Super Chump"],
["17", "Kraken Royale"],
["18", "Triple Splashdown"],
["19", "Splattercolor Screen"],
]);
function entriesOf(map: ReadonlyMap<string, string>, type: WeaponType): WeaponEntry[] {
return [...map.entries()].map(([id, name]) => ({ id, name, type }));
}
/** Every weapon that can appear in the death message, across all kinds. */
export const ALL_WEAPON_ENTRIES: readonly WeaponEntry[] = [
...entriesOf(WEAPON_NAMES, "MAIN"),
...entriesOf(SUB_WEAPON_NAMES, "SUB"),
...entriesOf(SPECIAL_WEAPON_NAMES, "SPECIAL"),
];

334
app/features/cv/core/detectors/map-start/index.ts Normal file
View File

@@ -0,0 +1,334 @@
/**
* MapStartDetector: parses the match-intro splash shown as a game begins —
* the mode title on the black center splat and the stage name bottom-right,
* both snapped to the localized closed sets (core/localized.ts) and
* reported as the canonical English names.
*
* The mode title wraps to two lines for the longer names ("Splat" /
* "Zones"; the localized wrap variants carry their own hyphenation), so
* parse finds the actual text lines inside MODE_BLOCK_ROI by row
* projection, OCRs each band, and snaps the joined reading. The constant
* "MODE" label ("Kampfart", "Mode", ...) doubles as a parse-time
* confirmation: if it does not read back as any language's label, the gate
* hit was a lookalike frame and no event is emitted.
*
* The splash sits on live gameplay, so on bright stages the text regions
* pick up background past the splat/outline edges. The title block is
* masked to pixels near darkness for line finding only (the raw crop OCRs
* better once the bands are right); the stage line reads its min channel —
* which drops blue-tinted water that gray keeps — under several
* binarizations, keeping the best-snapping one (see rois.ts).
*/
import { getCV, type Mat, minMaxLoc } from "../../cv";
import { type GlyphSet, type RecognizedText, recognizeText, scaleGlyphSet } from "../../glyphs";
import { copyRoi, meanBrightness, minChannel } from "../../image";
import { ALL_MODE_ENTRIES, ALL_MODE_LABELS, ALL_STAGE_ENTRIES } from "../../localized";
import { closestBy } from "../../text";
import type { ScoreboardResources } from "../scoreboard/index";
import type { DetectedEvent, Detector, GateResult } from "../types";
import {
BLOCK_MASK_RADIUS,
GATE_DARK_MAX_MEAN,
GATE_DARK_PROBES,
GATE_INK_BAND,
GATE_INK_BAND_MAX_BRIGHT,
GATE_INK_BAND_MIN_DARK,
GATE_TEXT_MAX_FRACTION,
GATE_TEXT_MIN_FRACTION,
GATE_TEXT_MIN_MAX,
LINE_GAP_TOLERANCE,
LINE_MIN_HEIGHT,
LINE_MIN_ROW_PIXELS,
LINE_ROW_FRACTION,
MASK_DARK_MAX,
MODE_BLOCK_ROI,
MODE_LABEL_ROI,
MODE_LABEL_TEXT_HEIGHT,
MODE_TEXT_HEIGHT,
STAGE_BIN_THRESHOLD,
STAGE_MASK_RADIUS,
STAGE_RAW_BIN_THRESHOLDS,
STAGE_ROI,
STAGE_TEXT_HEIGHT,
TEXT_BIN_THRESHOLD,
} from "./rois";
export interface MapStartData {
/** e.g. "Splat Zones"; null if unreadable */
mode: string | null;
/** e.g. "Undertow Spillway"; null if unreadable */
stage: string | null;
}
export const MAP_START_EVENT_TYPE = "MapStart";
/** "MODE" must read back at least this well for parse to emit. */
const LABEL_MIN_SCORE = 0.5;
/** Accept a closed-set match only above this score (1 = exact). */
const MIN_MATCH_SCORE = 0.62;
interface LineBand {
y0: number;
y1: number;
}
/** Zero every pixel with no near-black pixel within `radius` of it. */
function maskNearDark(gray: Mat, radius: number): Mat {
const cv = getCV();
const dark = new cv.Mat();
cv.threshold(gray, dark, MASK_DARK_MAX, 255, cv.THRESH_BINARY_INV);
const kernel = cv.getStructuringElement(
cv.MORPH_ELLIPSE,
new cv.Size(2 * radius + 1, 2 * radius + 1),
);
const near = new cv.Mat();
cv.dilate(dark, near, kernel);
kernel.delete();
dark.delete();
const out = new cv.Mat(gray.rows, gray.cols, cv.CV_8UC1, new cv.Scalar(0));
gray.copyTo(out, near);
near.delete();
return out;
}
/** Find text line bands in a binarized block by row projection. */
function findLineBands(binary: Mat): LineBand[] {
const { rows, cols, data } = binary;
const counts = new Array<number>(rows).fill(0);
for (let y = 0; y < rows; y++) {
for (let x = 0; x < cols; x++) {
if (data[y * cols + x]! > 0) counts[y]!++;
}
}
// Dual threshold: a band must contain rows above a cutoff scaled to the
// strongest row — the block is sized for the widest localized titles,
// and background leaking past the splat's rim puts a scene-dependent
// noise floor under every row that the fixed floor alone can't sit
// above — but then extends over the fixed floor, so the antialiased
// glyph tops/bottoms stay inside the band.
const core = Math.max(LINE_MIN_ROW_PIXELS, LINE_ROW_FRACTION * Math.max(...counts));
const bands: LineBand[] = [];
let start = -1;
let gap = 0;
for (let y = 0; y < rows; y++) {
if (counts[y]! >= core) {
if (start < 0) start = y;
gap = 0;
} else if (start >= 0 && ++gap > LINE_GAP_TOLERANCE) {
bands.push({ y0: start, y1: y - gap + 1 });
start = -1;
}
}
if (start >= 0) bands.push({ y0: start, y1: rows - gap });
for (const band of bands) {
while (band.y0 > 0 && counts[band.y0 - 1]! >= LINE_MIN_ROW_PIXELS) band.y0--;
while (band.y1 < rows && counts[band.y1]! >= LINE_MIN_ROW_PIXELS) band.y1++;
}
const merged: LineBand[] = [];
for (const band of bands) {
const last = merged[merged.length - 1];
if (last && band.y0 <= last.y1) last.y1 = Math.max(last.y1, band.y1);
else merged.push(band);
}
return merged.filter((b) => b.y1 - b.y0 >= LINE_MIN_HEIGHT);
}
/**
* Trim a line band to its text columns (on the masked binary), so the raw
* OCR crop excludes the background the wide block picks up at its edges.
*/
function bandExtent(binary: Mat, band: LineBand): { x0: number; x1: number } | null {
const { cols, data } = binary;
let x0 = -1;
let x1 = -1;
for (let x = 0; x < cols; x++) {
let count = 0;
for (let y = band.y0; y < band.y1; y++) {
if (data[y * cols + x]! > 0) count++;
}
if (count >= 2) {
if (x0 < 0) x0 = x;
x1 = x;
}
}
return x0 < 0 ? null : { x0, x1 };
}
export function createMapStartDetector(resources: ScoreboardResources): Detector<MapStartData> {
const cv = getCV();
const scaled = (set: GlyphSet | null | undefined, height: number): GlyphSet | null =>
set ? scaleGlyphSet(set, height / set.height) : null;
const modeGlyphs = scaled(resources.mapStartModeGlyphs, MODE_TEXT_HEIGHT);
const stageGlyphs = scaled(resources.mapStartStageGlyphs, STAGE_TEXT_HEIGHT);
const labelGlyphs = scaled(resources.mapStartStageGlyphs, MODE_LABEL_TEXT_HEIGHT);
function gate(frame: Mat): GateResult {
let darkOk = 0;
for (const roi of GATE_DARK_PROBES) {
if (meanBrightness(frame, roi) < GATE_DARK_MAX_MEAN) darkOk++;
}
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const label = copyRoi(gray, MODE_LABEL_ROI);
const { maxVal } = minMaxLoc(label);
const bin = new cv.Mat();
cv.threshold(label, bin, TEXT_BIN_THRESHOLD, 255, cv.THRESH_BINARY);
label.delete();
const whiteFraction = cv.countNonZero(bin) / (bin.rows * bin.cols);
bin.delete();
const textOk =
maxVal > GATE_TEXT_MIN_MAX &&
whiteFraction > GATE_TEXT_MIN_FRACTION &&
whiteFraction < GATE_TEXT_MAX_FRACTION;
// the label-to-title gap core is solid ink on this screen: it must be
// near-totally dark (the scoreboards' pills aren't) and carry no bright
// pixels (the death burst's "Splatted by" line crosses it)
const band = copyRoi(gray, GATE_INK_BAND);
const bandPixels = band.rows * band.cols;
const bandBright = new cv.Mat();
cv.threshold(band, bandBright, TEXT_BIN_THRESHOLD, 255, cv.THRESH_BINARY);
const brightFraction = cv.countNonZero(bandBright) / bandPixels;
bandBright.delete();
const bandDark = new cv.Mat();
cv.threshold(band, bandDark, MASK_DARK_MAX, 255, cv.THRESH_BINARY_INV);
const darkFraction = cv.countNonZero(bandDark) / bandPixels;
bandDark.delete();
band.delete();
const inkOk =
brightFraction <= GATE_INK_BAND_MAX_BRIGHT && darkFraction >= GATE_INK_BAND_MIN_DARK;
gray.delete();
const score = (darkOk / GATE_DARK_PROBES.length + (textOk ? 1 : 0) + (inkOk ? 1 : 0)) / 3;
return { pass: darkOk === GATE_DARK_PROBES.length && textOk && inkOk, score };
}
function parse(frame: Mat, t: number): DetectedEvent<MapStartData>[] {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
// 1. confirm the constant label — a gate hit without it is a lookalike
let label: RecognizedText | null = null;
let labelScore = 0;
if (labelGlyphs) {
const crop = copyRoi(gray, MODE_LABEL_ROI);
label = recognizeText(crop, labelGlyphs, {
binThreshold: TEXT_BIN_THRESHOLD,
minCharScore: 0.3,
});
crop.delete();
labelScore = closestBy(label.text, ALL_MODE_LABELS, (l) => l)?.score ?? 0;
if (labelScore < LABEL_MIN_SCORE) {
gray.delete();
return [];
}
}
// 2. mode title: find the 1-2 text lines, OCR each, snap the joined text
let mode: string | null = null;
let modeScore = 0;
let modeReading = "";
if (modeGlyphs) {
const block = copyRoi(gray, MODE_BLOCK_ROI);
// find the line bands on the masked block so bright background rows
// don't merge/invent bands, but OCR the raw crop: the mask radius
// clips the widest title strokes and costs recognition accuracy.
// Each band is trimmed to its text columns (also from the mask) —
// the block is wide enough for the longest localized titles, and at
// full width the raw OCR picks up background junk at the edges.
const masked = maskNearDark(block, BLOCK_MASK_RADIUS);
const binary = new cv.Mat();
cv.threshold(masked, binary, TEXT_BIN_THRESHOLD, 255, cv.THRESH_BINARY);
masked.delete();
const bands = findLineBands(binary);
const lines: string[] = [];
for (const band of bands) {
const extent = bandExtent(binary, band);
if (!extent) continue;
const pad = 3;
const y0 = Math.max(0, band.y0 - pad);
const x0 = Math.max(0, extent.x0 - pad);
const line = copyRoi(block, {
x: x0,
y: y0,
w: Math.min(block.cols, extent.x1 + 1 + pad) - x0,
h: Math.min(block.rows, band.y1 + pad) - y0,
});
const read = recognizeText(line, modeGlyphs, {
binThreshold: TEXT_BIN_THRESHOLD,
minCharScore: 0.3,
});
line.delete();
if (read.text.trim()) lines.push(read.text.trim());
}
binary.delete();
block.delete();
modeReading = lines.join(" ");
const match = modeReading ? closestBy(modeReading, ALL_MODE_ENTRIES, (e) => e.text) : null;
if (match) {
modeScore = match.score;
if (match.score >= MIN_MATCH_SCORE) mode = match.entry.canonical;
}
}
// 3. stage name. The backdrop is live gameplay — dark water on one
// stage, a white mall floor brighter than the text's antialiased edges
// on another — so no single binarization works everywhere: read the
// near-dark-masked crop (wins on dark scenes) and the raw crop at a few
// rising thresholds (a high one isolates the text's saturated-white
// core from an only-nearly-white floor), and keep whichever snaps best.
let stage: string | null = null;
let stageScore = 0;
let stageReading = "";
if (stageGlyphs) {
const rgbaCrop = copyRoi(frame, STAGE_ROI);
const bright = minChannel(rgbaCrop);
rgbaCrop.delete();
const masked = maskNearDark(bright, STAGE_MASK_RADIUS);
const attempts: [Mat, number][] = [
[masked, STAGE_BIN_THRESHOLD],
...STAGE_RAW_BIN_THRESHOLDS.map((thr): [Mat, number] => [bright, thr]),
];
for (const [input, binThreshold] of attempts) {
const read = recognizeText(input, stageGlyphs, { binThreshold, minCharScore: 0.3 });
const match = read.text ? closestBy(read.text, ALL_STAGE_ENTRIES, (e) => e.text) : null;
if (match && match.score > stageScore) {
stageScore = match.score;
stageReading = read.text;
if (match.score >= MIN_MATCH_SCORE) stage = match.entry.canonical;
}
}
masked.delete();
bright.delete();
}
gray.delete();
return [
{
type: MAP_START_EVENT_TYPE,
t,
// mean like every other detector: a clean mode read must survive an
// unreadable stage (map-start is the only mode source for VoD matches),
// not be zeroed by it
confidence: (modeScore + stageScore) / 2,
data: { mode, stage },
debug: {
label: label?.text,
labelScore,
modeReading,
modeScore,
stageReading,
stageScore,
},
},
];
}
return { id: "map-start", gate, parse };
}

115
app/features/cv/core/detectors/map-start/rois.ts Normal file
View File

@@ -0,0 +1,115 @@
/**
* ALL map-start ROI coordinates, in canonical 1920x1080 space.
* Calibrated against the map-start/ fixtures via row/column profiling.
*
* The match-intro splash overlays live gameplay with:
* - a big black ink splat top-center carrying the constant "MODE" label
* (~48px BlitzMain caps), the mode title in large BlitzBold (~76px tight
* caps, wrapping to two lines for the longer mode names), and an
* objective subtitle below it;
* - the stage name bottom-right in BlitzMain (~40px tight height);
* - the eight players' splash tags along the left/right edges.
*/
import type { Roi } from "../../canonical";
/**
* The constant "MODE" label (white caps centered at x=960). Kept tight
* inside the label's black pill (x≈825-1110): the splash sits on live
* gameplay, and on bright stages (Mahi-Mahi's water/docks) a wider crop
* picks up background past the pill edges that garbles the label read.
*/
export const MODE_LABEL_ROI: Roi = { x: 850, y: 268, w: 220, h: 62 };
export const MODE_LABEL_TEXT_HEIGHT = 48;
/**
* The mode title block below the label. The title is one or two lines
* depending on the mode name; parse finds the actual lines by row
* projection instead of assuming positions. Ends above the objective
* subtitle (~y678) so the subtitle never leaks into the last band. Wide
* enough for the longest single-line localized titles ("Herrschaft",
* "Spetterzone" — ~660px centered on 960): at 520 wide the German H/t
* were clipped off and the title read "lerrschaf".
*/
export const MODE_BLOCK_ROI: Roi = { x: 620, y: 380, w: 680, h: 285 };
export const MODE_TEXT_HEIGHT = 76;
/** A block row is text when it has at least this many bright pixels... */
export const LINE_MIN_ROW_PIXELS = 40;
/**
* ...and at least this fraction of the block's strongest row: background
* leaking past the splat's rim puts a scene-dependent floor under every
* row (Robo ROM-en's bright mall merged the title and the junk below it
* into one 227px band at the fixed threshold alone).
*/
export const LINE_ROW_FRACTION = 0.25;
/** Text rows closer than this merge into one line band. */
export const LINE_GAP_TOLERANCE = 8;
/** Discard bands shorter than this (splat-texture speckle, drips). */
export const LINE_MIN_HEIGHT = 40;
/** Stage name, bottom-right over live gameplay. */
export const STAGE_ROI: Roi = { x: 1300, y: 984, w: 600, h: 56 };
export const STAGE_TEXT_HEIGHT = 40;
/** White text on the near-black splat binarizes cleanly and high. */
export const TEXT_BIN_THRESHOLD = 190;
/**
* Bright-background suppression (needed on light stages like Mahi-Mahi,
* where water reads ~200 gray and white docks 230+): a pixel only counts
* as text when a near-black pixel sits within the mask radius — the mode
* title borders the splat ink and the stage name carries a dark drop
* shadow, while open bright background has no darkness nearby. The radius
* must exceed the text's stroke half-width or it eats the glyph cores:
* ~12 for the 76px title (band finding only; the OCR runs on the raw
* crop), ~6 for the 40px stage line.
*/
export const MASK_DARK_MAX = 70;
export const BLOCK_MASK_RADIUS = 12;
export const STAGE_MASK_RADIUS = 6;
/**
* The stage line reads the per-pixel min channel at a higher threshold:
* blue-tinted water drops with its low red channel while the white glyph
* cores stay near 255. The near-dark mask fails when the backdrop is
* brighter than the mask threshold everywhere (Robo ROM-en's white mall
* floor fuses into the glyphs), so parse also tries the raw crop at these
* rising thresholds — the text's saturated core outlasts a nearly-white
* floor — and keeps whichever read snaps best.
*/
export const STAGE_BIN_THRESHOLD = 210;
export const STAGE_RAW_BIN_THRESHOLDS: readonly number[] = [225, 235, 245];
/**
* Gate probes: splat ink flanking the "MODE" label and in the gap between
* the label and the title. The splat texture has holes that show the scene
* behind it (bright water on Mahi-Mahi), so these sit on spots verified
* solid across the fixtures (mean ≤8 regardless of the scene); the
* threshold is well above ink yet under the dark scoreboard pills (~51+).
* The flanking pair sits outside x 838-1082, the widest any language's
* label text reaches ("Kampfart", "Vechtstijl") — closer in, the label's
* own white strokes blow the probe means.
*/
export const GATE_DARK_PROBES: readonly Roi[] = [
{ x: 780, y: 280, w: 30, h: 20 },
{ x: 1105, y: 280, w: 30, h: 20 },
{ x: 850, y: 355, w: 30, h: 20 },
{ x: 1030, y: 355, w: 30, h: 20 },
];
export const GATE_DARK_MAX_MEAN = 45;
/**
* The core of the label-to-title gap is solid splat ink (no texture
* holes): near-total darkness and zero bright pixels. The death burst
* runs its "Splatted by" line through this band (bright fraction 0.17+)
* and the scoreboards' dark pills only reach ~0.83 dark fraction.
*/
export const GATE_INK_BAND: Roi = { x: 840, y: 350, w: 230, h: 36 };
export const GATE_INK_BAND_MAX_BRIGHT = 0.005;
export const GATE_INK_BAND_MIN_DARK = 0.95;
/** MODE_LABEL_ROI must contain near-white pixels... */
export const GATE_TEXT_MIN_MAX = 210;
/** ...at a text fill fraction: "MODE" fills ~0.25 of the tight ROI, while
* stray white on the other screens stays under ~0.13. */
export const GATE_TEXT_MIN_FRACTION = 0.05;
export const GATE_TEXT_MAX_FRACTION = 0.35;

20
app/features/cv/core/detectors/minimap/abilities.ts Normal file
View File

@@ -0,0 +1,20 @@
/**
* Ability badge templates at the minimap cards' badge size (⌀~44 on both
* the own cards and the enemy panel; one role, mains only — the cards show
* no sub-ability slots). Built via the death panel's badge compositor.
*/
import type { FrameData } from "../../image";
import { buildAbilityRole } from "../death/abilities";
import type { WeaponTemplate } from "../scoreboard/weapons";
import { BADGE_ART_RATIO, BADGE_TEMPLATE_SIZES, MINIMAP_ABILITY_INK_THRESHOLD } from "./rois";
export function prepareMinimapAbilityTemplates(
icons: { id: string; image: FrameData }[],
): WeaponTemplate[] {
return buildAbilityRole(
icons,
BADGE_TEMPLATE_SIZES,
BADGE_ART_RATIO,
MINIMAP_ABILITY_INK_THRESHOLD,
);
}

591
app/features/cv/core/detectors/minimap/index.ts Normal file
View File

@@ -0,0 +1,591 @@
/**
* MinimapDetector: parses the in-match map overlay (opened with X) — the
* own-team callout cards (name, main weapon, the three main-ability
* badges), the enemy panel rows (weapon, abilities; the game shows no
* enemy names) — plus the stage, matched from the drawn map (stage.ts).
* The goal is the most complete read of every card/row; per-match state
* (respawn cross-outs, special charge, map control) is deliberately not
* reported.
*
* Two screen states still steer the reads without being emitted:
* - a respawning player's card is struck through with a large team-color
* X that covers the name and badges (own cards also lose the weapon;
* enemy rows keep theirs — the X spares the row's weapon icon). Reading
* through the X yields garbage, so an occlusion probe skips the covered
* fields and reports them null;
* - a charged special swaps the card/row background for a light camo
* pattern; weapons are full-color icon art matched against art-cropped
* template sets composited for the surface actually behind them — bg-40
* for the translucent dark cards/rows, bg-150 for the camo (dark
* templates anti-correlate there) — so a corner-brightness probe picks
* the template set, ink threshold, and score floor per card.
*/
import { getCV, type Mat } from "../../cv";
import { type GlyphSet, scaleGlyphSet } from "../../glyphs";
import {
copyRoi,
cropRoi,
laplacianAbs,
maxBrightness,
meanBrightness,
type Roi,
} from "../../image";
import { WEAPON_NAMES } from "../death/weapon-names";
import type { ScoreboardResources } from "../scoreboard/index";
import { type ParsedName, parseName } from "../scoreboard/names";
import {
disambiguateWeaponBySub,
matchSpecial,
tiedWeaponsWithDistinctSubs,
} from "../scoreboard/specials";
import { matchWeapon, type WeaponMatch } from "../scoreboard/weapons";
import type { DetectedEvent, Detector, GateResult } from "../types";
import {
badgeRoi,
CARD_LAYOUTS,
type CardSlot,
CROSS_MIN_FRACTION,
CROSS_SATURATION_MIN,
CROSS_VALUE_MIN,
ENEMY_BADGE_XS,
ENEMY_ROW_CYS,
enemyCrossRoi,
enemySubTileRoi,
enemyWeaponRoi,
GATE_BRIGHT_MIN_MAX,
GATE_CLOSE_BRIGHT,
GATE_CLOSE_DARK_PROBES,
GATE_DARK_MAX_MEAN,
GATE_SPAWN_BRIGHT,
GATE_SPAWN_DARK_PROBES,
GATE_SPECTATOR_X_BRIGHT,
GATE_SPECTATOR_X_DARK,
MINIMAP_ABILITY_INK_THRESHOLD,
MINIMAP_WEAPON_INK_THRESHOLD,
NAME_BIN_THRESHOLD,
NAME_TEXT_HEIGHT,
PRESENCE_MIN_LAPLACIAN,
SPECIAL_READY_INK_THRESHOLD,
SPECIAL_READY_MIN_CORNER_MEAN,
SPECIAL_READY_WEAPON_MIN_SCORE,
SPECTATOR_ENEMY_DX,
SPECTATOR_NAME_TEXT_HEIGHTS,
SPECTATOR_SLOTS,
spectatorCardLayout,
WEAPON_MIN_SCORE,
} from "./rois";
import { matchStage, plannerSignature, type StageMatch } from "./stage";
export interface MinimapTeammate {
/** which callout card: super-jump slot, or the POV player's own card */
slot: CardSlot;
/** card name; null when covered by a respawn cross-out or unreadable */
name: string | null;
/** main weapon (canonical English name); null when unreadable/covered */
weapon: string | null;
/** sendou main-weapon id; null when unreadable/covered */
weaponId: number | null;
/**
* the card's three main abilities, [head, clothes, shoes]
* (assets/cv/abilities id space, null per unreadable badge);
* empty when a respawn cross-out sits over the badges
*/
abilities: (string | null)[];
}
export interface MinimapEnemy {
/**
* the POV overlay shows no enemy names (always null there); the
* spectator screen does, so spectator rows carry them
*/
name: string | null;
/** readable even on struck rows: the cross-out spares the weapon icon */
weapon: string | null;
weaponId: number | null;
abilities: (string | null)[];
}
export interface MinimapData {
/**
* canonical English stage name, matched from the drawn map against the
* planner renders (stage.ts); null when no stage matched confidently or
* the planner signatures were not loaded. The mode is not identifiable
* this way (see stage.ts) and is left to the mode-bearing detectors.
*/
stage: string | null;
/**
* true when the frame is a casted stream's 8-player spectator map screen
* rather than the POV overlay: the alpha (left) column is reported as
* teammates (d-pad slots up/right/down/left) and the bravo (right)
* column as enemy rows — with names, which this screen shows
*/
spectator: boolean;
/** own-team callout cards; a slot missing from the frame is omitted */
teammates: MinimapTeammate[];
/** enemy panel rows, top to bottom */
enemies: MinimapEnemy[];
}
export const MINIMAP_EVENT_TYPE = "Minimap";
/** Badge match below this is reported as null (kept in debug). */
const ABILITY_MIN_SCORE = 0.45;
/**
* Light-camo (special-charged) probe: min of the two 8x8 top-corner means
* of the weapon box. Camo backgrounds brighten both corners (140-165); on
* a dark card at least one stays dark even when avatar bleed or a
* cross-out stroke lights up the other.
*/
function minTopCornerMean(gray: Mat, roi: Roi): number {
const corners: Roi[] = [
{ x: roi.x, y: roi.y, w: 8, h: 8 },
{ x: roi.x + roi.w - 8, y: roi.y, w: 8, h: 8 },
];
return Math.min(...corners.map((c) => meanBrightness(gray, c)));
}
/** fraction of the probe that is saturated-and-bright (cross-out strokes) */
function saturatedFraction(hsv: Mat, roi: Roi): number {
const m = copyRoi(hsv, roi);
const n = m.rows * m.cols;
const md = m.data;
let hit = 0;
for (let i = 0; i < n; i++) {
if (md[i * 3 + 1]! >= CROSS_SATURATION_MIN && md[i * 3 + 2]! >= CROSS_VALUE_MIN) {
hit++;
}
}
m.delete();
return hit / n;
}
export function createMinimapDetector(resources: ScoreboardResources): Detector<MinimapData> {
const cv = getCV();
const nameGlyphs: GlyphSet | null = resources.nameGlyphs
? scaleGlyphSet(resources.nameGlyphs, NAME_TEXT_HEIGHT / resources.nameGlyphs.height)
: null;
const spectatorNameGlyphs: GlyphSet[] = resources.nameGlyphs
? SPECTATOR_NAME_TEXT_HEIGHTS.map((h) =>
h === NAME_TEXT_HEIGHT
? nameGlyphs!
: scaleGlyphSet(resources.nameGlyphs!, h / resources.nameGlyphs!.height),
)
: [];
const cardWeapons = resources.minimapCardWeapons ?? null;
const lightWeapons = resources.minimapLightWeapons ?? null;
const badges = resources.minimapAbilities ?? null;
const subWeapons = resources.minimapSubWeapons ?? null;
const plannerStages = resources.plannerStages ?? null;
/** Identify the stage from the drawn map; contributes to confidence. */
function detectStage(frame: Mat, confidences: number[]): StageMatch | null {
if (!plannerStages?.length) return null;
const sig = plannerSignature(frame);
const match = matchStage(sig, plannerStages);
if (match) confidences.push(match.score);
return match;
}
function probeGate(
gray: Mat,
darkProbes: readonly Roi[],
brightProbes: readonly Roi[],
): GateResult {
let darkOk = 0;
for (const roi of darkProbes) {
if (meanBrightness(gray, roi) <= GATE_DARK_MAX_MEAN) darkOk++;
}
let brightOk = 0;
for (const roi of brightProbes) {
if (maxBrightness(gray, roi) >= GATE_BRIGHT_MIN_MAX) brightOk++;
}
return {
pass: darkOk === darkProbes.length && brightOk === brightProbes.length,
score: (darkOk / darkProbes.length + brightOk / brightProbes.length) / 2,
};
}
/** POV overlay chrome: close-button disc + Spawn Point pill. */
function overlayGate(gray: Mat): GateResult {
return probeGate(
gray,
[...GATE_CLOSE_DARK_PROBES, ...GATE_SPAWN_DARK_PROBES],
[GATE_CLOSE_BRIGHT, GATE_SPAWN_BRIGHT],
);
}
/** Spectator screen: the X jump-button disc beside the 8th player card. */
function spectatorGate(gray: Mat): GateResult {
return probeGate(gray, GATE_SPECTATOR_X_DARK, GATE_SPECTATOR_X_BRIGHT);
}
function gate(frame: Mat): GateResult {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const overlay = overlayGate(gray);
const spectator = spectatorGate(gray);
gray.delete();
return {
pass: overlay.pass || spectator.pass,
score: Math.max(overlay.score, spectator.score),
variant: spectator.pass ? "spectator" : "overlay",
};
}
function matchBadges(
rgb: Mat,
centers: readonly (readonly [number, number])[],
inkThreshold: number,
confidences: number[],
debugRow: (WeaponMatch | null)[],
): (string | null)[] {
if (!badges) return [null, null, null];
return centers.map(([cx, cy]) => {
const crop = cropRoi(rgb, badgeRoi(cx, cy));
const match = matchWeapon(crop, badges, { inkThreshold });
crop.delete();
debugRow.push(match);
confidences.push(Math.max(0, match.score));
return match.score >= ABILITY_MIN_SCORE ? match.id : null;
});
}
/**
* Near-tied weapon icons whose kits differ by sub (plain vs Custom
* Dualie Squelchers): let the card/row's team-tinted sub tile break the
* tie. Shape-only matching survives the tint, the camo surface, and a
* cross-out stroke clipping the tile.
*/
function resolveTieBySubTile(rgb: Mat, weapon: WeaponMatch, tile: Roi): WeaponMatch {
if (!subWeapons?.length || !tiedWeaponsWithDistinctSubs(weapon)) return weapon;
const crop = cropRoi(rgb, tile);
const sub = matchSpecial(crop, subWeapons);
crop.delete();
return disambiguateWeaponBySub(weapon, sub);
}
/** Try the name band at each spectator glyph height; best read wins. */
function bestNameRead(gray: Mat, roi: Roi): ParsedName | null {
let best: ParsedName | null = null;
for (const set of spectatorNameGlyphs) {
const band = copyRoi(gray, roi);
const parsed = parseName(band, set, { binThreshold: NAME_BIN_THRESHOLD });
band.delete();
if (!best || parsed.confidence > best.confidence) best = parsed;
}
return best;
}
/**
* The spectator screen's 8-card grid doesn't share the overlay's ROIs
* (running the overlay parse against it reads phantom cards), so it gets
* its own card loop: same fields per card, both columns carry names.
*/
function parseSpectator(frame: Mat, gray: Mat, t: number): DetectedEvent<MinimapData>[] {
const rgb = new cv.Mat();
cv.cvtColor(frame, rgb, cv.COLOR_RGBA2RGB);
const hsv = new cv.Mat();
cv.cvtColor(rgb, hsv, cv.COLOR_RGB2HSV);
const lap = laplacianAbs(gray);
const confidences: number[] = [];
const debug: Record<string, unknown> = { spectator: true };
const teammates: MinimapTeammate[] = [];
const enemies: MinimapEnemy[] = [];
const cardDebug: Record<string, unknown>[] = [];
for (const dx of [0, SPECTATOR_ENEMY_DX]) {
for (let row = 0; row < 4; row++) {
const layout = spectatorCardLayout(row, dx);
const presence = meanBrightness(lap, layout.name);
if (presence < PRESENCE_MIN_LAPLACIAN) {
cardDebug.push({ dx, row, presence, skipped: true });
continue;
}
const crossFraction = saturatedFraction(hsv, layout.cross);
const occluded = crossFraction >= CROSS_MIN_FRACTION;
const cornerMin = minTopCornerMean(gray, layout.weapon);
const lightSurface = cornerMin >= SPECIAL_READY_MIN_CORNER_MEAN;
let name: string | null = null;
let nameRaw = "";
let weapon: WeaponMatch | null = null;
const badgeDebug: (WeaponMatch | null)[] = [];
let abilities: (string | null)[] = [];
if (!occluded) {
const parsed = bestNameRead(gray, layout.name);
if (parsed) {
nameRaw = parsed.raw.text;
if (parsed.name.length > 0) name = parsed.name;
confidences.push(parsed.confidence);
}
const templates = lightSurface ? lightWeapons : cardWeapons;
if (templates) {
const crop = cropRoi(rgb, layout.weapon);
weapon = matchWeapon(crop, templates, {
inkThreshold: lightSurface
? SPECIAL_READY_INK_THRESHOLD
: Math.max(MINIMAP_WEAPON_INK_THRESHOLD, Math.round(cornerMin) + 50),
});
crop.delete();
weapon = resolveTieBySubTile(rgb, weapon, layout.subTile);
confidences.push(Math.max(0, weapon.score));
}
abilities = matchBadges(
rgb,
layout.badges,
Math.max(MINIMAP_ABILITY_INK_THRESHOLD, Math.round(cornerMin) + 50),
confidences,
badgeDebug,
);
}
cardDebug.push({
dx,
row,
presence,
crossFraction,
occluded,
cornerMin,
lightSurface,
nameRaw,
weapon,
badges: badgeDebug,
});
const floor = lightSurface ? SPECIAL_READY_WEAPON_MIN_SCORE : WEAPON_MIN_SCORE;
const matched = weapon !== null && weapon.score >= floor ? weapon : null;
const fields = {
name,
weapon: matched ? (WEAPON_NAMES.get(matched.id) ?? null) : null,
weaponId: matched ? Number(matched.id) : null,
abilities,
};
if (dx === 0) {
teammates.push({ slot: SPECTATOR_SLOTS[row]!, ...fields });
} else {
enemies.push(fields);
}
}
}
debug.cards = cardDebug;
const stageMatch = detectStage(frame, confidences);
debug.stage = stageMatch;
rgb.delete();
hsv.delete();
lap.delete();
const confidence =
confidences.length > 0 ? confidences.reduce((a, b) => a + b, 0) / confidences.length : 0;
return [
{
type: MINIMAP_EVENT_TYPE,
t,
confidence,
data: {
stage: stageMatch?.stage ?? null,
spectator: true,
teammates,
enemies,
},
debug,
},
];
}
function parse(frame: Mat, t: number, gateResult?: GateResult): DetectedEvent<MinimapData>[] {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const isSpectator = gateResult?.variant
? gateResult.variant === "spectator"
: spectatorGate(gray).pass;
if (isSpectator) {
const events = parseSpectator(frame, gray, t);
gray.delete();
return events;
}
const rgb = new cv.Mat();
cv.cvtColor(frame, rgb, cv.COLOR_RGBA2RGB);
const hsv = new cv.Mat();
cv.cvtColor(rgb, hsv, cv.COLOR_RGB2HSV);
const lap = laplacianAbs(gray);
const confidences: number[] = [];
const debug: Record<string, unknown> = {};
// 1. own-team callout cards
const teammates: MinimapTeammate[] = [];
const cardDebug: Record<string, unknown>[] = [];
for (const layout of CARD_LAYOUTS) {
// presence: the card is crisp UI, absent slots show blurred scene
const presence = meanBrightness(lap, layout.name);
if (presence < PRESENCE_MIN_LAPLACIAN) {
cardDebug.push({ slot: layout.slot, presence, skipped: true });
continue;
}
const crossFraction = saturatedFraction(hsv, layout.cross);
const occluded = crossFraction >= CROSS_MIN_FRACTION;
const cornerMin = minTopCornerMean(gray, layout.weapon);
const lightSurface = cornerMin >= SPECIAL_READY_MIN_CORNER_MEAN;
let name: string | null = null;
let nameRaw = "";
let weapon: WeaponMatch | null = null;
const badgeDebug: (WeaponMatch | null)[] = [];
let abilities: (string | null)[] = [];
if (!occluded) {
if (nameGlyphs) {
const band = copyRoi(gray, layout.name);
const parsed = parseName(band, nameGlyphs, {
binThreshold: NAME_BIN_THRESHOLD,
});
band.delete();
nameRaw = parsed.raw.text;
if (parsed.name.length > 0) name = parsed.name;
confidences.push(parsed.confidence);
}
const templates = lightSurface ? lightWeapons : cardWeapons;
if (templates) {
const crop = cropRoi(rgb, layout.weapon);
weapon = matchWeapon(crop, templates, {
inkThreshold: lightSurface ? SPECIAL_READY_INK_THRESHOLD : MINIMAP_WEAPON_INK_THRESHOLD,
});
crop.delete();
weapon = resolveTieBySubTile(rgb, weapon, layout.subTile);
confidences.push(Math.max(0, weapon.score));
}
abilities = matchBadges(
rgb,
layout.badges,
lightSurface
? Math.max(MINIMAP_ABILITY_INK_THRESHOLD, Math.round(cornerMin) + 50)
: MINIMAP_ABILITY_INK_THRESHOLD,
confidences,
badgeDebug,
);
}
cardDebug.push({
slot: layout.slot,
presence,
crossFraction,
occluded,
cornerMin,
lightSurface,
nameRaw,
weapon,
badges: badgeDebug,
});
const floor = lightSurface ? SPECIAL_READY_WEAPON_MIN_SCORE : WEAPON_MIN_SCORE;
const matched = weapon !== null && weapon.score >= floor ? weapon : null;
// an occluding cross-out is itself proof the card is drawn
const hasEvidence =
occluded || name !== null || matched !== null || abilities.some((a) => a !== null);
if (!hasEvidence) continue;
teammates.push({
slot: layout.slot,
name,
weapon: matched ? (WEAPON_NAMES.get(matched.id) ?? null) : null,
weaponId: matched ? Number(matched.id) : null,
abilities,
});
}
debug.cards = cardDebug;
// 2. enemy panel rows
const enemies: MinimapEnemy[] = [];
const enemyDebug: Record<string, unknown>[] = [];
for (const cy of ENEMY_ROW_CYS) {
const weaponRoi = enemyWeaponRoi(cy);
const presence = meanBrightness(lap, weaponRoi);
if (presence < PRESENCE_MIN_LAPLACIAN) {
enemyDebug.push({ cy, presence, skipped: true });
continue;
}
const crossFraction = saturatedFraction(hsv, enemyCrossRoi(cy));
const occluded = crossFraction >= CROSS_MIN_FRACTION;
// light camo rows: pick the template variant by the weapon box's
// corner brightness and raise the ink threshold past that background
const cornerMin = minTopCornerMean(gray, weaponRoi);
const lightSurface = cornerMin >= SPECIAL_READY_MIN_CORNER_MEAN;
const templates = lightSurface ? lightWeapons : cardWeapons;
const inkThreshold = lightSurface
? SPECIAL_READY_INK_THRESHOLD
: Math.max(MINIMAP_WEAPON_INK_THRESHOLD, Math.round(cornerMin) + 50);
let weapon: WeaponMatch | null = null;
if (templates) {
const crop = cropRoi(rgb, weaponRoi);
weapon = matchWeapon(crop, templates, { inkThreshold });
crop.delete();
weapon = resolveTieBySubTile(rgb, weapon, enemySubTileRoi(cy));
confidences.push(Math.max(0, weapon.score));
}
const badgeDebug: (WeaponMatch | null)[] = [];
const abilities: (string | null)[] = occluded
? []
: matchBadges(
rgb,
ENEMY_BADGE_XS.map((cx) => [cx, cy] as const),
Math.max(MINIMAP_ABILITY_INK_THRESHOLD, Math.round(cornerMin) + 50),
confidences,
badgeDebug,
);
enemyDebug.push({
cy,
presence,
crossFraction,
occluded,
lightSurface,
cornerMin,
weapon,
badges: badgeDebug,
});
const floor = lightSurface ? SPECIAL_READY_WEAPON_MIN_SCORE : WEAPON_MIN_SCORE;
const matched = weapon !== null && weapon.score >= floor ? weapon : null;
enemies.push({
name: null,
weapon: matched ? (WEAPON_NAMES.get(matched.id) ?? null) : null,
weaponId: matched ? Number(matched.id) : null,
abilities,
});
}
debug.enemies = enemyDebug;
const stageMatch = detectStage(frame, confidences);
debug.stage = stageMatch;
gray.delete();
rgb.delete();
hsv.delete();
lap.delete();
const confidence =
confidences.length > 0 ? confidences.reduce((a, b) => a + b, 0) / confidences.length : 0;
return [
{
type: MINIMAP_EVENT_TYPE,
t,
confidence,
data: {
stage: stageMatch?.stage ?? null,
spectator: false,
teammates,
enemies,
},
debug,
},
];
}
return { id: "minimap", gate, parse };
}

287
app/features/cv/core/detectors/minimap/rois.ts Normal file
View File

@@ -0,0 +1,287 @@
/**
* ALL minimap ROI coordinates, in canonical 1920x1080 space. Calibrated
* against the minimap/ fixtures via tools/dump-crops.ts crops plus template
* relocation sweeps (each card's ability badges sit on an exact 48px pitch,
* which pins the grid origins).
*
* The in-match map overlay (opened with X) draws over gaussian-blurred live
* gameplay:
* - four own-team callout cards: three teammates at fixed super-jump slots
* (d-pad up = top-center, left = mid-left, right = mid-right) and the POV
* player's own card bottom-left (no d-pad). Each carries the player name
* (BlitzMain, ~29px caps), the main weapon as full-color icon art on the
* translucent dark card, team-tinted sub/special tiles, and three
* main-ability badges (⌀~44). A respawning player's card is struck
* through with a large team-color X whose arms cross at the card center;
* - the enemy panel top-right: four rows of full-color weapon icon art,
* sub/special tiles, and the same three ability badges — no names. A
* respawning enemy's row is struck through (the X spares the weapon icon
* at the row's left edge);
* - special ready: a card/row whose player has a charged special swaps its
* dark background for a light gray-green camo pattern (~150 mean, low
* saturation), same geometry, inverted contrast around the weapon art;
* - constant chrome: the close-button disc top-left (dark disc, white X)
* and the Spawn Point pill bottom-center. The pill label is localized,
* so the gate reads shapes (disc, strokes, the white jump-arrows icon),
* never text.
*/
import type { Roi } from "../../canonical";
/** "self" and "down" never coexist: the POV overlay has no down slot (the
* player's own card replaces it), the spectator grid has no self card. */
export type CardSlot = "up" | "left" | "right" | "self" | "down";
export interface CardLayout {
slot: CardSlot;
/** name text band (BlitzMain caps ~29px plus outline/descender margin) */
name: Roi;
/** main-weapon silhouette box (icons render ~31-48px tall) */
weapon: Roi;
/** sub-weapon tile: saturated team-color art, the ink-color anchor */
subTile: Roi;
/** the three main-ability badge centers, 48px pitch */
badges: readonly (readonly [number, number])[];
/**
* cross-out probe at the card center (where the X's arms meet): white
* name glyphs and the dark pill are unsaturated, the X core is not
*/
cross: Roi;
}
/**
* The right card is the left card's layout shifted +1352px (verified on the
* struck-through fixture card only — re-derive from an uncrossed right-slot
* fixture when one lands). The self card differs: avatar leftmost, larger
* left inset for the name, no d-pad.
*/
export const CARD_LAYOUTS: readonly CardLayout[] = [
{
slot: "up",
name: { x: 872, y: 46, w: 300, h: 44 },
weapon: { x: 860, y: 83, w: 84, h: 54 },
subTile: { x: 932, y: 98, w: 38, h: 41 },
badges: [
[1066, 114],
[1114, 114],
[1162, 114],
],
cross: { x: 925, y: 78, w: 60, h: 32 },
},
{
slot: "left",
name: { x: 198, y: 492, w: 300, h: 44 },
weapon: { x: 193, y: 529, w: 84, h: 54 },
subTile: { x: 265, y: 544, w: 38, h: 41 },
badges: [
[392, 564],
[440, 564],
[488, 564],
],
cross: { x: 258, y: 524, w: 60, h: 32 },
},
{
slot: "right",
name: { x: 1550, y: 492, w: 300, h: 44 },
weapon: { x: 1545, y: 529, w: 84, h: 54 },
subTile: { x: 1617, y: 544, w: 38, h: 41 },
badges: [
[1744, 564],
[1792, 564],
[1840, 564],
],
cross: { x: 1610, y: 524, w: 60, h: 32 },
},
{
slot: "self",
name: { x: 126, y: 942, w: 300, h: 46 },
weapon: { x: 118, y: 985, w: 94, h: 55 },
subTile: { x: 193, y: 995, w: 38, h: 36 },
badges: [
[320, 1014],
[368, 1014],
[416, 1014],
],
cross: { x: 255, y: 968, w: 60, h: 32 },
},
];
/** Enemy panel row centers (65px pitch) and per-row element boxes. */
export const ENEMY_ROW_CYS = [82, 147, 213, 278] as const;
export function enemyWeaponRoi(cy: number): Roi {
return { x: 1541, y: cy - 26, w: 58, h: 52 };
}
export function enemySubTileRoi(cy: number): Roi {
return { x: 1602, y: cy - 20, w: 39, h: 40 };
}
export const ENEMY_BADGE_XS = [1730, 1778, 1826] as const;
/**
* The row's X arms meet in the dark gap between the special tile and the
* first badge; on a clean row the gap stays unsaturated.
*/
export function enemyCrossRoi(cy: number): Roi {
return { x: 1685, y: cy - 12, w: 28, h: 24 };
}
/** Badge search box (badges ⌀~44; the box height keeps larger sets out). */
export function badgeRoi(cx: number, cy: number): Roi {
return { x: cx - 26, y: cy - 26, w: 52, h: 52 };
}
export const BADGE_TEMPLATE_SIZES = [38, 42, 46] as const;
/** Badge art fills the circle like the death panel's mains. */
export const BADGE_ART_RATIO = 1.0;
/**
* Badge ink threshold: badge circles are near-black; the enemy panel's
* translucent pink bleeds into the box corners at ~150-180, a constant
* penalty across candidates that leaves the ranking intact.
*/
export const MINIMAP_ABILITY_INK_THRESHOLD = 90;
/** BlitzMain caps measure 28-29px on every card (self included). */
export const NAME_TEXT_HEIGHT = 29;
export const NAME_BIN_THRESHOLD = 170;
/**
* Cross-out probe: fraction of saturated-and-bright pixels (HSV, 0..255
* channels). Struck cards measure 0.26-0.38, clean ones <= 0.01.
*/
export const CROSS_SATURATION_MIN = 110;
export const CROSS_VALUE_MIN = 110;
export const CROSS_MIN_FRACTION = 0.08;
/**
* Weapon template variants (template prep in scoreboard/weapons.ts): cards
* and enemy rows both draw full-color icon art. Both minimap sets are built
* with cropToArt — the game renders the icon at the equivalent of a
* ~44-60px padded square, and the 54px-tall card box can only host the
* larger sizes once the transparent padding is trimmed (the Splatana
* Stamper on the special-ready fixture card is unmatchable without it).
* Dark surfaces (translucent card/row over the blurred scene, measured
* ~15-90) match against a bg-40 composite; special-ready camo surfaces
* (~150 mean) against a bg-150 composite — each with an ink threshold
* clearing its background.
*/
/**
* Sub-weapon silhouette heights for the ~39x40 sub tiles (art renders
* ~26-34px inside the dark plate). Matched shape-only (specials.ts), which
* survives the team tint, the camo surround, and even a cross-out stroke
* clipping the tile — used only to split near-tied main-weapon icons whose
* kits carry different subs (plain vs Custom Dualie Squelchers).
*/
export const SUB_TILE_TEMPLATE_SIZES = [24, 27, 30, 33, 36] as const;
export const CARD_WEAPON_BACKGROUND = 40;
export const SPECIAL_READY_BACKGROUND = 150;
export const MINIMAP_WEAPON_TEMPLATE_SIZES = [40, 44, 48, 52, 56, 60, 64] as const;
export const MINIMAP_WEAPON_INK_THRESHOLD = CARD_WEAPON_BACKGROUND + 50;
export const SPECIAL_READY_INK_THRESHOLD = SPECIAL_READY_BACKGROUND + 50;
/**
* Weapon-box corner mean above this = special-ready camo background. Probes
* take the MIN of the two top corners: camo corners measure 140-165 on both,
* while a dark card keeps at least one corner <=90 even when avatar bleed or
* a cross-out stroke brightens the other.
*/
export const SPECIAL_READY_MIN_CORNER_MEAN = 120;
/**
* Weapon score floors: camo surfaces score systematically lower (the blob
* pattern behind the art depresses NCC) — 0.45-0.61 on correct matches vs
* 0.77+ on dark surfaces.
*/
export const WEAPON_MIN_SCORE = 0.55;
export const SPECIAL_READY_WEAPON_MIN_SCORE = 0.42;
/**
* Card/row presence: the overlay is crisp while the scene behind it is
* gaussian-blurred, so mean |Laplacian| over the name band (cards) or the
* weapon box (enemy rows) separates a drawn element from see-through
* background regardless of what the blur happens to show.
*/
export const PRESENCE_MIN_LAPLACIAN = 8;
/** Gate probes (overlay variant): close-button disc + Spawn Point pill shapes. */
export const GATE_CLOSE_BRIGHT: Roi = { x: 90, y: 90, w: 16, h: 16 };
export const GATE_CLOSE_DARK_PROBES: readonly Roi[] = [
{ x: 88, y: 50, w: 20, h: 14 },
{ x: 88, y: 132, w: 20, h: 14 },
{ x: 58, y: 88, w: 14, h: 20 },
{ x: 132, y: 88, w: 14, h: 20 },
];
/** The white jump-arrows icon left of the (localized) pill label. */
export const GATE_SPAWN_BRIGHT: Roi = { x: 888, y: 963, w: 60, h: 60 };
export const GATE_SPAWN_DARK_PROBES: readonly Roi[] = [
{ x: 812, y: 958, w: 26, h: 14 },
{ x: 950, y: 1024, w: 60, h: 12 },
];
export const GATE_DARK_MAX_MEAN = 85;
export const GATE_BRIGHT_MIN_MAX = 210;
/**
* Spectator-variant gate: casted streams show the map as the 8-player
* spectator screen (all eight players carded left/right, A/B/Y/X jump
* buttons) instead of the POV overlay, and stream layouts routinely cover
* the overlay gate's corner chrome (close disc behind the branding
* top-left, spawn-pill area behind the bottom bar). The X jump-button disc
* beside the 8th player card (bottom-right) is rarely covered, so this
* variant gates there: bright probes trace the X glyph itself (crossing
* point plus the four stroke arms), dark probes sit in the disc's gaps at
* the glyph's cardinal edges — a solid white blob or a non-X glyph at the
* same spot lights the gaps and fails. Disc center (1424,712), aligned
* within ±4px across the attested stream layouts; the 12x12/8x8 probe
* boxes absorb that jitter. Measured margins: spectator frames read
* bright>=249 / dark<=65 against the shared 210/85 thresholds.
*/
export const GATE_SPECTATOR_X_BRIGHT: readonly Roi[] = [
{ x: 1418, y: 706, w: 12, h: 12 },
{ x: 1411, y: 692, w: 12, h: 12 },
{ x: 1425, y: 692, w: 12, h: 12 },
{ x: 1411, y: 720, w: 12, h: 12 },
{ x: 1425, y: 720, w: 12, h: 12 },
];
export const GATE_SPECTATOR_X_DARK: readonly Roi[] = [
{ x: 1399, y: 708, w: 8, h: 8 },
{ x: 1441, y: 708, w: 8, h: 8 },
{ x: 1420, y: 682, w: 8, h: 8 },
{ x: 1420, y: 734, w: 8, h: 8 },
];
/**
* Spectator screen card grid: four cards per column on a 120px row pitch,
* the right column the left one shifted +1348px (same relation as the
* overlay's right card, verified within ±4px across the attested stream
* layouts). The left column carries the alpha team — its cards show a
* d-pad glyph highlighting up/right/down/left, reported as the teammate
* slot — and the right column the bravo team (reported as enemy rows,
* though this screen does show their names). Each card: name line above
* (BlitzMain, same 29-30px caps as the overlay cards), then weapon icon
* art, sub + special tiles, and three ability badges on the overlay's
* 48px pitch. The cross-out probe sits in the dark gap between the
* special tile and the first badge (like the overlay enemy rows); no
* struck-through or special-ready spectator fixture is attested yet, so
* those probes reuse the overlay thresholds untuned.
*/
export const SPECTATOR_SLOTS: readonly CardSlot[] = ["up", "right", "down", "left"];
export const SPECTATOR_ENEMY_DX = 1348;
const SPECTATOR_ROW_PITCH = 120;
export function spectatorCardLayout(row: number, dx: number): Omit<CardLayout, "slot"> {
const dy = SPECTATOR_ROW_PITCH * row;
return {
name: { x: 198 + dx, y: 306 + dy, w: 310, h: 44 },
weapon: { x: 196 + dx, y: 350 + dy, w: 66, h: 54 },
subTile: { x: 264 + dx, y: 354 + dy, w: 38, h: 42 },
badges: [
[390 + dx, 374 + dy],
[438 + dx, 374 + dy],
[486 + dx, 374 + dy],
],
cross: { x: 344 + dx, y: 362 + dy, w: 20, h: 24 },
};
}
/**
* Spectator name reads are tried at both heights and the more confident
* read wins: the JPEG/upscale blur across stream captures moves the best
* fit between 29 and 30 per card (measured 0.85-0.93 at the winning
* height, with the loser dropping glyphs' dakuten or bar lengths).
*/
export const SPECTATOR_NAME_TEXT_HEIGHTS = [29, 30] as const;

217
app/features/cv/core/detectors/minimap/stage.ts Normal file
View File

@@ -0,0 +1,217 @@
/**
* Stage identification for the minimap overlay by matching the drawn map
* against the sendou.ink planner renders (planner-maps/, one PNG per stage x
* mode, committed compactly as the signature atlas in assets/cv/planner/).
*
* The map is drawn sharp over gaussian-blurred gameplay, so a Laplacian edge
* mask isolates the render; team ink is saturated while the structural
* skeleton (white walls, grate crosshatch, platform edges) is not, so
* dropping saturated edges leaves an ink-invariant signature that survives
* whatever colors the match happens to paint. The signature is downscaled and
* blurred, which also absorbs the small scale/offset difference between the
* POV overlay (map fills the screen) and the spectator screen (map smaller,
* centered) and lets one atlas serve both via a short translation search.
*
* Stage separates cleanly this way (best-stage NCC leads the next stage by
* ~0.17-0.26 on the fixtures). Mode does NOT: the only thing distinguishing a
* stage's five renders is the colored objective marker (splat-zone checker,
* clam baskets, rainmaker checkpoints) that the ink-invariance strips, so the
* atlas keeps all five per stage only to match whichever mode the frame is;
* the reported result is the winning tile's stage. Mode is left to the
* mode-bearing detectors (map-start, scoreboard header).
*/
import { getCV, type Mat } from "../../cv";
import type { FrameData } from "../../image";
import { STAGES } from "../scoreboard/header-entries";
/** Downscaled signature dimensions (canonical 1920x1080 / 16). */
export const PLANNER_SIG_W = 120;
export const PLANNER_SIG_H = 68;
/** |Laplacian| floor separating render edges from the blur (blur ~0-5). */
const EDGE_MIN = 24;
/** HSV saturation at/above which an edge pixel is team ink, not structure. */
const INK_SATURATION_MIN = 80;
/** Half-range (signature px, ~16x canonical) of the alignment search. */
const MATCH_RANGE = 8;
/** Below this best NCC, or this lead over the next stage, report nothing. */
const MIN_SCORE = 0.5;
const MIN_MARGIN = 0.05;
export interface PlannerStage {
/** "<stageId>-<MODE>", e.g. "6-SZ" */
key: string;
/** sendou stage id (index into STAGES) */
stageId: number;
/** unit-L2-normalized structural signature, row-major PLANNER_SIG_W x _H */
sig: Float32Array;
}
export interface StageMatch {
/** canonical English stage name (STAGES[stageId]) */
stage: string;
stageId: number;
/** best NCC of the winning stage */
score: number;
/** lead over the best-scoring other stage */
margin: number;
}
/**
* Ink-invariant structural signature of a canonical-normalized RGBA frame:
* downscaled, blurred, unit-L2-normalized float mask of the non-ink render
* edges. Shared by the build tool and the runtime matcher so the atlas and
* the live frame are computed identically.
*/
export function plannerSignature(frame: Mat): Float32Array {
const cv = getCV();
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const rgb = new cv.Mat();
cv.cvtColor(frame, rgb, cv.COLOR_RGBA2RGB);
const hsv = new cv.Mat();
cv.cvtColor(rgb, hsv, cv.COLOR_RGB2HSV);
rgb.delete();
const lap = new cv.Mat();
cv.Laplacian(gray, lap, cv.CV_16S, 3);
gray.delete();
const edges = new cv.Mat();
cv.convertScaleAbs(lap, edges);
lap.delete();
const mask = new cv.Mat();
cv.threshold(edges, mask, EDGE_MIN, 255, cv.THRESH_BINARY);
edges.delete();
// drop saturated (ink) edges, keeping only the structural skeleton
const n = mask.rows * mask.cols;
const md = mask.data;
const hd = hsv.data;
for (let i = 0; i < n; i++) {
if (hd[i * 3 + 1]! >= INK_SATURATION_MIN) md[i] = 0;
}
hsv.delete();
const down = new cv.Mat();
cv.resize(mask, down, new cv.Size(PLANNER_SIG_W, PLANNER_SIG_H), 0, 0, cv.INTER_AREA);
mask.delete();
const blur = new cv.Mat();
cv.GaussianBlur(down, blur, new cv.Size(5, 5), 0);
down.delete();
const out = new Float32Array(PLANNER_SIG_W * PLANNER_SIG_H);
const bd = blur.data;
let sumSq = 0;
for (let i = 0; i < out.length; i++) {
out[i] = bd[i]!;
sumSq += out[i]! * out[i]!;
}
blur.delete();
const norm = Math.sqrt(sumSq) || 1;
for (let i = 0; i < out.length; i++) out[i]! /= norm;
return out;
}
/** Dot product of `a` against `b` shifted by (dx, dy) over their overlap. */
function shiftedDot(a: Float32Array, b: Float32Array, dx: number, dy: number): number {
let dot = 0;
for (let y = 0; y < PLANNER_SIG_H; y++) {
const sy = y + dy;
if (sy < 0 || sy >= PLANNER_SIG_H) continue;
const ar = y * PLANNER_SIG_W;
const br = sy * PLANNER_SIG_W;
for (let x = 0; x < PLANNER_SIG_W; x++) {
const sx = x + dx;
if (sx < 0 || sx >= PLANNER_SIG_W) continue;
dot += a[ar + x]! * b[br + sx]!;
}
}
return dot;
}
/** Best NCC of two unit signatures over a small translation search. */
function bestNcc(a: Float32Array, b: Float32Array): number {
let best = -1;
for (let dy = -MATCH_RANGE; dy <= MATCH_RANGE; dy += 2) {
for (let dx = -MATCH_RANGE; dx <= MATCH_RANGE; dx += 2) {
const v = shiftedDot(a, b, dx, dy);
if (v > best) best = v;
}
}
return best;
}
/**
* Identify the stage of a minimap frame's signature against the planner
* atlas. Returns null when no stage matches confidently (score floor) or two
* stages are too close to call (margin floor) — e.g. a stage not in the set.
*/
export function matchStage(
sig: Float32Array,
planners: readonly PlannerStage[],
): StageMatch | null {
if (planners.length === 0) return null;
const byStage = new Map<number, number>();
for (const p of planners) {
const score = bestNcc(sig, p.sig);
const prev = byStage.get(p.stageId);
if (prev === undefined || score > prev) byStage.set(p.stageId, score);
}
let bestId = -1;
let best = -1;
let second = -1;
for (const [id, score] of byStage) {
if (score > best) {
second = best;
best = score;
bestId = id;
} else if (score > second) {
second = score;
}
}
const margin = second < 0 ? best : best - second;
if (best < MIN_SCORE || margin < MIN_MARGIN) return null;
return {
stage: STAGES[bestId] ?? "",
stageId: bestId,
score: Math.round(best * 1000) / 1000,
margin: Math.round(margin * 1000) / 1000,
};
}
export interface PlannerManifest {
width: number;
height: number;
/** tiles packed left-to-right, top-to-bottom, this many per row */
cols: number;
/** tile keys in packing order */
keys: string[];
}
/**
* Slice the packed signature atlas (grayscale uint8 tiles) back into
* unit-normalized PlannerStage signatures. Mirrors loadGlyphSet's atlas
* convention; the build tool writes the atlas + manifest.
*/
export function loadPlannerStages(atlas: FrameData, manifest: PlannerManifest): PlannerStage[] {
const { width, height, cols, keys } = manifest;
const aw = atlas.width;
const data = atlas.data;
return keys.map((key, i) => {
const tx = (i % cols) * width;
const ty = Math.floor(i / cols) * height;
const sig = new Float32Array(width * height);
let sumSq = 0;
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
// atlas is RGBA; the tiles are grayscale, so read the red channel
const v = data[((ty + y) * aw + (tx + x)) * 4]!;
sig[y * width + x] = v;
sumSq += v * v;
}
}
const norm = Math.sqrt(sumSq) || 1;
for (let j = 0; j < sig.length; j++) sig[j]! /= norm;
const stageId = Number(key.split("-")[0]);
return { key, stageId, sig };
});
}

40
app/features/cv/core/detectors/registry.ts Normal file
View File

@@ -0,0 +1,40 @@
/**
* The full detector registry — the single source of truth for "every
* detector that runs on a frame". New event types get added here and are
* picked up by the analyzer worker.
*/
import { createDeathDetector } from "./death/index";
import { createMapStartDetector } from "./map-start/index";
import { createMinimapDetector } from "./minimap/index";
import {
createScoreboardDetector,
SCOREBOARD_EVENT_TYPE,
type ScoreboardResources,
} from "./scoreboard/index";
import { createScoreboardOwnDetector } from "./scoreboard-own/index";
import {
createScoreboardReplayDetector,
SCOREBOARD_REPLAY_EVENT_TYPE,
} from "./scoreboard-replay/index";
import type { Detector } from "./types";
/**
* Event types whose data is the full 8-player scoreboard shape
* (ScoreboardData): the results screen and the replay-browser detail.
*/
export const SCOREBOARD_EVENT_TYPES: readonly string[] = [
SCOREBOARD_EVENT_TYPE,
SCOREBOARD_REPLAY_EVENT_TYPE,
];
export function createAllDetectors(resources: ScoreboardResources): Detector<unknown>[] {
return [
createScoreboardDetector(resources) as Detector<unknown>,
createScoreboardReplayDetector(resources) as Detector<unknown>,
createScoreboardOwnDetector(resources) as Detector<unknown>,
createDeathDetector(resources) as Detector<unknown>,
createMapStartDetector(resources) as Detector<unknown>,
createMinimapDetector(resources) as Detector<unknown>,
];
}

32
app/features/cv/core/detectors/scoreboard-own/abilities.ts Normal file
View File

@@ -0,0 +1,32 @@
/**
* Ability badge templates for the personal-results gear cards. Same
* composite-and-resize pipeline as the death panel (death/abilities.ts),
* at this screen's smaller badge sizes and its gray-strip ink threshold.
*/
import type { FrameData } from "../../image";
import { type AbilityTemplates, buildAbilityRole } from "../death/abilities";
import {
OWN_ABILITY_ART_RATIO,
OWN_ABILITY_INK_THRESHOLD,
OWN_ABILITY_MAIN_SIZES,
OWN_ABILITY_SUB_SIZES,
} from "./rois";
export function prepareOwnAbilityTemplates(
icons: { id: string; image: FrameData }[],
): AbilityTemplates {
return {
mains: buildAbilityRole(
icons,
OWN_ABILITY_MAIN_SIZES,
OWN_ABILITY_ART_RATIO,
OWN_ABILITY_INK_THRESHOLD,
),
subs: buildAbilityRole(
icons,
OWN_ABILITY_SUB_SIZES,
OWN_ABILITY_ART_RATIO,
OWN_ABILITY_INK_THRESHOLD,
),
};
}

246
app/features/cv/core/detectors/scoreboard-own/index.ts Normal file
View File

@@ -0,0 +1,246 @@
/**
* ScoreboardOwnDetector: parses the personal results screen (the "your
* results" view after a match) — lobby/mode/stage from the header tags
* (identical to the live scoreboard header, parsing is shared), the
* player's main weapon from the weapon card's title tag, and the own gear
* abilities from the three gear cards' badge strips.
*
* The weapon arrives as text: the title is OCR'd with the death-weapon
* atlas (the only atlas carrying the weapon-name charset) rescaled to this
* screen's title size, then snapped against every language's main-weapon
* names at once and reported under its canonical English name.
*/
import { getCV, type Mat } from "../../cv";
import { type GlyphSet, recognizeText, scaleGlyphSet } from "../../glyphs";
import { copyRoi, cropRoi, maxBrightness, meanBrightness } from "../../image";
import { closestBy, matchKey } from "../../text";
import { LOCALIZED_WEAPON_NAMES } from "../death/localized-messages";
import { ALL_WEAPON_ENTRIES, type WeaponEntry } from "../death/weapon-names";
import { type ParsedHeader, parseHeader } from "../scoreboard/header";
import type { ScoreboardResources } from "../scoreboard/index";
import { matchWeapon, type WeaponMatch } from "../scoreboard/weapons";
import type { DetectedEvent, Detector, GateResult } from "../types";
import {
GATE_PANEL_MAX_MEAN,
GATE_PANEL_PROBES,
GATE_STRIP_MAX_MEAN,
GATE_STRIP_MIN_MEAN,
GATE_TEXT_MIN_MAX,
GATE_TITLE_TEXT_PROBES,
GEAR_ROWS,
gateStripProbe,
gearMainRoi,
gearSubRoi,
OWN_ABILITY_INK_THRESHOLD,
WEAPON_TITLE_BAND,
WEAPON_TITLE_BIN_THRESHOLD,
WEAPON_TITLE_TEXT_HEIGHT,
} from "./rois";
export interface ScoreboardOwnData {
/** e.g. "Private Battle", from the header tag; null when unreadable */
lobby: string | null;
/** e.g. "Splat Zones" */
mode: string | null;
/** e.g. "Museum d'Alfonsino" */
stage: string | null;
/** the player's main weapon (canonical English name); null if unreadable */
weapon: string | null;
/** the weapon's id in the assets/cv/main-weapons id space */
weaponId: number | null;
/**
* own gear abilities, [head, clothes, shoes] rows of
* [main, sub, sub, sub] ability ids (assets/cv/abilities id space)
*/
abilities: string[][];
}
export const SCOREBOARD_OWN_EVENT_TYPE = "ScoreboardOwn";
/** Snapped weapon reading below this is reported as null (kept in debug). */
const WEAPON_MIN_SCORE = 0.55;
interface WeaponCandidate {
text: string;
entry: WeaponEntry;
}
/**
* Every string the weapon card title can show: all languages' localized
* main-weapon names plus the canonical English names (localized-messages
* omits names identical to English).
*/
let weaponCandidates: WeaponCandidate[] | null = null;
function mainWeaponCandidates(): WeaponCandidate[] {
if (weaponCandidates) return weaponCandidates;
const mains = ALL_WEAPON_ENTRIES.filter((e) => e.type === "MAIN");
const byName = new Map(mains.map((e) => [e.name, e]));
const seen = new Set<string>();
weaponCandidates = [];
const push = (text: string, entry: WeaponEntry | undefined) => {
const k = matchKey(text);
if (!entry || seen.has(k)) return;
seen.add(k);
weaponCandidates!.push({ text, entry });
};
for (const entry of mains) push(entry.name, entry);
for (const names of Object.values(LOCALIZED_WEAPON_NAMES)) {
for (const { text, name } of names) push(text, byName.get(name));
}
return weaponCandidates;
}
export function createScoreboardOwnDetector(
resources: ScoreboardResources,
): Detector<ScoreboardOwnData> {
const cv = getCV();
const titleGlyphs: GlyphSet | null = resources.deathWeaponGlyphs
? scaleGlyphSet(
resources.deathWeaponGlyphs,
WEAPON_TITLE_TEXT_HEIGHT / resources.deathWeaponGlyphs.height,
)
: null;
const abilities = resources.ownAbilities ?? null;
function gate(frame: Mat): GateResult {
let panelOk = 0;
for (const roi of GATE_PANEL_PROBES) {
if (meanBrightness(frame, roi) < GATE_PANEL_MAX_MEAN) panelOk++;
}
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
let textOk = 0;
for (const roi of GATE_TITLE_TEXT_PROBES) {
if (maxBrightness(gray, roi) > GATE_TEXT_MIN_MAX) textOk++;
}
gray.delete();
let stripOk = 0;
for (let row = 0; row < GEAR_ROWS; row++) {
const mean = meanBrightness(frame, gateStripProbe(row));
if (mean >= GATE_STRIP_MIN_MEAN && mean <= GATE_STRIP_MAX_MEAN) stripOk++;
}
const score =
(panelOk / GATE_PANEL_PROBES.length +
textOk / GATE_TITLE_TEXT_PROBES.length +
stripOk / GEAR_ROWS) /
3;
const pass =
panelOk === GATE_PANEL_PROBES.length &&
textOk === GATE_TITLE_TEXT_PROBES.length &&
stripOk === GEAR_ROWS;
return { pass, score };
}
function parse(frame: Mat, t: number): DetectedEvent<ScoreboardOwnData>[] {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const rgb = new cv.Mat();
cv.cvtColor(frame, rgb, cv.COLOR_RGBA2RGB);
const confidences: number[] = [];
// header tags sit at the live scoreboard's positions — shared parser
let header: ParsedHeader | null = null;
if (resources.headerLobbyGlyphs && resources.headerLineGlyphs) {
header = parseHeader(gray, resources.headerLobbyGlyphs, resources.headerLineGlyphs);
confidences.push(header.confidence);
}
// Weapon card title, snapped to the main-weapon closed set. The band is
// recognized whole, NOT via readTagBand: the tag is fixed-width (the
// band lies entirely inside it, there is nothing to trim away), and the
// extent trim actively harms it — long names render horizontally
// condensed, whose dense antialiased columns fail the dark-or-bright
// tag-column test and truncate the read mid-name.
let weapon: string | null = null;
let weaponId: number | null = null;
let weaponScore = 0;
let weaponReading = "";
if (titleGlyphs) {
const band = copyRoi(gray, WEAPON_TITLE_BAND);
weaponReading = recognizeText(band, titleGlyphs, {
binThreshold: WEAPON_TITLE_BIN_THRESHOLD,
spaceGap: 9,
minCharScore: 0.3,
}).text.trim();
band.delete();
const match = weaponReading
? closestBy(weaponReading, mainWeaponCandidates(), (c) => c.text)
: null;
if (match) {
weaponScore = match.score;
if (match.score >= WEAPON_MIN_SCORE) {
weapon = match.entry.entry.name;
weaponId = Number(match.entry.entry.id);
}
}
confidences.push(weaponScore);
}
// gear-card ability strips: [head, clothes, shoes] x [main, sub, sub, sub]
const abilityRows: string[][] = [];
const abilityDebug: (WeaponMatch | null)[][] = [];
if (abilities) {
for (let row = 0; row < GEAR_ROWS; row++) {
const ids: string[] = [];
const debug: (WeaponMatch | null)[] = [];
const mainCrop = cropRoi(rgb, gearMainRoi(row));
const main = matchWeapon(mainCrop, abilities.mains, {
inkThreshold: OWN_ABILITY_INK_THRESHOLD,
});
mainCrop.delete();
ids.push(main.id);
debug.push(main);
confidences.push(Math.max(0, main.score));
for (let slot = 0; slot < 3; slot++) {
const crop = cropRoi(rgb, gearSubRoi(row, slot));
const sub = matchWeapon(crop, abilities.subs, {
inkThreshold: OWN_ABILITY_INK_THRESHOLD,
});
crop.delete();
ids.push(sub.id);
debug.push(sub);
confidences.push(Math.max(0, sub.score));
}
abilityRows.push(ids);
abilityDebug.push(debug);
}
}
gray.delete();
rgb.delete();
const confidence =
confidences.length > 0 ? confidences.reduce((a, b) => a + b, 0) / confidences.length : 0;
return [
{
type: SCOREBOARD_OWN_EVENT_TYPE,
t,
confidence,
data: {
lobby: header?.lobby ?? null,
mode: header?.mode ?? null,
stage: header?.stage ?? null,
weapon,
weaponId,
abilities: abilityRows,
},
debug: {
header: header?.debug,
weaponReading,
weaponScore,
abilityRows: abilityDebug.map((row) =>
row.map((m) => m && { top: m.top, score: m.score }),
),
},
},
];
}
return { id: "scoreboard-own", gate, parse };
}

96
app/features/cv/core/detectors/scoreboard-own/rois.ts Normal file
View File

@@ -0,0 +1,96 @@
/**
* ALL scoreboard-own ROI coordinates, in canonical 1920x1080 space.
* Calibrated against scoreboard-own/private-battle-splat-zones-museum via
* tools/dump-crops.ts and column/row projection measurement.
*
* The personal results screen ("your results" after a match) shows the
* same lobby/mode/stage header tags as the live scoreboard — at the same
* positions, so header parsing reuses the scoreboard bands — over a large
* dark panel (~35) with the player's own banner, the medals list, and a
* bottom row of four cards: the weapon card (yellow-bordered title tag
* with the weapon name, big render on a white square), then one gear card
* per slot [head, clothes, shoes], each with a light-gray ability strip of
* one main badge (⌀~49) and three sub badges (⌀~38).
*/
import type { Roi } from "../../canonical";
/**
* Weapon card title tag interior (black, fixed width, white weapon name
* left-aligned). The band starts inside the tag so readTagBand's extent
* trim anchors on tag columns immediately.
*/
export const WEAPON_TITLE_BAND: Roi = { x: 876, y: 766, w: 200, h: 32 };
/** Tight cap height of the title text at 1080p. */
export const WEAPON_TITLE_TEXT_HEIGHT = 18;
/** White-core title text on the black tag binarizes high, like the burst text. */
export const WEAPON_TITLE_BIN_THRESHOLD = 190;
/** Gear cards [head, clothes, shoes]: main-ability badge center x per row. */
export const GEAR_MAIN_CXS = [1142, 1372, 1602] as const;
/** Sub badge center x offsets from the row's main badge center. */
const GEAR_SUB_DXS = [48, 88, 127] as const;
/** All badges share one vertical center (the ability strip line). */
export const GEAR_BADGE_CY = 927;
export const GEAR_ROWS = 3;
/**
* Search boxes around each badge. Heights double as the size filter:
* matchTemplate silently skips templates taller than the ROI, so the 52px
* sub box excludes the main-size templates within a shared template list.
*/
export function gearMainRoi(row: number): Roi {
const cx = GEAR_MAIN_CXS[row]!;
return { x: cx - 32, y: GEAR_BADGE_CY - 32, w: 64, h: 64 };
}
export function gearSubRoi(row: number, slot: number): Roi {
const cx = GEAR_MAIN_CXS[row]! + GEAR_SUB_DXS[slot]!;
return { x: cx - 26, y: GEAR_BADGE_CY - 26, w: 52, h: 52 };
}
/** Template heights (px at 1080p) per badge role (main ⌀~49, sub ⌀~38). */
export const OWN_ABILITY_MAIN_SIZES = [45, 49, 53] as const;
export const OWN_ABILITY_SUB_SIZES = [34, 38, 42] as const;
/**
* Icon art diameter as a fraction of the badge box: the ratio sweep peaks
* at 1.0 for both roles — the art overflows the circle slightly (arrows
* poke past the ring), so the badge ring contributes nothing.
*/
export const OWN_ABILITY_ART_RATIO = 1.0;
/**
* Ink threshold inside a badge box. Unlike the death panel these badges sit
* on a light-gray strip (~140-155); 170 keeps the strip out of the ink
* count while the saturated icon art still clears it on its max channel.
*/
export const OWN_ABILITY_INK_THRESHOLD = 170;
/**
* Gate probes. The results panel is uniform dark (~35, bottom edge ~54)
* at spots that dodge the banner, medals and cards; each card's title tag
* holds bright white name text starting at its left edge; the ability
* strip's gray shows in the constant gap after the third sub badge.
*/
export const GATE_PANEL_PROBES: readonly Roi[] = [
{ x: 860, y: 245, w: 30, h: 20 },
{ x: 1690, y: 395, w: 30, h: 20 },
{ x: 875, y: 695, w: 30, h: 20 },
{ x: 1400, y: 985, w: 30, h: 20 },
];
export const GATE_PANEL_MAX_MEAN = 65;
/** Left edge of each card's title text (weapon card first). */
export const GATE_TITLE_TEXT_PROBES: readonly Roi[] = [
{ x: 880, y: 768, w: 70, h: 26 },
{ x: 1090, y: 768, w: 70, h: 26 },
{ x: 1319, y: 768, w: 70, h: 26 },
{ x: 1550, y: 768, w: 70, h: 26 },
];
/** Title text regions must contain bright (white) pixels. */
export const GATE_TEXT_MIN_MAX = 180;
/** Gray ability-strip gap after the third sub badge, one per gear card. */
export function gateStripProbe(row: number): Roi {
return { x: GEAR_MAIN_CXS[row]! + 148, y: 917, w: 10, h: 18 };
}
export const GATE_STRIP_MIN_MEAN = 110;
export const GATE_STRIP_MAX_MEAN = 200;

122
app/features/cv/core/detectors/scoreboard-replay/code.ts Normal file
View File

@@ -0,0 +1,122 @@
/**
* Replay code recognition ("R6KE-DO64-3CXD-XVKL"): bright green text under
* the team panels. Green on the dark background lands below the default
* grayscale binarization threshold, so recognition runs on the green
* channel, where the glyphs are near-white.
*/
import { getCV, type Mat } from "../../cv";
import { type GlyphSet, type RecognizedText, recognizeText } from "../../glyphs";
import { cropRoi } from "../../image";
import { REPLAY_CODE_ROI } from "./rois";
/**
* FOT-RowdyStd's 'Q' is a '0' bowl with a small tail below the baseline;
* the bowl dominates template correlation, so a real Q ranks as '0' by a
* hair. Like the P/p rule in scoreboard/names.ts, the segment geometry
* decides what the templates cannot: a 0/O read whose ink reaches well
* below the line's baseline (the median ink bottom of the other
* alphanumerics) is a Q.
*/
const Q_TWINS = new Set(["0", "O"]);
const Q_DESCENT_MIN_PX = 4;
function resolveQsByDescent(raw: RecognizedText): string {
const anchors = raw.chars
.filter((c) => !Q_TWINS.has(c.char) && c.char !== "-")
.map((c) => c.y1)
.sort((a, b) => a - b);
if (anchors.length === 0) return raw.text;
const baseline = anchors[Math.floor(anchors.length / 2)]!;
return raw.chars
.map((c) => (Q_TWINS.has(c.char) && c.y1 - baseline >= Q_DESCENT_MIN_PX ? "Q" : c.char))
.join("");
}
/**
* On blurry captures a 'U' smears and the narrow 'L' template correlates on
* its left stroke + bottom curve, edging out the true 'U' by a hair. As with
* the Q rule, segment geometry decides what the templates cannot: a U's
* right stroke fills the segment's top-right quadrant, where an L has no ink
* at all (measured 0.00-0.15 leak on true Ls vs 0.27+ on Us across the
* fixtures). The ink probe is the discriminator; the margin only skips
* confident reads, and no true L on the fixtures carries a 'U' candidate
* at all, so it stays generous (low-res-2's misread U trails by 0.086).
*/
const LU_SCORE_MARGIN = 0.12;
const LU_INK_THRESHOLD = 150;
const LU_TOP_RIGHT_MIN_FRACTION = 0.2;
function resolveUsByTopRightInk(raw: RecognizedText, green: Mat): RecognizedText {
const { cols, data } = green;
const chars = raw.chars.map((c) => {
if (c.char !== "L") return c;
const u = c.candidates?.find((k) => k.char === "U");
if (!u || c.score - u.score > LU_SCORE_MARGIN) return c;
const xMid = Math.ceil((c.x0 + c.x1) / 2);
const yMid = Math.floor((c.y0 + c.y1) / 2);
let ink = 0;
let total = 0;
for (let y = c.y0; y < yMid; y++) {
for (let x = xMid; x < c.x1; x++) {
total++;
if (data[y * cols + x]! > LU_INK_THRESHOLD) ink++;
}
}
return total > 0 && ink / total >= LU_TOP_RIGHT_MIN_FRACTION ? { ...c, char: "U" } : c;
});
return { ...raw, chars, text: chars.map((c) => c.char).join("") };
}
export interface ParsedReplayCode {
/** normalized "XXXX-XXXX-XXXX-XXXX", or null when the shape is wrong */
code: string | null;
/** min glyph score across recognized characters */
confidence: number;
raw: RecognizedText;
}
const CODE_RE = /^[0-9A-Z]{4}(-[0-9A-Z]{4}){3}$/;
/**
* Restrict a glyph set to the characters codes can contain — a shallow
* view over the same template mats, so dispose only the source set.
*/
export function codeCharsetOf(set: GlyphSet): GlyphSet {
const glyphs = set.glyphs.filter((g) => /^[0-9A-Z-]$/.test(g.char));
const widths = glyphs.map((g) => g.mat.cols).sort((a, b) => a - b);
return {
glyphs,
height: set.height,
medianWidth: widths[Math.floor(widths.length / 2)] ?? set.medianWidth,
};
}
/** rgb: full normalized frame in RGB (not RGBA). */
export function parseReplayCode(rgb: Mat, glyphs: GlyphSet): ParsedReplayCode {
const cv = getCV();
const view = cropRoi(rgb, REPLAY_CODE_ROI);
const channels = new cv.MatVector();
cv.split(view, channels);
const g = channels.get(1);
const green = new cv.Mat();
g.copyTo(green);
g.delete();
channels.delete();
view.delete();
const raw = recognizeText(green, glyphs, { spaceGap: Infinity, minCharScore: 0.3 });
const resolved = resolveUsByTopRightInk(raw, green);
green.delete();
let text = resolveQsByDescent(resolved).toUpperCase();
// Dashes are thin and can drop out of segmentation; a clean 16-char
// alphanumeric read is unambiguous, so re-insert them.
if (/^[0-9A-Z]{16}$/.test(text)) {
text = text.replace(/(.{4})(?=.)/g, "$1-");
}
return {
code: CODE_RE.test(text) ? text : null,
confidence: raw.confidence,
raw,
};
}

132
app/features/cv/core/detectors/scoreboard-replay/header.ts Normal file
View File

@@ -0,0 +1,132 @@
/**
* Replay-browser header parsing. Same black auto-sized tag style as the
* live scoreboard header, but different content: the top line holds the
* recording timestamp ("3/7/2026 22:28") followed by the stage, the bottom
* line the lobby (bold) followed by the mode.
*
* The timestamp is locale-formatted and open-ended, so it is validated by
* shape and kept as a raw string; stage and lobby+mode snap to the closed
* sets shared with the live header.
*/
import type { Mat } from "../../cv";
import type { GlyphSet } from "../../glyphs";
import { ALL_STAGE_ENTRIES, LOBBY_MODE_COMBOS } from "../../localized";
import { closestBy } from "../../text";
import { readTagBand } from "../scoreboard/header";
import { HEADER_BOTTOM_BAND, HEADER_TOP_BAND } from "./rois";
export interface ParsedReplayHeader {
timestamp: string | null;
stage: string | null;
lobby: string | null;
mode: string | null;
/** min of the closed-set match scores that were attempted */
confidence: number;
debug: {
topReading: string;
bottomReading: string;
stageScore: number;
bottomScore: number;
};
}
const MIN_MATCH_SCORE = 0.62;
/**
* Lifted-blacks captures (720p streams upscaled and re-encoded) raise the
* tag background to ~80-115 gray, above readTagBand's default dark ceiling,
* so the tag-extent trim truncates the band to a sliver and the read comes
* back empty. A band whose closed-set snap fails is re-read with this
* ceiling; the retry is adopted only when it snaps at least as well.
*/
const TAG_DARK_MAX_LIFTED = 120;
/**
* "3/7/2026 22:28" and friends; capture the rest of the line (the stage).
* The console formats the date per locale — "7.3.2026" (de), "2026/3/7"
* (ja) — so any . / - separated triple followed by a time is accepted.
* Adjacent skinny time digits can read with a spurious gap on compressed
* captures ("14:1 1"), so a lone space is tolerated between them and
* stripped when the timestamp is assembled.
*/
const TIMESTAMP_RE = /^(\d{1,4}[./-]\d{1,2}[./-]\d{1,4})\s+(\d(?: ?\d)?: ?\d ?\d)\s*(.*)$/;
interface TopBandParse {
reading: string;
timestamp: string | null;
stage: string | null;
stageScore: number;
}
function parseTopBand(reading: string): TopBandParse {
let timestamp: string | null = null;
let stage: string | null = null;
let stageScore = 0;
// The top band reads with the BlitzMain name glyphs, where 1/I/l/| are
// identical bars ("I9:04") and O rides a hair above 0 ("2O26"); in the
// digits-only timestamp every bar is a '1' and every O a '0'. Match on
// the normalized line, keep the stage part's original reading (the
// replacements are 1:1, so offsets line up).
const normalized = reading.replace(/[Il|]/g, "1").replace(/O/g, "0");
const m = TIMESTAMP_RE.exec(normalized);
const stageReading = m ? reading.slice(reading.length - m[3]!.length) : reading;
if (m) timestamp = `${m[1]!} ${m[2]!.replace(/ /g, "")}`;
if (stageReading) {
const match = closestBy(stageReading, ALL_STAGE_ENTRIES, (e) => e.text);
if (match) {
stageScore = match.score;
if (match.score >= MIN_MATCH_SCORE) stage = match.entry.canonical;
}
}
return { reading, timestamp, stage, stageScore };
}
export function parseReplayHeader(
gray: Mat,
topGlyphs: GlyphSet,
bottomGlyphs: GlyphSet,
): ParsedReplayHeader {
let top = parseTopBand(readTagBand(gray, HEADER_TOP_BAND, topGlyphs));
if (top.stage === null) {
const retry = parseTopBand(
readTagBand(gray, HEADER_TOP_BAND, topGlyphs, { tagDarkMax: TAG_DARK_MAX_LIFTED }),
);
if (retry.stageScore >= top.stageScore) top = retry;
}
let bottomReading = readTagBand(gray, HEADER_BOTTOM_BAND, bottomGlyphs);
let bottomMatch = bottomReading
? closestBy(bottomReading, LOBBY_MODE_COMBOS, (c) => c.text)
: null;
if (!bottomMatch || bottomMatch.score < MIN_MATCH_SCORE) {
const reading = readTagBand(gray, HEADER_BOTTOM_BAND, bottomGlyphs, {
tagDarkMax: TAG_DARK_MAX_LIFTED,
});
const match = reading ? closestBy(reading, LOBBY_MODE_COMBOS, (c) => c.text) : null;
if ((match?.score ?? 0) >= (bottomMatch?.score ?? 0)) {
bottomReading = reading;
bottomMatch = match;
}
}
let lobby: string | null = null;
let mode: string | null = null;
if (bottomMatch && bottomMatch.score >= MIN_MATCH_SCORE) {
lobby = bottomMatch.entry.lobby;
mode = bottomMatch.entry.mode;
}
return {
timestamp: top.timestamp,
stage: top.stage,
lobby,
mode,
confidence: Math.min(top.stageScore, bottomMatch?.score ?? 0),
debug: {
topReading: top.reading,
bottomReading,
stageScore: top.stageScore,
bottomScore: bottomMatch?.score ?? 0,
},
};
}

385
app/features/cv/core/detectors/scoreboard-replay/index.ts Normal file
View File

@@ -0,0 +1,385 @@
/**
* ScoreboardReplayDetector: parses the replay-browser detail screen — the
* same match data as the live scoreboard (header, team scores, 8 player
* rows) plus the recording timestamp and the replay code.
*
* Layout differs from the live scoreboard: the two team panels sit side by
* side and the replay owner's team may be on either side, so the
* VICTORY/DEFEAT panel tags are read to keep `players`/`scores` ordered
* winners-first like the live event. Field parsing reuses the scoreboard
* helpers with glyph sets rescaled to this screen's text sizes.
*/
import { getCV, type Mat } from "../../cv";
import { type GlyphSet, recognizeText, scaleGlyphSet } from "../../glyphs";
import { cropRoi, maxBrightness, maxChannel, meanBrightness, type Roi } from "../../image";
import { RESULT_TAG_ENTRIES } from "../../localized";
import { closestBy } from "../../text";
import { type ParsedNumber, parseNumber } from "../scoreboard/digits";
import type {
ScoreboardData,
ScoreboardPlayer,
ScoreboardResources,
ScoreboardRowDebug,
} from "../scoreboard/index";
import { findPovIndex } from "../scoreboard/pov";
import { parseScoreboardRow, type RowRois } from "../scoreboard/row";
import type { DetectedEvent, Detector, GateResult } from "../types";
import { codeCharsetOf, type ParsedReplayCode, parseReplayCode } from "./code";
import { type ParsedReplayHeader, parseReplayHeader } from "./header";
import {
CODE_TEXT_HEIGHT,
GATE_CODE_BLUE_MAX,
GATE_CODE_GREEN_MIN,
GATE_CODE_MIN_FRACTION,
GATE_FLAT_MAX_MEAN,
GATE_FLAT_MIN_MEAN,
GATE_GAP_MAX_MEAN,
GATE_GAP_PROBES,
GATE_TEXT_MIN_MAX,
gateFlatProbe,
HEADER_LINE_HEIGHT,
HEADER_TIMESTAMP_HEIGHT,
MATCH_SCORE_DIGIT_HEIGHT,
MATCH_SCORE_ROIS,
NAME_TEXT_HEIGHT,
nameRoi,
PAINT_DIGIT_HEIGHT,
PANEL_XS,
paintRoi,
paintSuffixRoi,
povArrowRoi,
REPLAY_CODE_ROI,
RESULT_TAG_TEXT_HEIGHT,
ROW_CENTERS,
resultTagRoi,
STAT_DIGIT_HEIGHT,
specialIconRoi,
statRoi,
TEAM_DIGIT_HEIGHT,
teamScoreRoi,
weaponRoi,
} from "./rois";
export interface ScoreboardReplayData extends ScoreboardData {
/** recording timestamp as shown, e.g. "3/7/2026 22:28"; locale-formatted */
timestamp: string | null;
/** "XXXX-XXXX-XXXX-XXXX" */
replayCode: string | null;
/**
* the colored "Score:" banner values, [winner, loser] like `scores`;
* a knockout's winner reports 100 (the burst hides the real banner)
*/
matchScores: [number | null, number | null];
}
export const SCOREBOARD_REPLAY_EVENT_TYPE = "ScoreboardReplay";
/** Replay pills are mid-gray (~61), not near-black; see matchWeapon docs. */
const REPLAY_INK_THRESHOLD = 90;
/**
* White banner digits on saturated team color: applies to the "Score:"
* banners AND the team totals — a green DEFEAT panel weighs in at ~184
* on the green-heavy grayscale, above the default 150.
*/
const BANNER_BIN_THRESHOLD = 190;
/**
* On a knockout the winner's "Score:" banner is replaced by the localized
* "KNOCKOUT!" burst, whose letters overlap the score ROI and weakly match
* digit templates (an O reads as a ~0.42 zero; real digits score 0.9+).
* Reads below this floor are discarded rather than trusted as a score — the
* knockout that put the burst there is recovered from the team count below.
*/
const MATCH_SCORE_MIN_CONF = 0.6;
/** The count a knockout wins at — the burst hides it, so it is never read. */
const KO_MATCH_SCORE = 100;
/**
* The team box prints the count times five ("440 p" alongside a 88 banner),
* so a knockout's full 100 count shows as 500 — a total only a knockout
* reaches, which is what separates a burst-covered banner from an unread one.
*/
const FULL_COUNT_TEAM_SCORE = KO_MATCH_SCORE * 5;
/** Canonical results the localized VICTORY/DEFEAT panel tags snap to. */
type PanelResult = "VICTORY" | "DEFEAT";
const RESULT_MIN_SCORE = 0.6;
/**
* The chunky outlined tag letters bridge at the default 150 on the
* max-channel image; 190 keeps the cores separated (and drops the gray
* gear/signal icons trailing the text).
*/
const RESULT_TAG_BIN_THRESHOLD = 190;
interface PanelParse {
players: ScoreboardPlayer[];
rows: ScoreboardRowDebug[];
teamScore: ParsedNumber | null;
matchScore: ParsedNumber | null;
result: PanelResult | null;
resultReading: string;
resultScore: number;
confidences: number[];
}
/** Fraction of ROI pixels matching the replay code's green (RGBA frame). */
function greenFraction(frame: Mat, roi: Roi): number {
const cv = getCV();
const view = cropRoi(frame, roi);
const cont = new cv.Mat();
view.copyTo(cont);
view.delete();
const d = cont.data;
const n = cont.rows * cont.cols;
let green = 0;
for (let i = 0; i < n; i++) {
if (d[i * 4 + 1]! > GATE_CODE_GREEN_MIN && d[i * 4 + 2]! < GATE_CODE_BLUE_MAX) green++;
}
cont.delete();
return n > 0 ? green / n : 0;
}
export function createScoreboardReplayDetector(
resources: ScoreboardResources,
): Detector<ScoreboardReplayData> {
const cv = getCV();
const scaled = (set: GlyphSet | null, height: number): GlyphSet | null =>
set ? scaleGlyphSet(set, height / set.height) : null;
const nameGlyphs = scaled(resources.nameGlyphs, NAME_TEXT_HEIGHT);
const paintDigits = scaled(resources.paintDigits, PAINT_DIGIT_HEIGHT);
const statDigits = scaled(resources.statDigits, STAT_DIGIT_HEIGHT);
const teamBase = resources.teamDigits ?? resources.paintDigits;
const teamDigits = scaled(teamBase, TEAM_DIGIT_HEIGHT);
const matchScoreDigits = scaled(teamBase, MATCH_SCORE_DIGIT_HEIGHT);
/** Timestamp needs digits + '/' + ':' — only the names atlas has them. */
const headerTopGlyphs = scaled(resources.nameGlyphs, HEADER_TIMESTAMP_HEIGHT);
const headerBottomGlyphs = scaled(resources.headerLineGlyphs, HEADER_LINE_HEIGHT);
// Code and result tags render in FOT-RowdyStd — use the dedicated atlases
// when present; the BlitzMain-based fallbacks read them only roughly.
const resultGlyphs =
scaled(resources.replayResultGlyphs ?? null, RESULT_TAG_TEXT_HEIGHT) ??
scaled(resources.headerLineGlyphs, RESULT_TAG_TEXT_HEIGHT);
const codeGlyphs = resources.replayCodeGlyphs
? scaled(resources.replayCodeGlyphs, CODE_TEXT_HEIGHT)
: resources.nameGlyphs
? scaleGlyphSet(
codeCharsetOf(resources.nameGlyphs),
CODE_TEXT_HEIGHT / resources.nameGlyphs.height,
)
: null;
function gate(frame: Mat): GateResult {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
let flatOk = 0;
let suffixOk = 0;
for (const dx of PANEL_XS) {
for (const cy of ROW_CENTERS) {
const flat = meanBrightness(frame, gateFlatProbe(cy, dx));
if (flat >= GATE_FLAT_MIN_MEAN && flat <= GATE_FLAT_MAX_MEAN) flatOk++;
if (maxBrightness(gray, paintSuffixRoi(cy, dx)) > GATE_TEXT_MIN_MAX) suffixOk++;
}
}
let gapOk = 0;
for (const roi of GATE_GAP_PROBES) {
if (meanBrightness(frame, roi) < GATE_GAP_MAX_MEAN) gapOk++;
}
const codeFraction = greenFraction(frame, REPLAY_CODE_ROI);
gray.delete();
const rowCount = PANEL_XS.length * ROW_CENTERS.length;
const score =
(flatOk / rowCount +
suffixOk / rowCount +
gapOk / GATE_GAP_PROBES.length +
Math.min(1, codeFraction / (2 * GATE_CODE_MIN_FRACTION))) /
4;
const pass =
flatOk >= 7 && suffixOk >= 7 && gapOk === 2 && codeFraction >= GATE_CODE_MIN_FRACTION;
return { pass, score };
}
function parsePanel(gray: Mat, rgb: Mat, dx: number): PanelParse {
const players: ScoreboardPlayer[] = [];
const rows: ScoreboardRowDebug[] = [];
const confidences: number[] = [];
const rowRois: RowRois = {
weapon: (cy) => weaponRoi(cy, dx),
specialIcon: (cy) => specialIconRoi(cy, dx),
paint: (cy) => paintRoi(cy, dx),
name: (cy) => nameRoi(cy, dx),
stat: (cy, i) => statRoi(cy, dx, i),
povArrow: (cy) => povArrowRoi(cy, dx),
};
const rowResources = {
weapons: resources.weapons,
specials: resources.specials,
paintDigits,
statDigits,
nameGlyphs,
};
for (const cy of ROW_CENTERS) {
// A short team (e.g. a 7-player private battle) renders no pill for
// the unused bottom row — just near-black panel background where the
// flat probe expects the mid-gray pill (the gate's flatOk >= 7 already
// tolerates the missing row). Skip it: no phantom player.
const flat = meanBrightness(rgb, gateFlatProbe(cy, dx));
if (flat < GATE_FLAT_MIN_MEAN || flat > GATE_FLAT_MAX_MEAN) continue;
// replay rows render smaller (icons ~26px, inside the live template
// set's slide range) on a lighter panel; the paint number is
// left-aligned so the "p" suffix lands inside the ROI on short paints
const row = parseScoreboardRow(gray, rgb, cy, rowRois, rowResources, confidences, {
weaponInkThreshold: REPLAY_INK_THRESHOLD,
paintDropLoweredTrailing: true,
});
players.push(row.player);
rows.push(row.debug);
}
let teamScore: ParsedNumber | null = null;
if (teamDigits) {
const crop = cropRoi(gray, teamScoreRoi(dx));
teamScore = parseNumber(crop, teamDigits, { binThreshold: BANNER_BIN_THRESHOLD });
crop.delete();
confidences.push(teamScore.confidence);
}
let matchScore: ParsedNumber | null = null;
if (matchScoreDigits) {
const crop = cropRoi(gray, MATCH_SCORE_ROIS[dx === 0 ? 0 : 1]!);
matchScore = parseNumber(crop, matchScoreDigits, {
binThreshold: BANNER_BIN_THRESHOLD,
});
if (matchScore.confidence < MATCH_SCORE_MIN_CONF) {
matchScore = { ...matchScore, value: null };
}
crop.delete();
confidences.push(matchScore.confidence);
// No number under the floor + a full team count = the KNOCKOUT! burst
// sitting where the banner's score would be. Report the count it won at
// rather than a hole; an unreadable banner on a lesser total stays null.
if (matchScore.value === null && teamScore?.value === FULL_COUNT_TEAM_SCORE) {
matchScore = { ...matchScore, value: KO_MATCH_SCORE };
}
}
let result: PanelParse["result"] = null;
let resultReading = "";
let resultScore = 0;
if (resultGlyphs) {
const bright = maxChannel(rgb, resultTagRoi(dx));
const raw = recognizeText(bright, resultGlyphs, {
binThreshold: RESULT_TAG_BIN_THRESHOLD,
spaceGap: Infinity,
minCharScore: 0.25,
});
bright.delete();
resultReading = raw.text;
if (resultReading) {
const match = closestBy(resultReading, RESULT_TAG_ENTRIES, (e) => e.text);
if (match) {
resultScore = match.score;
if (match.score >= RESULT_MIN_SCORE) result = match.entry.canonical as PanelResult;
}
}
}
return {
players,
rows,
teamScore,
matchScore,
result,
resultReading,
resultScore,
confidences,
};
}
function parse(frame: Mat, t: number): DetectedEvent<ScoreboardReplayData>[] {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const rgb = new cv.Mat();
cv.cvtColor(frame, rgb, cv.COLOR_RGBA2RGB);
const [left, right] = PANEL_XS.map((dx) => parsePanel(gray, rgb, dx)) as [
PanelParse,
PanelParse,
];
// Winners first. Trust a confident VICTORY/DEFEAT tag read; when the
// tags are inconclusive, the higher "Score:" banner marks the winner
// (the shown match score decides the game). Default to left otherwise.
let swapped = false;
if (left.result !== null || right.result !== null) {
swapped = left.result === "DEFEAT" || right.result === "VICTORY";
} else if (left.matchScore?.value != null && right.matchScore?.value != null) {
swapped = right.matchScore.value > left.matchScore.value;
}
const [winner, loser] = swapped ? [right, left] : [left, right];
// POV arrow row, indexed into the winners-first players ordering
const povIndex = findPovIndex([...winner.rows, ...loser.rows].map((r) => r.povFraction));
let header: ParsedReplayHeader | null = null;
if (headerTopGlyphs && headerBottomGlyphs) {
header = parseReplayHeader(gray, headerTopGlyphs, headerBottomGlyphs);
}
let code: ParsedReplayCode | null = null;
if (codeGlyphs) {
code = parseReplayCode(rgb, codeGlyphs);
}
gray.delete();
rgb.delete();
const confidences = [
...winner.confidences,
...loser.confidences,
...(header ? [header.confidence] : []),
...(code ? [code.confidence] : []),
];
const confidence =
confidences.length > 0 ? confidences.reduce((a, b) => a + b, 0) / confidences.length : 0;
return [
{
type: SCOREBOARD_REPLAY_EVENT_TYPE,
t,
confidence,
data: {
lobby: header?.lobby ?? null,
mode: header?.mode ?? null,
stage: header?.stage ?? null,
timestamp: header?.timestamp ?? null,
replayCode: code?.code ?? null,
scores: [winner.teamScore?.value ?? null, loser.teamScore?.value ?? null],
matchScores: [winner.matchScore?.value ?? null, loser.matchScore?.value ?? null],
players: [...winner.players, ...loser.players],
povIndex,
},
debug: {
rows: [...winner.rows, ...loser.rows],
teamScoreConf: [winner.teamScore?.confidence ?? 0, loser.teamScore?.confidence ?? 0],
matchScoreConf: [winner.matchScore?.confidence ?? 0, loser.matchScore?.confidence ?? 0],
header: header?.debug,
codeRaw: code?.raw.text,
winnerSide: swapped ? "right" : "left",
resultTags: {
left: { reading: left.resultReading, score: left.resultScore, result: left.result },
right: { reading: right.resultReading, score: right.resultScore, result: right.result },
},
},
},
];
}
return { id: "scoreboard-replay", gate, parse };
}

172
app/features/cv/core/detectors/scoreboard-replay/rois.ts Normal file
View File

@@ -0,0 +1,172 @@
/**
* ALL scoreboard-replay ROI coordinates, in canonical 1920x1080 space.
* Calibrated against scoreboard-replay/private-battle-splat-zones-hagglefish
* via tools/overlay-rois.ts and column-projection measurement.
*
* The replay-browser detail screen shows the two team panels SIDE BY SIDE
* (left panel first, right panel = left shifted by PANEL_DX), four gray
* pill rows each. Unlike the live scoreboard's near-black pills (~12),
* replay pills are mid-gray (~61) with darker gaps (~23) between them.
* Below the panels sit the replay owner line and the bright green replay
* code; above them, the two "Score:" banners and the stage-photo header
* with black auto-sized tags (line 1 = timestamp + stage, line 2 = lobby
* + mode).
*/
import type { Roi } from "../../canonical";
/** Vertical centers of the 4 player rows within each panel. */
export const ROW_CENTERS = [573, 654, 735, 816] as const;
/** Horizontal shift from a left-panel ROI to its right-panel twin. */
export const PANEL_DX = 676;
/** dx per panel: [left (index 0), right (index 1)]. */
export const PANEL_XS = [0, PANEL_DX] as const;
/**
* Weapon icon search region within a row — the full pill height, so the
* largest weapon template (64; replay icons render at ~60-64px) fits with
* vertical slide room. matchTemplate silently skips taller templates.
*/
export function weaponRoi(cy: number, dx: number): Roi {
return { x: 522 + dx, y: cy - 34, w: 96, h: 68 };
}
/**
* The player's special-weapon icon, drawn on a black disc above the third
* stat counter (measured art ~25x29 at x 1109-1134, y cy-29..cy across
* fixtures). Only read to break near-tied weapon-icon matches whose kits
* carry different specials. The box stays inside the disc: the mid-gray
* pill around it sits above matchSpecial's ink threshold.
*/
export function specialIconRoi(cy: number, dx: number): Roi {
return { x: 1104 + dx, y: cy - 32, w: 38, h: 33 };
}
/**
* Player name text region (white text, left-aligned; descenders reach
* cy+18). Long names run into the paint column; parse paint first and trim
* at the leftmost paint digit. Wide weapon icons can bleed a column or two
* into the left edge — kept narrower than any observed first glyph.
*/
export function nameRoi(cy: number, dx: number): Roi {
return { x: 620 + dx, y: cy - 16, w: 226, h: 37 };
}
/**
* Paint amount digits, LEFT-aligned starting at x~843 (~18px pitch); the
* trailing "p" moves with the digit count, so 3-digit paints (short
* knockout games) put it inside this region — parseNumber's digit-only
* charset drops it on score.
*/
export function paintRoi(cy: number, dx: number): Roi {
return { x: 821 + dx, y: cy - 15, w: 88, h: 34 };
}
/**
* Gate anchor over the "p" after the paint number. Because the number is
* left-aligned, the "p" position tracks the digit count: measured x
* 898-907 after 3 digits, 915-924 after 4 — the probe spans both.
*/
export function paintSuffixRoi(cy: number, dx: number): Roi {
return { x: 896 + dx, y: cy - 13, w: 31, h: 26 };
}
/**
* Stat counter digits (two, zero-padded; the small "x" prefix is excluded).
* Digits sit at cy+4..cy+23; the stat icons directly above bleed ink into
* anything higher, so the top edge must stay below them.
*/
export function statRoi(cy: number, dx: number, index: 0 | 1 | 2): Roi {
const x = [1000, 1057, 1114][index]!;
return { x: x + dx, y: cy + 3, w: 36, h: 24 };
}
/**
* POV arrow probe: the smaller replay-browser arrow sits on the pill's left
* edge (measured x 487-530, y cy-28..cy+17 on the fixture). Right edge stays
* short of the weapon-icon region (x 522+) core so icon yellows can't leak in.
*/
export function povArrowRoi(cy: number, dx: number): Roi {
return { x: 480 + dx, y: cy - 32, w: 54, h: 56 };
}
/** Team totals ("440p") on the VICTORY/DEFEAT banner, digits ending x~1119. */
export function teamScoreRoi(dx: number): Roi {
return { x: 1040 + dx, y: 481, w: 86, h: 36 };
}
/**
* VICTORY / DEFEAT tag on each panel banner — read to decide which panel
* won (the replay owner's team may sit on either side).
*/
export function resultTagRoi(dx: number): Roi {
return { x: 540 + dx, y: 460, w: 220, h: 50 };
}
/** The colored "Score: NN" banners; digits after the constant label. */
export const MATCH_SCORE_ROIS: readonly [Roi, Roi] = [
{ x: 742, y: 340, w: 130, h: 56 },
{ x: 1620, y: 340, w: 130, h: 56 },
];
/**
* Header bands on the stage-photo banner. Line 1 holds the timestamp and
* the stage tag; line 2 the lobby and mode. Tags are black boxes sized to
* their text; the replay header parser trims each band to the tag extent.
*/
export const HEADER_TOP_BAND: Roi = { x: 500, y: 68, w: 560, h: 46 };
/**
* Wide lobby tags push the mode tag right — "Anarchy Battle (Open)" +
* "Rainmaker" ends at x~1134 — so the band runs well past the longest
* observed pair; readTagBand trims to the actual tag extent.
*/
export const HEADER_BOTTOM_BAND: Roi = { x: 500, y: 124, w: 700, h: 58 };
/**
* Bright green replay code line ("XXXX-XXXX-XXXX-XXXX"). Left-aligned after
* the magnifier icon; the width tracks the glyphs, so a wide-letter code
* (W/M/G-heavy) runs past x=913 — the box extends into the black background
* to fit the widest possible code.
*/
export const REPLAY_CODE_ROI: Roi = { x: 574, y: 960, w: 400, h: 38 };
/**
* Gate probe: flat pill background strip between the paint "p" suffix and
* the first stat "x" — mid-gray on this screen, not near-black.
*/
export function gateFlatProbe(cy: number, dx: number): Roi {
return { x: 930 + dx, y: cy - 10, w: 42, h: 20 };
}
/** Dark gap between row 1 and row 2 pills, one strip per panel. */
export const GATE_GAP_PROBES: readonly Roi[] = [
{ x: 560, y: 611, w: 540, h: 5 },
{ x: 560 + PANEL_DX, y: 611, w: 540, h: 5 },
];
/** Flat pill strips must sit in this mid-gray band. */
export const GATE_FLAT_MIN_MEAN = 45;
export const GATE_FLAT_MAX_MEAN = 78;
/** The inter-pill gap is darker than the pills. */
export const GATE_GAP_MAX_MEAN = 40;
/** The paint "p" suffix region must contain bright pixels. */
export const GATE_TEXT_MIN_MAX = 180;
/**
* Replay-code color probe: fraction of REPLAY_CODE_ROI pixels that are
* green-ish (high G, low B) — unique to this screen.
*/
export const GATE_CODE_GREEN_MIN = 140;
export const GATE_CODE_BLUE_MAX = 90;
export const GATE_CODE_MIN_FRACTION = 0.03;
/** Text metrics measured on the fixture, used for glyph scaling / tooling. */
export const NAME_TEXT_HEIGHT = 24;
export const PAINT_DIGIT_HEIGHT = 26;
export const STAT_DIGIT_HEIGHT = 20;
export const TEAM_DIGIT_HEIGHT = 27;
export const MATCH_SCORE_DIGIT_HEIGHT = 41;
export const HEADER_TIMESTAMP_HEIGHT = 24;
export const HEADER_LINE_HEIGHT = 29;
export const RESULT_TAG_TEXT_HEIGHT = 27;
export const CODE_TEXT_HEIGHT = 25;

51
app/features/cv/core/detectors/scoreboard/digits.ts Normal file
View File

@@ -0,0 +1,51 @@
/**
* Number field parsing on top of glyph recognition.
*/
import type { Mat } from "../../cv";
import { type GlyphSet, type RecognizedText, recognizeText } from "../../glyphs";
export interface ParsedNumber {
value: number | null;
/** min glyph score across recognized digits */
confidence: number;
/** x of the leftmost digit, relative to the crop (null when nothing found) */
leftX: number | null;
raw: RecognizedText;
}
/**
* Digits share a cap line; a lowercase suffix ("p") starts at x-height,
* ~7px lower at the sizes we parse. A trailing char whose ink top sits at
* least this far below the other chars' top line is the suffix, not a digit.
*/
const LOWERED_TRAILING_MIN_PX = 5;
export function parseNumber(
gray: Mat,
digits: GlyphSet,
options: { binThreshold?: number; dropLoweredTrailing?: boolean } = {},
): ParsedNumber {
const raw = recognizeText(gray, digits, {
spaceGap: Infinity,
minCharScore: 0.3,
binThreshold: options.binThreshold,
});
// The replay paint column is left-aligned, so its "p" suffix moves with
// the digit count and can land inside the ROI, where the digit-only
// charset misreads it (a "6"). The geometry still tells it apart.
let chars = raw.chars;
if (options.dropLoweredTrailing && chars.length > 1) {
const capY0 = Math.min(...chars.slice(0, -1).map((c) => c.y0));
if (chars[chars.length - 1]!.y0 - capY0 >= LOWERED_TRAILING_MIN_PX) {
chars = chars.slice(0, -1);
}
}
const text = chars.map((c) => c.char).join("");
const isNumeric = /^[0-9]+$/.test(text);
return {
value: isNumeric ? Number.parseInt(text, 10) : null,
confidence: chars.length > 0 ? Math.min(...chars.map((c) => c.score)) : raw.confidence,
leftX: chars.length > 0 ? chars[0]!.x0 : null,
raw,
};
}

47
app/features/cv/core/detectors/scoreboard/header-entries.ts Normal file
View File

@@ -0,0 +1,47 @@
/**
* Closed sets for the scoreboard header. OCR output is snapped to the
* nearest entry, so recognition only has to be roughly right.
* Source: header.txt (repo root).
*/
export const LOBBIES = [
"X Battle",
"Anarchy Battle (Series)",
"Anarchy Battle (Open)",
"Private Battle",
] as const;
export const MODES = [
"Turf War",
"Splat Zones",
"Tower Control",
"Rainmaker",
"Clam Blitz",
] as const;
export const STAGES = [
"Scorch Gorge",
"Eeltail Alley",
"Hagglefish Market",
"Undertow Spillway",
"Mincemeat Metalworks",
"Hammerhead Bridge",
"Museum d'Alfonsino",
"Mahi-Mahi Resort",
"Inkblot Art Academy",
"Sturgeon Shipyard",
"MakoMart",
"Wahoo World",
"Flounder Heights",
"Brinewater Springs",
"Manta Maria",
"Um'ami Ruins",
"Humpback Pump Track",
"Barnacle & Dime",
"Crableg Capital",
"Shipshape Cargo Co.",
"Bluefin Depot",
"Robo ROM-en",
"Marlin Airport",
"Lemuria Hub",
"Urchin Underpass",
] as const;

137
app/features/cv/core/detectors/scoreboard/header.ts Normal file
View File

@@ -0,0 +1,137 @@
/**
* Header parsing: lobby type ("X Battle"), mode ("Splat Zones") and stage
* ("Scorch Gorge") from the black tags above the team boxes.
*
* The tags auto-size to their text, so the stage's x position depends on the
* mode's length. Each band is first trimmed to the tag extent (tag columns
* are near-black background + white text; the map thumbnail around them is
* mid-brightness), then OCR'd as one line and snapped to the known entries:
* the mode+stage line is matched against every language's mode × stage
* combinations (core/localized.ts), and the reported values are always the
* canonical English names regardless of the game's language.
*/
import { getCV, type Mat } from "../../cv";
import { type GlyphSet, type RecognizeOptions, recognizeText } from "../../glyphs";
import { copyRoi } from "../../image";
import { ALL_LOBBY_ENTRIES, MODE_STAGE_COMBOS } from "../../localized";
import { closestBy } from "../../text";
import { HEADER_LINE_BAND, HEADER_LOBBY_BAND } from "./rois";
export interface ParsedHeader {
lobby: string | null;
mode: string | null;
stage: string | null;
/** min of the closed-set match scores that were attempted */
confidence: number;
debug: {
lobbyReading: string;
lineReading: string;
lobbyScore: number;
lineScore: number;
};
}
/** Accept a closed-set match only above this score (1 = exact). */
const MIN_MATCH_SCORE = 0.62;
/** A column belongs to a tag when nearly all its pixels are dark bg or bright text. */
const TAG_COLUMN_FRACTION = 0.85;
const TAG_DARK_MAX = 75;
const TAG_BRIGHT_MIN = 165;
/** Stop extending the tag after this many consecutive non-tag columns. */
const TAG_GAP_TOLERANCE = 6;
/**
* Trim a band crop to the black-tag extent starting from its left edge.
* Returns the trimmed width (0 when no tag is present at all).
*/
function tagExtent(crop: Mat, darkMax: number): number {
const { cols, rows, data } = crop;
let end = 0;
let gap = 0;
for (let x = 0; x < cols; x++) {
let tagLike = 0;
for (let y = 0; y < rows; y++) {
const v = data[y * cols + x]!;
if (v < darkMax || v > TAG_BRIGHT_MIN) tagLike++;
}
if (tagLike / rows >= TAG_COLUMN_FRACTION) {
end = x + 1;
gap = 0;
} else if (++gap > TAG_GAP_TOLERANCE) {
break;
}
}
return end;
}
export interface TagBandOptions extends RecognizeOptions {
/**
* Dark ceiling for the tag-extent trim. Lifted-blacks captures (720p
* streams upscaled and re-encoded) raise the tag background above the
* default, truncating the trim to a sliver — callers whose closed-set
* snap fails retry with a lifted ceiling.
*/
tagDarkMax?: number;
}
/**
* OCR one header band: trim the crop to the black-tag extent, then
* recognize it as a single line. Shared with the scoreboard-replay header,
* whose tags have the same style at different positions/sizes.
*/
export function readTagBand(
gray: Mat,
band: { x: number; y: number; w: number; h: number },
glyphs: GlyphSet,
options: TagBandOptions = {},
): string {
const crop = copyRoi(gray, band);
const width = tagExtent(crop, options.tagDarkMax ?? TAG_DARK_MAX);
if (width < 12) {
crop.delete();
return "";
}
const cv = getCV();
const view = crop.roi(new cv.Rect(0, 0, width, crop.rows));
const trimmed = new cv.Mat();
view.copyTo(trimmed);
view.delete();
crop.delete();
const result = recognizeText(trimmed, glyphs, { spaceGap: 9, minCharScore: 0.3, ...options });
trimmed.delete();
return result.text.trim();
}
export function parseHeader(gray: Mat, lobbyGlyphs: GlyphSet, lineGlyphs: GlyphSet): ParsedHeader {
const lobbyReading = readTagBand(gray, HEADER_LOBBY_BAND, lobbyGlyphs);
const lineReading = readTagBand(gray, HEADER_LINE_BAND, lineGlyphs);
const lobbyMatch = lobbyReading
? closestBy(lobbyReading, ALL_LOBBY_ENTRIES, (e) => e.text)
: null;
const lineMatch = lineReading ? closestBy(lineReading, MODE_STAGE_COMBOS, (c) => c.text) : null;
const lobby =
lobbyMatch && lobbyMatch.score >= MIN_MATCH_SCORE ? lobbyMatch.entry.canonical : null;
let mode: string | null = null;
let stage: string | null = null;
if (lineMatch && lineMatch.score >= MIN_MATCH_SCORE) {
mode = lineMatch.entry.mode;
stage = lineMatch.entry.stage;
}
const attempted = [lobbyMatch?.score ?? 0, lineMatch?.score ?? 0];
return {
lobby,
mode,
stage,
confidence: Math.min(...attempted),
debug: {
lobbyReading,
lineReading,
lobbyScore: lobbyMatch?.score ?? 0,
lineScore: lineMatch?.score ?? 0,
},
};
}

269
app/features/cv/core/detectors/scoreboard/index.ts Normal file
View File

@@ -0,0 +1,269 @@
/**
* ScoreboardDetector: parses the end-of-match results scoreboard
* (team scores + 8 rows of weapon / name / paint / splats / deaths / specials).
*/
import { getCV, type Mat } from "../../cv";
import { type GlyphSet, scaleGlyphSet } from "../../glyphs";
import { cropRoi, maxBrightness, meanBrightness } from "../../image";
import type { DetectedEvent, Detector, GateResult } from "../types";
import { parseNumber } from "./digits";
import { type ParsedHeader, parseHeader } from "./header";
import { findPovIndex } from "./pov";
import {
GATE_DARK_MAX_MEAN,
GATE_PANEL_MAX_MEAN,
GATE_PANEL_PROBES,
GATE_TEXT_MIN_MAX,
gateDarkProbe,
nameRoi,
PAINT_DIGIT_HEIGHT,
paintRoi,
paintSuffixRoi,
povArrowRoi,
ROW_CENTERS,
specialIconRoi,
statRoi,
TEAM_DIGIT_HEIGHT,
TEAM_SCORE_ROIS,
weaponRoi,
} from "./rois";
import { parseScoreboardRow, type RowRois } from "./row";
import type { SpecialMatch, SpecialTemplate } from "./specials";
import type { WeaponMatch, WeaponTemplate } from "./weapons";
export interface ScoreboardPlayer {
name: string;
/** sendou main-weapon id; null when the row's weapon was unreadable */
weaponId: number | null;
paint: number | null;
/** kills+assists (the combined counter as shown) */
ka: number | null;
d: number | null;
s: number | null;
}
export interface ScoreboardData {
/** e.g. "X Battle", from the header tag; null when unreadable */
lobby: string | null;
/** e.g. "Splat Zones" */
mode: string | null;
/** e.g. "Scorch Gorge" */
stage: string | null;
/** [winning team total, losing team total] as shown ("500 p") */
scores: [number | null, number | null];
/** 8 players: rows 0-3 winning team, rows 4-7 losing team */
players: ScoreboardPlayer[];
/**
* index into `players` of the recording player's row, marked by the
* yellow arrow; null when no arrow is found (spectator/overhead footage)
*/
povIndex: number | null;
}
export interface ScoreboardRowDebug {
weapon: WeaponMatch | null;
/** row's special icon match, when it was consulted for a weapon tie */
special?: SpecialMatch;
paintScore: number;
nameScore: number;
statScores: [number, number, number];
/** fraction of the row's POV-arrow probe that is arrow-yellow */
povFraction: number;
}
export interface ScoreboardResources {
weapons: WeaponTemplate[];
/**
* Special-weapon silhouettes (assets/cv/specials). Optional: without
* them, near-tied weapon icons (Splash- vs Sploosh-o-matic) stay decided
* by icon score alone.
*/
specials?: SpecialTemplate[] | null;
/** digit templates at paint size (h~28); team scores reuse these, scaled */
paintDigits: GlyphSet | null;
/** digit templates at stat-counter size (h~17) */
statDigits: GlyphSet | null;
/**
* digit templates for team totals (h~33, outlined, on team-color box).
* Optional: falls back to scaled paint digits, at reduced accuracy.
*/
teamDigits: GlyphSet | null;
nameGlyphs: GlyphSet | null;
/** header tag glyphs: lobby line, and the mode+stage line */
headerLobbyGlyphs: GlyphSet | null;
headerLineGlyphs: GlyphSet | null;
/**
* Replay-browser extras (FOT-RowdyStd face, unlike everything above):
* the replay code line, and the VICTORY/DEFEAT panel tags. Optional:
* the replay detector falls back to rescaled name/header glyphs, at
* reduced accuracy.
*/
replayCodeGlyphs?: GlyphSet | null;
replayResultGlyphs?: GlyphSet | null;
/**
* Death-screen extras. Optional: the death detector skips the fields it
* has no resources for. The weapon atlas carries both Blitz faces; the
* tag-name atlas is BlitzBold + Rowdy (kana).
*/
abilities?: import("../death/abilities").AbilityTemplates | null;
deathWeaponGlyphs?: GlyphSet | null;
/**
* JA death-message glyphs (Kurokane/Rowdy condensed + fixture crops),
* built at the on-screen text's native size. Optional: without them JA
* death screens fail the constant-line confirmation and emit nothing.
*/
deathWeaponJaGlyphs?: GlyphSet | null;
deathTagNameGlyphs?: GlyphSet | null;
/**
* The main-weapon icons again, rebuilt at the death burst's icon size
* (~124px vs the rows' 40-64px). Optional: without them the death
* detector cannot recover the killer's weapon when the message text is
* unreadable (e.g. the WIPEOUT banner covering the weapon name line).
*/
deathBurstWeapons?: WeaponTemplate[] | null;
/**
* Personal-results extras: the same ability icons rebuilt at the gear
* cards' badge sizes (scoreboard-own/abilities.ts). Optional: without
* them the scoreboard-own detector skips the ability grid.
*/
ownAbilities?: import("../death/abilities").AbilityTemplates | null;
/**
* Map-start intro extras: the big BlitzBold mode title, and the BlitzMain
* stage name (whose atlas also reads the "MODE" label, rescaled).
* Optional: without them the map-start detector emits nothing.
*/
mapStartModeGlyphs?: GlyphSet | null;
mapStartStageGlyphs?: GlyphSet | null;
/**
* Minimap extras (minimap/rois.ts documents the screen): the main-weapon
* icons composited on the card-pill background for the teammate cards'
* light silhouettes, a light-background variant for highlighted enemy
* rows, and the ability icons at the cards' badge size. Optional: without
* them the minimap detector skips the corresponding fields (enemy rows
* fall back to the standard `weapons` set). Names reuse `nameGlyphs`.
*/
minimapCardWeapons?: WeaponTemplate[] | null;
minimapLightWeapons?: WeaponTemplate[] | null;
minimapAbilities?: WeaponTemplate[] | null;
/**
* Sub-weapon silhouettes (assets/cv/sub-weapons) at the minimap sub
* tile's sizes. Optional: without them, near-tied minimap weapon icons
* whose kits differ only by sub (plain vs Custom Dualie Squelchers) stay
* decided by icon score alone.
*/
minimapSubWeapons?: SpecialTemplate[] | null;
/**
* Planner-map structural signatures (assets/cv/planner) used by the
* minimap detector to identify the stage. Optional: without them the
* minimap event's `stage` stays null.
*/
plannerStages?: import("../minimap/stage").PlannerStage[] | null;
}
export const SCOREBOARD_EVENT_TYPE = "Scoreboard";
export function createScoreboardDetector(resources: ScoreboardResources): Detector<ScoreboardData> {
const cv = getCV();
const teamDigits =
resources.teamDigits ??
(resources.paintDigits
? scaleGlyphSet(resources.paintDigits, TEAM_DIGIT_HEIGHT / PAINT_DIGIT_HEIGHT)
: null);
function gate(frame: Mat): GateResult {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
let darkOk = 0;
let suffixOk = 0;
for (const cy of ROW_CENTERS) {
if (meanBrightness(frame, gateDarkProbe(cy)) < GATE_DARK_MAX_MEAN) darkOk++;
if (maxBrightness(gray, paintSuffixRoi(cy)) > GATE_TEXT_MIN_MAX) suffixOk++;
}
let panelOk = 0;
for (const roi of GATE_PANEL_PROBES) {
if (meanBrightness(frame, roi) < GATE_PANEL_MAX_MEAN) panelOk++;
}
gray.delete();
const score =
(darkOk / ROW_CENTERS.length +
suffixOk / ROW_CENTERS.length +
panelOk / GATE_PANEL_PROBES.length) /
3;
const pass = darkOk >= 7 && suffixOk >= 6 && panelOk >= 2;
return { pass, score };
}
function parse(frame: Mat, t: number): DetectedEvent<ScoreboardData>[] {
const gray = new cv.Mat();
cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
const rgb = new cv.Mat();
cv.cvtColor(frame, rgb, cv.COLOR_RGBA2RGB);
const players: ScoreboardPlayer[] = [];
const rowDebug: ScoreboardRowDebug[] = [];
const confidences: number[] = [];
const rowRois: RowRois = {
weapon: weaponRoi,
specialIcon: specialIconRoi,
paint: paintRoi,
name: nameRoi,
stat: statRoi,
povArrow: povArrowRoi,
};
for (const cy of ROW_CENTERS) {
const row = parseScoreboardRow(gray, rgb, cy, rowRois, resources, confidences);
players.push(row.player);
rowDebug.push(row.debug);
}
const povIndex = findPovIndex(rowDebug.map((r) => r.povFraction));
let header: ParsedHeader | null = null;
if (resources.headerLobbyGlyphs && resources.headerLineGlyphs) {
header = parseHeader(gray, resources.headerLobbyGlyphs, resources.headerLineGlyphs);
confidences.push(header.confidence);
}
const scores: [number | null, number | null] = [null, null];
const teamScoreConf: number[] = [];
if (teamDigits) {
for (const i of [0, 1] as const) {
// Team totals sit on the team-colored box (light swirl pattern),
// so binarize more aggressively than on the black pills.
const crop = cropRoi(gray, TEAM_SCORE_ROIS[i]);
const parsed = parseNumber(crop, teamDigits, { binThreshold: 175 });
crop.delete();
scores[i] = parsed.value;
teamScoreConf.push(parsed.confidence);
confidences.push(parsed.confidence);
}
}
gray.delete();
rgb.delete();
const confidence =
confidences.length > 0 ? confidences.reduce((a, b) => a + b, 0) / confidences.length : 0;
return [
{
type: SCOREBOARD_EVENT_TYPE,
t,
confidence,
data: {
lobby: header?.lobby ?? null,
mode: header?.mode ?? null,
stage: header?.stage ?? null,
scores,
players,
povIndex,
},
debug: { rows: rowDebug, teamScoreConf, header: header?.debug },
},
];
}
return { id: "scoreboard", gate, parse };
}

190
app/features/cv/core/detectors/scoreboard/kits.ts Normal file
View File

@@ -0,0 +1,190 @@
/**
* Main-weapon kits (sub + special ids) keyed by in-game main weapon id,
* generated from sendou.ink app/features/build-analyzer/data/weapon-params.ts
* (weaponKits section; regenerate when new weapons or kit shuffles ship).
* Special ids match assets/cv/specials/<id>.png; the results scoreboard
* shows each player's special icon above the specials counter, which lets
* near-tied weapon icon matches be disambiguated by kit.
*/
export interface WeaponKit {
/** sendou.ink sub weapon id */
sub: number;
/** sendou.ink special weapon id, matching assets/cv/specials */
special: number;
}
export const WEAPON_KITS: ReadonlyMap<string, WeaponKit> = new Map([
["0", { sub: 6, special: 11 }],
["1", { sub: 8, special: 9 }],
["10", { sub: 0, special: 2 }],
["11", { sub: 13, special: 7 }],
["20", { sub: 2, special: 12 }],
["21", { sub: 1, special: 14 }],
["22", { sub: 11, special: 5 }],
["30", { sub: 5, special: 13 }],
["31", { sub: 3, special: 6 }],
["32", { sub: 2, special: 19 }],
["40", { sub: 1, special: 1 }],
["41", { sub: 0, special: 14 }],
["42", { sub: 2, special: 17 }],
["45", { sub: 1, special: 1 }],
["46", { sub: 0, special: 14 }],
["47", { sub: 1, special: 1 }],
["50", { sub: 4, special: 9 }],
["51", { sub: 6, special: 19 }],
["60", { sub: 1, special: 15 }],
["61", { sub: 7, special: 16 }],
["70", { sub: 12, special: 12 }],
["71", { sub: 1, special: 6 }],
["72", { sub: 0, special: 4 }],
["80", { sub: 3, special: 8 }],
["81", { sub: 4, special: 17 }],
["82", { sub: 12, special: 15 }],
["90", { sub: 12, special: 8 }],
["91", { sub: 11, special: 5 }],
["92", { sub: 2, special: 18 }],
["100", { sub: 9, special: 9 }],
["101", { sub: 10, special: 10 }],
["200", { sub: 0, special: 3 }],
["201", { sub: 5, special: 11 }],
["205", { sub: 0, special: 3 }],
["210", { sub: 7, special: 2 }],
["211", { sub: 9, special: 18 }],
["212", { sub: 8, special: 12 }],
["220", { sub: 1, special: 7 }],
["221", { sub: 0, special: 17 }],
["230", { sub: 0, special: 1 }],
["231", { sub: 6, special: 16 }],
["240", { sub: 10, special: 14 }],
["241", { sub: 13, special: 10 }],
["250", { sub: 11, special: 8 }],
["251", { sub: 12, special: 9 }],
["252", { sub: 1, special: 15 }],
["260", { sub: 3, special: 13 }],
["261", { sub: 2, special: 6 }],
["300", { sub: 6, special: 12 }],
["301", { sub: 2, special: 11 }],
["302", { sub: 0, special: 10 }],
["310", { sub: 9, special: 15 }],
["311", { sub: 4, special: 2 }],
["312", { sub: 1, special: 14 }],
["400", { sub: 4, special: 1 }],
["401", { sub: 7, special: 19 }],
["1000", { sub: 7, special: 3 }],
["1001", { sub: 2, special: 1 }],
["1002", { sub: 5, special: 16 }],
["1010", { sub: 6, special: 2 }],
["1011", { sub: 8, special: 17 }],
["1015", { sub: 6, special: 2 }],
["1020", { sub: 3, special: 15 }],
["1021", { sub: 0, special: 16 }],
["1022", { sub: 9, special: 9 }],
["1030", { sub: 10, special: 4 }],
["1031", { sub: 1, special: 19 }],
["1040", { sub: 4, special: 8 }],
["1041", { sub: 12, special: 5 }],
["1042", { sub: 13, special: 18 }],
["1100", { sub: 0, special: 9 }],
["1101", { sub: 10, special: 11 }],
["1110", { sub: 1, special: 3 }],
["1111", { sub: 8, special: 5 }],
["1112", { sub: 7, special: 17 }],
["1115", { sub: 1, special: 3 }],
["1120", { sub: 6, special: 7 }],
["1121", { sub: 9, special: 4 }],
["1122", { sub: 4, special: 1 }],
["2000", { sub: 9, special: 2 }],
["2001", { sub: 7, special: 3 }],
["2010", { sub: 0, special: 8 }],
["2011", { sub: 4, special: 14 }],
["2012", { sub: 3, special: 12 }],
["2015", { sub: 0, special: 8 }],
["2020", { sub: 0, special: 8 }],
["2021", { sub: 4, special: 14 }],
["2022", { sub: 3, special: 12 }],
["2030", { sub: 10, special: 7 }],
["2031", { sub: 8, special: 17 }],
["2040", { sub: 10, special: 7 }],
["2041", { sub: 8, special: 17 }],
["2050", { sub: 7, special: 9 }],
["2051", { sub: 5, special: 16 }],
["2060", { sub: 13, special: 4 }],
["2061", { sub: 5, special: 11 }],
["2070", { sub: 3, special: 15 }],
["2071", { sub: 4, special: 5 }],
["3000", { sub: 0, special: 14 }],
["3001", { sub: 12, special: 3 }],
["3005", { sub: 0, special: 14 }],
["3010", { sub: 11, special: 10 }],
["3011", { sub: 5, special: 15 }],
["3012", { sub: 0, special: 19 }],
["3020", { sub: 5, special: 6 }],
["3021", { sub: 9, special: 1 }],
["3030", { sub: 3, special: 5 }],
["3031", { sub: 12, special: 17 }],
["3040", { sub: 9, special: 5 }],
["3041", { sub: 4, special: 18 }],
["3050", { sub: 1, special: 13 }],
["3051", { sub: 8, special: 7 }],
["3052", { sub: 6, special: 12 }],
["4000", { sub: 2, special: 11 }],
["4001", { sub: 11, special: 2 }],
["4002", { sub: 8, special: 1 }],
["4010", { sub: 3, special: 7 }],
["4011", { sub: 9, special: 17 }],
["4015", { sub: 3, special: 7 }],
["4020", { sub: 7, special: 6 }],
["4021", { sub: 10, special: 19 }],
["4022", { sub: 3, special: 2 }],
["4030", { sub: 5, special: 10 }],
["4031", { sub: 10, special: 8 }],
["4040", { sub: 9, special: 5 }],
["4041", { sub: 1, special: 18 }],
["4050", { sub: 6, special: 15 }],
["4051", { sub: 0, special: 12 }],
["5000", { sub: 8, special: 15 }],
["5001", { sub: 13, special: 13 }],
["5002", { sub: 0, special: 9 }],
["5010", { sub: 1, special: 12 }],
["5011", { sub: 6, special: 18 }],
["5012", { sub: 5, special: 2 }],
["5015", { sub: 1, special: 12 }],
["5020", { sub: 4, special: 6 }],
["5021", { sub: 9, special: 1 }],
["5030", { sub: 0, special: 7 }],
["5031", { sub: 8, special: 16 }],
["5032", { sub: 9, special: 19 }],
["5040", { sub: 7, special: 13 }],
["5041", { sub: 3, special: 3 }],
["5050", { sub: 10, special: 9 }],
["5051", { sub: 2, special: 14 }],
["6000", { sub: 3, special: 14 }],
["6001", { sub: 7, special: 10 }],
["6005", { sub: 3, special: 14 }],
["6010", { sub: 8, special: 8 }],
["6011", { sub: 10, special: 1 }],
["6012", { sub: 11, special: 16 }],
["6020", { sub: 10, special: 13 }],
["6021", { sub: 13, special: 19 }],
["6022", { sub: 6, special: 9 }],
["6030", { sub: 12, special: 2 }],
["6031", { sub: 11, special: 18 }],
["7010", { sub: 11, special: 9 }],
["7011", { sub: 3, special: 16 }],
["7012", { sub: 12, special: 10 }],
["7015", { sub: 11, special: 9 }],
["7020", { sub: 6, special: 4 }],
["7021", { sub: 4, special: 13 }],
["7022", { sub: 13, special: 6 }],
["7030", { sub: 7, special: 11 }],
["7031", { sub: 9, special: 7 }],
["8000", { sub: 2, special: 3 }],
["8001", { sub: 11, special: 12 }],
["8002", { sub: 7, special: 6 }],
["8005", { sub: 2, special: 3 }],
["8010", { sub: 13, special: 11 }],
["8011", { sub: 8, special: 4 }],
["8012", { sub: 6, special: 1 }],
["8020", { sub: 1, special: 2 }],
["8021", { sub: 4, special: 10 }],
]);

220
app/features/cv/core/detectors/scoreboard/names.ts Normal file
View File

@@ -0,0 +1,220 @@
/**
* Player name recognition: glyph matching over the name ROI.
*/
import type { Mat } from "../../cv";
import {
type GlyphSet,
type RecognizedChar,
type RecognizedText,
recognizeText,
} from "../../glyphs";
export interface ParsedName {
name: string;
/** min glyph score; 0 when ink was present but nothing recognized */
confidence: number;
raw: RecognizedText;
}
/**
* BlitzMain renders 'I', 'l', '|', and '1' as near-identical bars — no pixel
* evidence separates them, so fall back to context, by decreasing weight of
* evidence: a bar next to a lowercase letter is overwhelmingly an 'l' in
* latin names ("Olise"), next to a digit it's a '1' ("Jrod_14"), next to an
* uppercase letter it's an 'I' ("SHIP"), a bare bar hanging off an
* underscore is a numbered-alt suffix ("gori_1"), and with no latin/digit
* context at all (kana, symbols, edges) 'l' is the common case in the wild.
* (This will genuinely miss e.g. "McIntosh", but so would a human reading
* the pixels.)
*/
const BAR_CHARS = new Set(["I", "l", "|", "1"]);
function normalizeBars(name: string): string {
const chars = [...name];
// Context is the nearest NON-BAR char within the word: consecutive bars
// ("ll" in "Chill") must all resolve from the same real-letter neighbor,
// not from each other's arbitrary raw reading. Underscores bound words
// like spaces do — "gori_1"'s lowercase must not leak across the
// separator onto the suffix.
const neighbor = (i: number, step: -1 | 1): string | undefined => {
for (let j = i + step; j >= 0 && j < chars.length; j += step) {
const c = chars[j]!;
if (c === " " || c === "_") return undefined;
if (!BAR_CHARS.has(c)) return c;
}
return undefined;
};
const test = (re: RegExp) => (c: string | undefined) => c !== undefined && re.test(c);
const isLower = test(/\p{Ll}/u);
const isDigit = test(/\d/);
const isUpper = test(/\p{Lu}/u);
return chars
.map((c, i) => {
if (!BAR_CHARS.has(c)) return c;
const left = neighbor(i, -1);
const right = neighbor(i, 1);
if (isLower(left) || isLower(right)) return "l";
if (isDigit(left) || isDigit(right)) return "1";
if (isUpper(left) || isUpper(right)) return "I";
if (chars[i - 1] === "_" || chars[i + 1] === "_") return "1";
return "l";
})
.join("");
}
/**
* The kana long-vowel bar 'ー' and the ASCII hyphen '-' are horizontal-bar
* homoglyphs the same way the vertical bars are: both are a single
* horizontal stroke whose length difference drowns in capture blur, so
* which template wins is noise ("ドラグ-ン"). Resolve by script context —
* a kana neighbor reads 'ー', a Latin/digit neighbor reads '-', and with
* no context the raw pick stands.
*/
const LONG_BAR_CHARS = new Set(["-", "ー"]);
function normalizeLongBars(name: string): string {
const chars = [...name];
const neighbor = (i: number, step: -1 | 1): string | undefined => {
for (let j = i + step; j >= 0 && j < chars.length; j += step) {
const c = chars[j]!;
if (c === " " || c === "_") return undefined;
if (!LONG_BAR_CHARS.has(c)) return c;
}
return undefined;
};
const test = (re: RegExp) => (c: string | undefined) => c !== undefined && re.test(c);
const isKana = test(/[ぁ-ヾ]/u);
const isLatinOrDigit = test(/[a-zA-Z0-9]/);
return chars
.map((c, i) => {
if (!LONG_BAR_CHARS.has(c)) return c;
const left = neighbor(i, -1);
const right = neighbor(i, 1);
if (isKana(left) || isKana(right)) return "ー";
if (isLatinOrDigit(left) || isLatinOrDigit(right)) return "-";
return c;
})
.join("");
}
/**
* BlitzMain's 'O' and '0' are the same rounded box at capture fidelity —
* which template wins is noise — so, like the bars, resolve by context via
* the nearest unambiguous neighbor in the word: a digit neighbor keeps '0',
* an uppercase neighbor reads 'O' ("AHOO"), after a lowercase letter it's a
* stylized digit ("y0s"), and word-initial before lowercase it's a
* capitalized name ("Olise").
*/
function normalizeOhs(name: string): string {
const chars = [...name];
const ambiguous = (c: string | undefined) => c === "O" || c === "0";
const neighbor = (i: number, step: -1 | 1): string | undefined => {
for (let j = i + step; j >= 0 && j < chars.length; j += step) {
const c = chars[j]!;
if (c === " ") return undefined;
if (!ambiguous(c)) return c;
}
return undefined;
};
const test = (re: RegExp) => (c: string | undefined) => c !== undefined && re.test(c);
const isDigit = test(/\d/);
const isUpper = test(/\p{Lu}/u);
const isLower = test(/\p{Ll}/u);
return chars
.map((c, i) => {
if (!ambiguous(c)) return c;
const left = neighbor(i, -1);
const right = neighbor(i, 1);
if (isDigit(left) || isDigit(right)) return "0";
if (isUpper(left) || isUpper(right)) return "O";
if (isLower(left)) return "0";
if (isLower(right)) return "O";
return c;
})
.join("");
}
/**
* The round dots '.', '・', and '·' tight-crop to near-identical blobs, so
* which template wins is noise. Vertical position decides in one direction
* only: a period sits on the baseline, so a dot floating well above it
* cannot be '.' — reread it as the best-scoring middle-dot candidate. The
* reverse does not hold: BlitzMain draws '・' ON the baseline in some names
* (scoreboard/robot row 5, "..・"), so baseline dots stay with the template
* ranking, where the exact fixture crops separate the sizes.
*/
const DOT_CHARS = new Set([".", "・", "·"]);
const DOT_BASELINE_SLACK_PX = 3;
function fixRaisedDots(raw: RecognizedText): RecognizedText {
if (!raw.chars.some((c) => c.char === ".")) return raw;
const anchors = raw.chars
.filter((c) => !DOT_CHARS.has(c.char))
.map((c) => c.y1)
.sort((a, b) => a - b);
if (anchors.length === 0) return raw;
const baseline = anchors[Math.floor(anchors.length / 2)]!;
const chars = raw.chars.map((c) => {
if (c.char !== "." || baseline - c.y1 <= DOT_BASELINE_SLACK_PX) return c;
const alt = c.candidates?.find((k) => DOT_CHARS.has(k.char) && k.char !== ".");
return { ...c, char: alt?.char ?? "・" };
});
let ci = 0;
const text = [...raw.text].map((ch) => (ch === " " ? ch : chars[ci++]!.char)).join("");
return { ...raw, text, chars };
}
/**
* BlitzMain's 'P' and 'p' tight-crop to the same stem-and-bowl shape, so the
* template scores between them are noise — but unlike the bars and ohs the
* pixels do decide: 'p' hangs below the baseline while 'P' sits on it. The
* templates lose that position, the segment keeps it. Take the baseline as
* the median ink bottom of the non-twin glyphs (most chars rest exactly on
* it, so the median shrugs off real descenders and symbols) and pick the
* case by whether the segment descends past it. Skipped when no other glyph
* anchors the baseline.
*/
const DESCENDER_TWINS: Record<string, [upper: string, lower: string]> = {
P: ["P", "p"],
p: ["P", "p"],
};
const DESCENT_MIN_PX = 3;
function resolveCaseByDescent(raw: RecognizedText): string {
if (!raw.chars.some((c) => c.char in DESCENDER_TWINS)) return raw.text;
const anchors = raw.chars
.filter((c) => !(c.char in DESCENDER_TWINS))
.map((c) => c.y1)
.sort((a, b) => a - b);
if (anchors.length === 0) return raw.text;
const baseline = anchors[Math.floor(anchors.length / 2)]!;
let ci = 0;
return [...raw.text]
.map((ch) => {
if (ch === " ") return ch;
const rc: RecognizedChar = raw.chars[ci++]!;
const twin = DESCENDER_TWINS[rc.char];
if (!twin) return rc.char;
return rc.y1 - baseline >= DESCENT_MIN_PX ? twin[1] : twin[0];
})
.join("");
}
export function parseName(
gray: Mat,
glyphs: GlyphSet,
options: { spaceGap?: number; binThreshold?: number } = {},
): ParsedName {
const raw = recognizeText(gray, glyphs, {
spaceGap: options.spaceGap ?? 7,
binThreshold: options.binThreshold,
minCharScore: 0.35,
});
return {
name: normalizeLongBars(
normalizeOhs(normalizeBars(resolveCaseByDescent(fixRaisedDots(raw)).trim())),
),
confidence: raw.confidence,
raw,
};
}

67
app/features/cv/core/detectors/scoreboard/pov.ts Normal file
View File

@@ -0,0 +1,67 @@
/**
* POV arrow detection: both results screens mark the recording player's row
* with a solid yellow arrow at the row's left edge (absent in spectator or
* overhead footage). The arrow is the only saturated pure-yellow blob in
* that strip — team-ink yellows there are patterned/duller and stay under
* the mask thresholds — so a plain color-mask pixel count decides, no
* template needed.
*/
import { getCV, type Mat } from "../../cv";
import { cropRoi, type Roi } from "../../image";
const YELLOW_R_MIN = 190;
const YELLOW_G_MIN = 180;
const YELLOW_B_MAX = 90;
/** Pure arrow yellow has R≈G; team golds/chartreuse skew one channel. */
const YELLOW_RG_MAX_DIFF = 60;
/** Arrow pixels fill ~15-30% of the probe ROI; arrow-less rows measure ~0. */
const POV_MIN_FRACTION = 0.05;
/**
* If the runner-up row reaches this share of the best row's fraction, the
* yellow is ambient (team ink leaking into the strip), not the arrow.
*/
const POV_RUNNER_UP_MAX_RATIO = 0.5;
/** Fraction of `roi` pixels that are arrow-yellow. `rgb` is a 3-channel RGB mat. */
export function povYellowFraction(rgb: Mat, roi: Roi): number {
const cv = getCV();
const view = cropRoi(rgb, roi);
const cont = new cv.Mat();
view.copyTo(cont);
view.delete();
const d = cont.data;
const n = cont.rows * cont.cols;
let yellow = 0;
for (let i = 0; i < n; i++) {
const r = d[i * 3]!;
const g = d[i * 3 + 1]!;
const b = d[i * 3 + 2]!;
if (
r > YELLOW_R_MIN &&
g > YELLOW_G_MIN &&
b < YELLOW_B_MAX &&
Math.abs(r - g) < YELLOW_RG_MAX_DIFF
) {
yellow++;
}
}
cont.delete();
return n > 0 ? yellow / n : 0;
}
/**
* Pick the arrow row from per-row yellow fractions: the best row must clear
* POV_MIN_FRACTION and stand clear of the runner-up; null means no arrow.
*/
export function findPovIndex(fractions: readonly number[]): number | null {
let best = -1;
for (let i = 0; i < fractions.length; i++) {
if (best < 0 || fractions[i]! > fractions[best]!) best = i;
}
if (best < 0 || fractions[best]! < POV_MIN_FRACTION) return null;
for (let i = 0; i < fractions.length; i++) {
if (i !== best && fractions[i]! > fractions[best]! * POV_RUNNER_UP_MAX_RATIO) return null;
}
return best;
}

111
app/features/cv/core/detectors/scoreboard/rois.ts Normal file
View File

@@ -0,0 +1,111 @@
/**
* ALL scoreboard ROI coordinates, in canonical 1920x1080 space.
* Calibrated against scoreboard/xbattle-splat-zones-ko via
* tools/overlay-rois.ts and column-projection measurement.
*
* The results-screen scoreboard is a fixed-layout panel on the right side:
* two team boxes (top = winner, bottom = loser), 4 dark "pill" rows each,
* white text on the pills. Per row, left to right: avatar, weapon icon,
* name (left-aligned from x=1125), paint ("842p", digits right-aligned
* ending at x=1409, constant "p" glyph at 1411-1424), three stat counters
* ("x12", zero-padded 2 digits) under the splat/death/special icons.
*/
import type { Roi } from "../../canonical";
/** Vertical centers of the 8 player rows: 4 winner rows, then 4 loser rows. */
export const ROW_CENTERS = [416, 482, 547, 613, 770, 836, 901, 967] as const;
/**
* Weapon icon search region within a row (icon size/offset varies per
* weapon). matchTemplate silently skips any template taller than the
* region, so the 56px height both fits the largest live-scoreboard icon
* and intentionally excludes the larger replay-browser template sizes.
*/
export function weaponRoi(cy: number): Roi {
return { x: 1054, y: cy - 28, w: 67, h: 56 };
}
/**
* Player name text region (white text, left-aligned starting x=1126).
* Long names run into the paint column; parse paint first and trim this
* region at the leftmost paint digit.
*/
export function nameRoi(cy: number): Roi {
return { x: 1122, y: cy - 14, w: 208, h: 32 };
}
/** Paint amount digits, right-aligned ending at x=1409 (the "p" suffix is excluded). */
export function paintRoi(cy: number): Roi {
return { x: 1325, y: cy - 17, w: 84, h: 34 };
}
/** The constant white "p" after the paint number — used as a gate anchor. */
export function paintSuffixRoi(cy: number): Roi {
return { x: 1409, y: cy - 14, w: 18, h: 28 };
}
/** Stat counter digits (two, zero-padded; the small "x" prefix at 1477/1540/1603 is excluded). */
export function statRoi(cy: number, index: 0 | 1 | 2): Roi {
const x = [1484, 1547, 1610][index]!;
return { x, y: cy + 1, w: 32, h: 23 };
}
/**
* The player's special-weapon icon, drawn above the specials counter in the
* team's ink color (~22px art around x 1602-1624, y cy-22..cy+1). Bounded
* below at cy+1 so the white counter digits stay out of the binarized shape.
*/
export function specialIconRoi(cy: number): Roi {
return { x: 1595, y: cy - 29, w: 40, h: 30 };
}
/**
* POV arrow probe: the yellow arrow marking the recording player's row sits
* left of the avatar, overlapping the team-box edge (measured x 933-985,
* y cy-29..cy+23 across fixtures). Right edge stays short of the avatar
* circle (~x 995) so yellow hair/gear can't leak in.
*/
export function povArrowRoi(cy: number): Roi {
return { x: 930, y: cy - 32, w: 58, h: 56 };
}
/** Team score totals ("500 p"), larger digits, right-aligned ending at x=1658. */
export const TEAM_SCORE_ROIS: readonly [Roi, Roi] = [
{ x: 1530, y: 330, w: 132, h: 44 },
{ x: 1530, y: 684, w: 132, h: 44 },
];
/**
* Gate probe: the strip between the paint "p" suffix (ends 1424) and the
* first stat "x" (starts 1477) is always empty pill background (near-black).
*/
export function gateDarkProbe(cy: number): Roi {
return { x: 1434, y: cy - 10, w: 38, h: 20 };
}
/** Panel background probes (dark gray ~35) outside the team boxes. */
export const GATE_PANEL_PROBES: readonly Roi[] = [
{ x: 800, y: 500, w: 30, h: 30 },
{ x: 800, y: 900, w: 30, h: 30 },
{ x: 1770, y: 930, w: 30, h: 30 },
];
export const GATE_DARK_MAX_MEAN = 60;
export const GATE_PANEL_MAX_MEAN = 85;
/** The paint "p" suffix region must contain bright (white) pixels. */
export const GATE_TEXT_MIN_MAX = 180;
/**
* Header bands. The lobby tag ("X Battle") sits on its own line; mode and
* stage tags share the next line. Tags are black boxes that size to their
* text, so these bands are generous — header.ts trims each band to the
* actual tag extent (map thumbnail pixels around the tags are excluded by
* column statistics, not by fixed coordinates).
*/
export const HEADER_LOBBY_BAND: Roi = { x: 828, y: 42, w: 330, h: 30 };
export const HEADER_LINE_BAND: Roi = { x: 828, y: 88, w: 580, h: 40 };
/** Text metrics measured on the fixture, used by atlas tooling. */
export const PAINT_DIGIT_HEIGHT = 28;
/** Same tight height as paint digits, but rendered in the bold face (BlitzBold). */
export const TEAM_DIGIT_HEIGHT = 28;

142
app/features/cv/core/detectors/scoreboard/row.ts Normal file
View File

@@ -0,0 +1,142 @@
/**
* The shared scoreboard row parse — weapon (with special-icon tie-break),
* paint, name trimmed at the leftmost paint digit, stat counters — used by
* both the results scoreboard and the replay-browser detail screen. The
* callers differ only in ROI geometry (the replay panels are dx-shifted),
* glyph sets (replay rows are smaller, so it passes rescaled sets), and two
* match options.
*/
import type { Mat } from "../../cv";
import type { GlyphSet } from "../../glyphs";
import { cropRoi, type Roi } from "../../image";
import { type ParsedNumber, parseNumber } from "./digits";
import type { ScoreboardPlayer, ScoreboardRowDebug } from "./index";
import { parseName } from "./names";
import { povYellowFraction } from "./pov";
import {
disambiguateWeaponBySpecial,
matchSpecial,
type SpecialMatch,
type SpecialTemplate,
tiedWeaponsWithDistinctSpecials,
} from "./specials";
import { matchWeapon, type WeaponMatch, type WeaponTemplate } from "./weapons";
/** Per-row ROI geometry; the replay detector closes these over its panel dx. */
export interface RowRois {
weapon(cy: number): Roi;
specialIcon(cy: number): Roi;
paint(cy: number): Roi;
name(cy: number): Roi;
stat(cy: number, i: 0 | 1 | 2): Roi;
povArrow(cy: number): Roi;
}
export interface RowResources {
weapons: WeaponTemplate[];
specials?: SpecialTemplate[] | null;
paintDigits: GlyphSet | null;
statDigits: GlyphSet | null;
nameGlyphs: GlyphSet | null;
}
export interface RowOptions {
/** passed through to matchWeapon (replay rows sit on a lighter panel) */
weaponInkThreshold?: number;
/**
* replay only: the left-aligned paint number puts the "p" suffix inside
* the ROI when the paint has fewer than 4 digits
*/
paintDropLoweredTrailing?: boolean;
}
/** Parses one player row; per-field confidences append to `confidences`. */
export function parseScoreboardRow(
gray: Mat,
rgb: Mat,
cy: number,
rois: RowRois,
resources: RowResources,
confidences: number[],
options: RowOptions = {},
): { player: ScoreboardPlayer; debug: ScoreboardRowDebug } {
let weapon: WeaponMatch | null = null;
let special: SpecialMatch | undefined;
if (resources.weapons.length > 0) {
const crop = cropRoi(rgb, rois.weapon(cy));
weapon = matchWeapon(
crop,
resources.weapons,
options.weaponInkThreshold !== undefined ? { inkThreshold: options.weaponInkThreshold } : {},
);
crop.delete();
// near-tied icons with different kit specials: let the row's special
// icon break the tie
if (resources.specials?.length && tiedWeaponsWithDistinctSpecials(weapon)) {
const spCrop = cropRoi(rgb, rois.specialIcon(cy));
special = matchSpecial(spCrop, resources.specials);
spCrop.delete();
weapon = disambiguateWeaponBySpecial(weapon, special);
}
confidences.push(Math.max(0, weapon.score));
}
// paint (parse first so the name region can be trimmed at the digits)
let paint: ParsedNumber | null = null;
const pRoi = rois.paint(cy);
if (resources.paintDigits) {
const crop = cropRoi(gray, pRoi);
paint = parseNumber(crop, resources.paintDigits, {
dropLoweredTrailing: options.paintDropLoweredTrailing,
});
crop.delete();
confidences.push(paint.confidence);
}
// name, trimmed at the leftmost paint digit
let name: ReturnType<typeof parseName> | null = null;
if (resources.nameGlyphs) {
const base = rois.name(cy);
const paintLeftAbs = paint && paint.leftX !== null ? pRoi.x + paint.leftX : pRoi.x + pRoi.w;
const w = Math.min(base.w, Math.max(0, paintLeftAbs - 6 - base.x));
if (w > 8) {
const crop = cropRoi(gray, { ...base, w });
name = parseName(crop, resources.nameGlyphs);
crop.delete();
confidences.push(name.confidence);
}
}
// stat counters
const statValues: (number | null)[] = [null, null, null];
const statScores: [number, number, number] = [0, 0, 0];
if (resources.statDigits) {
for (const i of [0, 1, 2] as const) {
const crop = cropRoi(gray, rois.stat(cy, i));
const parsed = parseNumber(crop, resources.statDigits);
crop.delete();
statValues[i] = parsed.value;
statScores[i] = parsed.confidence;
confidences.push(parsed.confidence);
}
}
return {
player: {
name: name?.name ?? "",
weaponId: weapon ? Number(weapon.id) : null,
paint: paint?.value ?? null,
ka: statValues[0] ?? null,
d: statValues[1] ?? null,
s: statValues[2] ?? null,
},
debug: {
weapon,
special,
paintScore: paint?.confidence ?? 0,
nameScore: name?.confidence ?? 0,
statScores,
povFraction: povYellowFraction(rgb, rois.povArrow(cy)),
},
};
}

252
app/features/cv/core/detectors/scoreboard/specials.ts Normal file
View File

@@ -0,0 +1,252 @@
/**
* Kit-icon identification: the team-tinted sub/special tiles next to a
* player's main weapon (results scoreboard: the special icon above the
* specials counter; minimap cards/rows: the sub-weapon tile).
*
* The tiles render the same art in each team's ink hue, so unlike weapon
* icons the match is shape-only: the template's alpha silhouette against
* the binarized search region, scored with NCC plus the same ink-coverage
* penalty matchWeapon uses.
*
* The icon itself carries little pixel budget (~22px specials, ~30px
* minimap subs), but it only ever needs to split main weapons whose
* *icons* are near-ties (Splash- vs Sploosh-o-matic, plain vs Custom
* Dualie Squelchers) — and near-tie icon twins ship with different kits,
* whose silhouettes (stamp vs crab, bomb vs beakon) are far apart.
*/
import { getCV, type Mat, minMaxLoc } from "../../cv";
import type { FrameData } from "../../image";
import { WEAPON_KITS } from "./kits";
import type { WeaponMatch } from "./weapons";
/** Icon heights (px at 1080p) to try; the row renders it at ~22px. */
const SPECIAL_TEMPLATE_SIZES = [19, 21, 23, 25] as const;
/** Colored icon ink vs the near-black pill, on max(r,g,b). */
const SPECIAL_INK_THRESHOLD = 48;
export interface SpecialTemplate {
id: string;
/** binary silhouette + ink pixel count at each templateSizes entry */
sizes: { mat: Mat; ink: number }[];
}
export interface SpecialMatch {
id: string;
score: number;
top: { id: string; score: number }[];
}
/**
* Build binary silhouette templates from raw RGBA icon images (128x128 with
* alpha): tight-crop the alpha extent, binarize, downscale to each height.
* `templateSizes` defaults to the scoreboard's special-icon sizes; the
* minimap builds its sub-weapon set at the sub tile's larger sizes.
*/
export function prepareSpecialTemplates(
icons: { id: string; image: FrameData }[],
templateSizes: readonly number[] = SPECIAL_TEMPLATE_SIZES,
): SpecialTemplate[] {
const cv = getCV();
return icons.map(({ id, image }) => {
const { width, height, data } = image;
let xMin = width;
let xMax = -1;
let yMin = height;
let yMax = -1;
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
if (data[(y * width + x) * 4 + 3]! > 128) {
if (x < xMin) xMin = x;
if (x > xMax) xMax = x;
if (y < yMin) yMin = y;
if (y > yMax) yMax = y;
}
}
}
const w = Math.max(1, xMax - xMin + 1);
const h = Math.max(1, yMax - yMin + 1);
const silhouette = new cv.Mat(h, w, cv.CV_8UC1, new cv.Scalar(0));
const dst = silhouette.data;
for (let y = 0; y < h; y++) {
for (let x = 0; x < w; x++) {
if (data[((y + yMin) * width + x + xMin) * 4 + 3]! > 128) {
dst[y * w + x] = 255;
}
}
}
const sizes = templateSizes.map((size) => {
const scale = size / Math.max(w, h);
const resized = new cv.Mat();
cv.resize(
silhouette,
resized,
new cv.Size(Math.max(1, Math.round(w * scale)), Math.max(1, Math.round(h * scale))),
0,
0,
cv.INTER_AREA,
);
// re-binarize the interpolated edges so template and search region
// live on the same two-level scale
const mat = new cv.Mat();
cv.threshold(resized, mat, 127, 255, cv.THRESH_BINARY);
resized.delete();
let ink = 0;
for (const v of mat.data) if (v > 0) ink++;
return { mat, ink };
});
silhouette.delete();
return { id, sizes };
});
}
/**
* searchRgb: RGB crop of the row's special-icon ROI (view is fine).
* The region is binarized on max(r,g,b) so any ink tint reads as shape.
*/
export function matchSpecial(searchRgb: Mat, templates: SpecialTemplate[]): SpecialMatch {
const cv = getCV();
// binarized copy of the search region (pixel access needs a copy)
const cont = new cv.Mat();
searchRgb.copyTo(cont);
const { rows, cols } = cont;
const binary = new cv.Mat(rows, cols, cv.CV_8UC1, new cv.Scalar(0));
const src = cont.data;
const dst = binary.data;
let searchInk = 0;
for (let i = 0; i < rows * cols; i++) {
const v = Math.max(src[i * 3]!, src[i * 3 + 1]!, src[i * 3 + 2]!);
if (v > SPECIAL_INK_THRESHOLD) {
dst[i] = 255;
searchInk++;
}
}
cont.delete();
const result = new cv.Mat();
const ranked: { id: string; score: number }[] = [];
for (const template of templates) {
let score = -1;
for (const { mat, ink } of template.sizes) {
if (mat.rows > binary.rows || mat.cols > binary.cols) continue;
cv.matchTemplate(binary, mat, result, cv.TM_CCOEFF_NORMED);
const { maxVal } = minMaxLoc(result);
const r = Math.min(ink, searchInk) / Math.max(Math.max(ink, searchInk), 1);
const adjusted = maxVal * (0.75 + 0.25 * r);
if (adjusted > score) score = adjusted;
}
ranked.push({ id: template.id, score });
}
result.delete();
binary.delete();
ranked.sort((a, b) => b.score - a.score);
return { id: ranked[0]?.id ?? "unknown", score: ranked[0]?.score ?? -1, top: ranked };
}
/** Weapon-icon score gap under which two candidates count as a tie. */
const WEAPON_TIE_MARGIN = 0.04;
/**
* Icon twins whose art differs only by the nozzle (Splash- vs
* Sploosh-o-matic share the body, gauge and handle). On the replay
* browser's dim, drop-shadowed rendering the wrong twin can beat the right
* one by far more than WEAPON_TIE_MARGIN (0.15 observed on the
* brinewater-1411 fixture), so whenever both appear among the top
* candidates the kit evidence is consulted regardless of the icon-score
* gap — the KIT_DECISION_MARGIN still guards the actual re-rank.
* Symmetric pairs, extended per attested fixture need only.
*/
const ICON_TWINS: ReadonlyMap<string, string> = new Map([
["0", "20"], // Sploosh-o-matic <-> Splash-o-matic
["20", "0"],
]);
/**
* The kit-icon evidence must separate the tied kits' entries by at least
* this much to override the icon ranking: the shape matcher's own confusions
* (Wave Breaker vs Ink Vac/Reef Slider on ~22px of silhouette) all land
* inside this band, while true silhouette splits (stamp vs crab) clear it.
*/
const KIT_DECISION_MARGIN = 0.06;
type KitPart = "sub" | "special";
/**
* Weapon-icon candidates tied within WEAPON_TIE_MARGIN (plus the leader's
* ICON_TWIN at any gap) whose kits carry different subs/specials — the only
* case the kit icon can help with. Returns null when the icon match already
* decided (or the kit part can't).
*/
function tiedWeaponsWithDistinctKit(
match: WeaponMatch,
part: KitPart,
): { id: string; score: number }[] | null {
const leader = match.top[0];
if (!leader) return null;
const tied = match.top.filter((t) => leader.score - t.score < WEAPON_TIE_MARGIN);
const twinId = ICON_TWINS.get(leader.id);
if (twinId && !tied.some((t) => t.id === twinId)) {
const twin = match.top.find((t) => t.id === twinId);
if (twin) tied.push(twin);
}
if (tied.length < 2) return null;
const parts = new Set(
tied.map((t) => WEAPON_KITS.get(t.id)?.[part]).filter((s) => s !== undefined),
);
return parts.size >= 2 ? tied : null;
}
export function tiedWeaponsWithDistinctSpecials(
match: WeaponMatch,
): { id: string; score: number }[] | null {
return tiedWeaponsWithDistinctKit(match, "special");
}
export function tiedWeaponsWithDistinctSubs(
match: WeaponMatch,
): { id: string; score: number }[] | null {
return tiedWeaponsWithDistinctKit(match, "sub");
}
/**
* Re-rank tied weapon candidates by how well their kits' sub/special
* matches the tile evidence. Only reorders on decisive evidence; ties whose
* kit entries all score alike keep the icon-match order.
*/
function disambiguateWeaponByKit(
match: WeaponMatch,
evidence: SpecialMatch,
part: KitPart,
): WeaponMatch {
const tied = tiedWeaponsWithDistinctKit(match, part);
if (!tied) return match;
// worst-possible floor: adjusted NCC never drops below -1
const kitScore = (weaponId: string): number => {
const kit = WEAPON_KITS.get(weaponId);
if (!kit) return -1;
return evidence.top.find((t) => t.id === String(kit[part]))?.score ?? -1;
};
const ranked = [...tied].sort((a, b) => kitScore(b.id) - kitScore(a.id));
const winner = ranked[0]!;
if (winner.id === match.id) return match;
const runnerUp = ranked.find((t) => kitScore(t.id) < kitScore(winner.id));
if (!runnerUp || kitScore(winner.id) - kitScore(runnerUp.id) < KIT_DECISION_MARGIN) {
return match;
}
const top = [winner, ...match.top.filter((t) => t.id !== winner.id)].slice(0, 3);
const flag: Partial<WeaponMatch> =
part === "special" ? { specialResolved: true } : { subResolved: true };
return { ...match, id: winner.id, score: winner.score, top, ...flag };
}
export function disambiguateWeaponBySpecial(
match: WeaponMatch,
special: SpecialMatch,
): WeaponMatch {
return disambiguateWeaponByKit(match, special, "special");
}
export function disambiguateWeaponBySub(match: WeaponMatch, sub: SpecialMatch): WeaponMatch {
return disambiguateWeaponByKit(match, sub, "sub");
}

352
app/features/cv/core/detectors/scoreboard/weapons.ts Normal file
View File

@@ -0,0 +1,352 @@
/**
* Weapon icon identification: NCC of every candidate icon (pre-scaled to a
* few plausible sizes) against the row's weapon ROI.
*
* Raw sliding NCC lets a small wrong template win by matching a lucky
* sub-window of a bigger icon, so scores carry an ink-coverage penalty:
* the template's opaque area should account for the icon pixels actually
* present in the search region.
*
* Large template sets go through a coarse-to-fine pass: every icon is first
* ranked at quarter resolution (same NCC + ink-coverage scoring), then only
* the shortlist is re-matched at full resolution. Small sets (the ability
* badges) skip the coarse pass — with few templates the extra calls cost
* more than they save.
*/
import { getCV, type Mat, minMaxLoc } from "../../cv";
import type { FrameData } from "../../image";
/**
* Icon heights (px at 1080p) to try. The live scoreboard renders in-row
* icons at ~44-56px; the replay browser at ~60-64px. The oversized entries
* are skipped inside the live detector's 56px-tall weapon ROI at match
* time, so they only ever compete on the replay screen.
*/
const WEAPON_TEMPLATE_SIZES = [40, 44, 48, 52, 56, 60, 64] as const;
/** Row pill background the icons sit on (near-black). */
const PILL_BACKGROUND = 12;
/** Pixels brighter than this count as icon ink (pill is ~10-15). */
const INK_THRESHOLD = 40;
/**
* Scoped chargers' icons differ from their unscoped twins only by the scope
* tube, which is near-black and sits on the near-black pill — measurably
* invisible at ~50px (a synthetic scoped row probes *darker* in the scope
* band than a real unscoped row does from barrel bleed). When the two
* variants score within noise of each other, resolve to the unscoped one
* (matches labeled fixtures and typical usage) and flag the ambiguity.
* Map: scoped id -> unscoped id.
*/
const SCOPED_TWINS: ReadonlyMap<string, string> = new Map([
["2040", "2030"], // Splat Scope -> Splat Charger
["2041", "2031"], // Z+F variants
["2070", "2060"], // E-liter 4K Scope -> E-liter 4K
["2071", "2061"], // Custom variants
]);
const TWIN_MARGIN = 0.05;
/** Coarse-pass resolution, relative to full templates. */
const COARSE_SCALE = 0.25;
/** How many coarse-ranked ids survive into the full-resolution pass. */
const COARSE_SHORTLIST = 16;
export interface TemplateSize {
mat: Mat;
ink: number;
/** the same template at COARSE_SCALE, for the coarse ranking pass */
coarse: { mat: Mat; ink: number };
}
export interface WeaponTemplate {
id: string;
/** RGB template + ink pixel count at each candidate size */
sizes: TemplateSize[];
}
export interface WeaponMatch {
id: string;
score: number;
top: { id: string; score: number }[];
/** true when a scoped/unscoped twin tie was resolved by the unscoped prior */
twinAmbiguous?: boolean;
/** true when a near-tie was re-ranked by the row's special icon (specials.ts) */
specialResolved?: boolean;
/** true when a near-tie was re-ranked by the minimap sub tile (specials.ts) */
subResolved?: boolean;
}
/** Icon-ink pixel count of a continuous RGB mat (NOT an ROI view). */
function countInkRgb(mat: Mat, threshold: number): number {
let ink = 0;
const d = mat.data;
const n = mat.rows * mat.cols;
for (let i = 0; i < n; i++) {
const v = Math.max(d[i * 3]!, d[i * 3 + 1]!, d[i * 3 + 2]!);
if (v > threshold) ink++;
}
return ink;
}
/**
* Downscale a composited RGB icon to each candidate size, with the coarse
* variant and ink counts matchWeapon needs. Ink is counted on the resized
* pixels exactly the way matchWeapon measures the search region: scaling the
* source-resolution alpha count undercounts the antialiased edges that the
* screen capture (and this resize) light up, and that skew alone can flip
* the coverage ratio toward a wrong icon.
*
* `referenceSize` switches the meaning of each size entry: instead of
* resizing `composited` to a size×size square, it is scaled by
* size/referenceSize — i.e. "the art as it renders inside a size-sized
* padded square", where `composited` is a crop out of a referenceSize
* square. That lets a template whose *icon square* is taller than the ROI
* still compete when its art fits (see prepareWeaponTemplates cropToArt).
*/
export function buildTemplateSizes(
composited: Mat,
sizes: readonly number[],
inkThreshold: number,
referenceSize?: number,
): TemplateSize[] {
const cv = getCV();
return sizes.map((size) => {
const mat = new cv.Mat();
if (referenceSize) {
const scale = size / referenceSize;
cv.resize(
composited,
mat,
new cv.Size(
Math.max(1, Math.round(composited.cols * scale)),
Math.max(1, Math.round(composited.rows * scale)),
),
0,
0,
cv.INTER_AREA,
);
} else {
cv.resize(composited, mat, new cv.Size(size, size), 0, 0, cv.INTER_AREA);
}
const coarseMat = new cv.Mat();
cv.resize(mat, coarseMat, new cv.Size(0, 0), COARSE_SCALE, COARSE_SCALE, cv.INTER_AREA);
return {
mat,
ink: countInkRgb(mat, inkThreshold),
coarse: { mat: coarseMat, ink: countInkRgb(coarseMat, inkThreshold) },
};
});
}
/**
* Build match-ready templates from raw RGBA icon images (256x256 with alpha):
* composite over the pill background, then downscale to each candidate size.
* `templateSizes` defaults to the scoreboard row sizes; the death detector
* builds a second set at burst-icon size from the same images. Screens whose
* pills aren't near-black (the minimap's cards and special-ready camo
* surfaces) override `background` — and with it `inkThreshold`, which must
* clear the new background or the template ink counts saturate.
*
* `cropToArt` trims each template to the icon's alpha bounding box while
* keeping the padded-square scale semantics (each size still means "drawn
* inside a size-sized square"). The source icons carry generous transparent
* padding, and matchWeapon skips any template taller/wider than the ROI —
* so without the trim, a screen that renders icons near the ROI height
* (the minimap cards render at the equivalent of ~56-60px squares inside a
* 54px-tall box) can never match at the size actually on screen.
*/
export function prepareWeaponTemplates(
icons: { id: string; image: FrameData }[],
templateSizes: readonly number[] = WEAPON_TEMPLATE_SIZES,
options: { background?: number; inkThreshold?: number; cropToArt?: boolean } = {},
): WeaponTemplate[] {
const cv = getCV();
const background = options.background ?? PILL_BACKGROUND;
const inkThreshold = options.inkThreshold ?? INK_THRESHOLD;
return icons.map(({ id, image }) => {
const rgba = cv.matFromImageData(image as unknown as ImageData);
const source = options.cropToArt ? cropToAlphaBbox(rgba) : rgba;
const composited = compositeOnBackground(source, background);
if (source !== rgba) source.delete();
const referenceSize = options.cropToArt ? rgba.cols : undefined;
rgba.delete();
const sizes = buildTemplateSizes(composited, templateSizes, inkThreshold, referenceSize);
composited.delete();
return { id, sizes };
});
}
/** Alpha threshold and margin (source px) for the cropToArt bounding box. */
const ART_ALPHA_MIN = 32;
const ART_BBOX_MARGIN = 6;
function cropToAlphaBbox(rgba: Mat): Mat {
const cv = getCV();
const d = rgba.data;
const cols = rgba.cols;
const rows = rgba.rows;
let x0 = cols,
y0 = rows,
x1 = -1,
y1 = -1;
for (let y = 0; y < rows; y++) {
const rowBase = y * cols * 4 + 3;
for (let x = 0; x < cols; x++) {
if (d[rowBase + x * 4]! >= ART_ALPHA_MIN) {
if (x < x0) x0 = x;
if (x > x1) x1 = x;
if (y < y0) y0 = y;
if (y > y1) y1 = y;
}
}
}
if (x1 < 0) {
const copy = new cv.Mat();
rgba.copyTo(copy);
return copy;
}
x0 = Math.max(0, x0 - ART_BBOX_MARGIN);
y0 = Math.max(0, y0 - ART_BBOX_MARGIN);
x1 = Math.min(cols - 1, x1 + ART_BBOX_MARGIN);
y1 = Math.min(rows - 1, y1 + ART_BBOX_MARGIN);
const view = rgba.roi(new cv.Rect(x0, y0, x1 - x0 + 1, y1 - y0 + 1));
const out = new cv.Mat();
view.copyTo(out); // ROI views: .data/.clone broken in this build, copy out
view.delete();
return out;
}
function compositeOnBackground(rgba: Mat, background: number): Mat {
const cv = getCV();
const out = new cv.Mat(rgba.rows, rgba.cols, cv.CV_8UC3);
const src = rgba.data;
const dst = out.data;
const n = rgba.rows * rgba.cols;
for (let i = 0; i < n; i++) {
const a = src[i * 4 + 3]! / 255;
dst[i * 3] = Math.round(src[i * 4]! * a + background * (1 - a));
dst[i * 3 + 1] = Math.round(src[i * 4 + 1]! * a + background * (1 - a));
dst[i * 3 + 2] = Math.round(src[i * 4 + 2]! * a + background * (1 - a));
}
return out;
}
/**
* Coarse pass: rank every template by its best quarter-resolution score
* (same NCC + ink-coverage formula as the full pass) and return the
* COARSE_SHORTLIST ids worth full-resolution matching. Scoped ids drag
* their unscoped twin along so the twin tie-break always has both scores.
* Returns null when no coarse template fits (caller falls back to the
* full set, which then skips the same templates and reports unknown).
*/
function coarseShortlist(
searchRgb: Mat,
templates: WeaponTemplate[],
inkThreshold: number,
): Set<string> | null {
const cv = getCV();
const region = new cv.Mat();
cv.resize(searchRgb, region, new cv.Size(0, 0), COARSE_SCALE, COARSE_SCALE, cv.INTER_AREA);
const searchInk = countInkRgb(region, inkThreshold);
const result = new cv.Mat();
const scored: { id: string; score: number }[] = [];
const searchRows = searchRgb.rows;
const searchCols = searchRgb.cols;
const regionRows = region.rows;
const regionCols = region.cols;
for (const template of templates) {
let score = -1;
for (const { mat, coarse } of template.sizes) {
// gate on the *full-res* dims so a size competes here iff it competes
// in the full pass, then also require the coarse pair to fit
if (mat.rows > searchRows || mat.cols > searchCols) continue;
if (coarse.mat.rows > regionRows || coarse.mat.cols > regionCols) continue;
cv.matchTemplate(region, coarse.mat, result, cv.TM_CCOEFF_NORMED);
const { maxVal } = minMaxLoc(result);
const r = Math.min(coarse.ink, searchInk) / Math.max(Math.max(coarse.ink, searchInk), 1);
const adjusted = maxVal * (0.75 + 0.25 * r);
if (adjusted > score) score = adjusted;
}
if (score > -1) scored.push({ id: template.id, score });
}
result.delete();
region.delete();
if (scored.length === 0) return null;
scored.sort((a, b) => b.score - a.score);
const ids = new Set(scored.slice(0, COARSE_SHORTLIST).map((s) => s.id));
for (const id of [...ids]) {
const twin = SCOPED_TWINS.get(id);
if (twin) ids.add(twin);
}
return ids;
}
/**
* searchRgb: RGB crop of the row's weapon ROI (view is fine).
* inkThreshold separates icon ink from the pill background in the search
* region — raise it on screens whose pills are lighter than the live
* scoreboard's (~12), e.g. the replay browser (~61), or everything counts
* as ink and the coverage penalty collapses.
*/
export function matchWeapon(
searchRgb: Mat,
templates: WeaponTemplate[],
options: { inkThreshold?: number } = {},
): WeaponMatch {
const cv = getCV();
const inkThreshold = options.inkThreshold ?? INK_THRESHOLD;
// icon ink present in the search region (pixel access needs a copy)
const cont = new cv.Mat();
searchRgb.copyTo(cont);
const searchInk = countInkRgb(cont, inkThreshold);
cont.delete();
// shortlist large sets at coarse resolution first; below ~2x the shortlist
// size the coarse pass costs more calls than it saves
let pool = templates;
if (templates.length > COARSE_SHORTLIST * 2) {
const ids = coarseShortlist(searchRgb, templates, inkThreshold);
if (ids) pool = templates.filter((t) => ids.has(t.id));
}
const result = new cv.Mat();
const best = new Map<string, number>();
const searchRows = searchRgb.rows;
const searchCols = searchRgb.cols;
for (const template of pool) {
let score = -1;
for (const { mat, ink } of template.sizes) {
if (mat.rows > searchRows || mat.cols > searchCols) continue;
cv.matchTemplate(searchRgb, mat, result, cv.TM_CCOEFF_NORMED);
const { maxVal } = minMaxLoc(result);
const r = Math.min(ink, searchInk) / Math.max(Math.max(ink, searchInk), 1);
const adjusted = maxVal * (0.75 + 0.25 * r);
if (adjusted > score) score = adjusted;
}
best.set(template.id, score);
}
result.delete();
const ranked = [...best.entries()]
.map(([id, score]) => ({ id, score }))
.sort((a, b) => b.score - a.score);
const top = ranked.slice(0, 3);
let first = top[0] ?? { id: "unknown", score: -1 };
let twinAmbiguous = false;
const unscopedId = SCOPED_TWINS.get(first.id);
if (unscopedId) {
const twin = ranked.find((r) => r.id === unscopedId);
if (twin && first.score - twin.score < TWIN_MARGIN) {
twinAmbiguous = true;
first = twin;
const i = top.findIndex((t) => t.id === twin.id);
if (i >= 0) top.splice(i, 1);
top.unshift(twin);
top.length = Math.min(top.length, 3);
}
}
return { id: first.id, score: first.score, top, twinAmbiguous };
}

72
app/features/cv/core/detectors/suppressor.ts Normal file
View File

@@ -0,0 +1,72 @@
/**
* ParseSuppressor: bails out of re-parsing a static screen.
*
* A screen that keeps a detector's gate firing (the results scoreboard can
* sit for tens of seconds, a paused VoD indefinitely) makes the pipeline
* re-run the expensive parse() on every sampled frame even though nothing
* changes. Per detector this tracks the best event confidence seen during a
* continuous gate-pass streak; once `maxStagnantParses` consecutive parses
* fail to improve on it, parse() is skipped (the cheap gate keeps running)
* until the gate drops — i.e. the screen actually changed.
*/
export interface SuppressorOptions {
/** consecutive non-improving parses tolerated before suppression kicks in */
maxStagnantParses: number;
/** minimum confidence gain that counts as an improvement */
minImprovement: number;
}
const DEFAULT_SUPPRESSOR_OPTIONS: SuppressorOptions = {
// at the 2fps sample rate: give a stable screen ~3s to produce its best
// read, then stop paying for parses until the screen changes
maxStagnantParses: 6,
minImprovement: 0.001,
};
interface StreakState {
best: number;
stagnant: number;
suppressed: boolean;
}
export class ParseSuppressor {
#options: SuppressorOptions;
#streaks = new Map<string, StreakState>();
constructor(options: Partial<SuppressorOptions> = {}) {
this.#options = { ...DEFAULT_SUPPRESSOR_OPTIONS, ...options };
}
/**
* Call once per detector per frame with the gate outcome; returns whether
* parse() should run. A failed gate ends the streak, so the next gate pass
* starts a fresh, unsuppressed streak.
*/
shouldParse(detectorId: string, gatePass: boolean): boolean {
if (!gatePass) {
this.#streaks.delete(detectorId);
return false;
}
return !this.#streaks.get(detectorId)?.suppressed;
}
/** Report the outcome of a parse this suppressor approved. */
recordParse(detectorId: string, events: readonly { confidence: number }[]): void {
// no events counts as confidence 0: a false-firing gate on a static
// screen stagnates and gets suppressed just like a parsed one
const confidence = events.reduce((max, e) => Math.max(max, e.confidence), 0);
const streak = this.#streaks.get(detectorId);
if (!streak) {
this.#streaks.set(detectorId, { best: confidence, stagnant: 0, suppressed: false });
return;
}
if (confidence > streak.best + this.#options.minImprovement) {
streak.best = confidence;
streak.stagnant = 0;
return;
}
streak.stagnant += 1;
if (streak.stagnant >= this.#options.maxStagnantParses) streak.suppressed = true;
}
}

37
app/features/cv/core/detectors/types.ts Normal file
View File

@@ -0,0 +1,37 @@
import type { Mat } from "../cv";
export interface DetectedEvent<TData = unknown> {
type: string;
/** seconds into the stream/video */
t: number;
/** 0..1 aggregate confidence; TimelineBuilder drops events below threshold */
confidence: number;
data: TData;
/** per-field match scores etc. for the harness/debugging; not persisted upstream */
debug?: Record<string, unknown>;
}
export interface GateResult {
pass: boolean;
/** raw score of the gate check, for tuning */
score: number;
/**
* which screen variant the gate recognized, for detectors that gate more
* than one (minimap: "overlay" | "spectator") — parse() branches on it
* instead of re-running the gate probes
*/
variant?: string;
}
/**
* A detector recognizes one event type on a canonical 1920x1080 RGBA frame.
* gate() must be cheap (probes / tiny template match); parse() may be expensive
* and only runs when gate() passes. Callers pass the gate result they already
* computed into parse(); a detector must still cope without it (re-deriving
* whatever it needs) so one-shot tools can call parse() directly.
*/
export interface Detector<TData = unknown> {
id: string;
gate(frame: Mat): GateResult;
parse(frame: Mat, t: number, gate?: GateResult): DetectedEvent<TData>[];
}

583
app/features/cv/core/glyphs.ts Normal file
View File

@@ -0,0 +1,583 @@
/**
* Glyph-template text recognition. Font, size, and position are known, so
* instead of general OCR we segment the text crop by column projection and
* classify each segment with sliding NCC against grayscale glyph templates.
*
* Three details matter for accuracy:
* - matching runs on the *masked* grayscale crop (background zeroed via a
* dilated binary mask) so colored backgrounds (team score boxes) behave
* like the black pills, while antialiased glyph edges survive;
* - scores carry an ink-coverage penalty, otherwise a narrow template
* sliding inside a wider glyph wins by ignoring ink it doesn't cover
* ('c' outscoring 'o' on an actual 'o');
* - segments much wider than the set's median glyph get split at deep
* projection dips ('T' overhangs the next letter, merging segments);
* - multi-stroke glyphs with a column gap between strokes (katakana パ, ハ,
* リ) segment as separate fragments no full-glyph template can match, so
* close segment pairs are re-classified merged and the merged read wins
* when it decisively outscores both fragments.
*
* Templates are matched at native scale: callers matching larger text
* (team scores) pre-scale the glyph set with scaleGlyphSet.
*/
import { getCV, type Mat } from "./cv";
import type { FrameData } from "./image";
interface AtlasGlyphMeta {
char: string;
x: number;
y: number;
w: number;
h: number;
/**
* Where the template came from. Atlases are hybrids: "fixture" glyphs are
* exact pixel crops from labeled frames (highest fidelity, sparse
* coverage), "font" glyphs are rendered from the game fonts (full
* coverage, slightly off in-game rendering). Recognition scores every
* glyph and takes the max, so fixture glyphs win where they exist.
*/
source?: "fixture" | "font";
}
export interface AtlasMeta {
/** nominal glyph height in canonical-1080p pixels */
height: number;
glyphs: AtlasGlyphMeta[];
}
interface Glyph {
char: string;
/** grayscale white-on-black, tight box */
mat: Mat;
/** count of pixels above the binarization threshold */
ink: number;
/** exact fixture crop vs font-rendered approximation */
source: "fixture" | "font";
}
export interface GlyphSet {
glyphs: Glyph[];
height: number;
/** median tight-box glyph width, used for split/space heuristics */
medianWidth: number;
}
const TEMPLATE_BIN_THRESHOLD = 128;
/** Slice glyph templates out of an atlas image using its metadata. */
export function loadGlyphSet(atlas: FrameData, meta: AtlasMeta): GlyphSet {
const cv = getCV();
const full = cv.matFromImageData(atlas as unknown as ImageData);
const gray = new cv.Mat();
cv.cvtColor(full, gray, cv.COLOR_RGBA2GRAY);
full.delete();
const glyphs: Glyph[] = meta.glyphs.map((g) => {
// NB: .data/.clone() are broken on ROI views in this opencv.js build;
// always copyTo into a fresh mat before pixel access.
const view = gray.roi(new cv.Rect(g.x, g.y, g.w, g.h));
const cell = new cv.Mat();
view.copyTo(cell);
view.delete();
const mat = tightCropGray(cell, TEMPLATE_BIN_THRESHOLD);
cell.delete();
let ink = 0;
for (const v of mat.data) if (v > TEMPLATE_BIN_THRESHOLD) ink++;
// untagged glyphs predate hybrid atlases and were all fixture crops
return { char: g.char, mat, ink, source: g.source ?? "fixture" };
});
gray.delete();
const widths = glyphs.map((g) => g.mat.cols).sort((a, b) => a - b);
const medianWidth = widths[Math.floor(widths.length / 2)] ?? 8;
return { glyphs, height: meta.height, medianWidth };
}
/** Resize every glyph by `factor` (e.g. to reuse paint digits for team scores). */
export function scaleGlyphSet(set: GlyphSet, factor: number): GlyphSet {
const cv = getCV();
const glyphs = set.glyphs.map((g) => {
const mat = new cv.Mat();
cv.resize(g.mat, mat, new cv.Size(0, 0), factor, factor, cv.INTER_CUBIC);
let ink = 0;
for (const v of mat.data) if (v > TEMPLATE_BIN_THRESHOLD) ink++;
return { char: g.char, mat, ink, source: g.source };
});
return {
glyphs,
height: Math.round(set.height * factor),
medianWidth: Math.round(set.medianWidth * factor),
};
}
/** Crop a grayscale mat to the tight bounds of its above-threshold pixels. */
function tightCropGray(gray: Mat, threshold: number): Mat {
const cv = getCV();
const { cols, rows, data } = gray;
let xMin = cols;
let xMax = -1;
let yMin = rows;
let yMax = -1;
for (let y = 0; y < rows; y++) {
for (let x = 0; x < cols; x++) {
if (data[y * cols + x]! > threshold) {
if (x < xMin) xMin = x;
if (x > xMax) xMax = x;
if (y < yMin) yMin = y;
if (y > yMax) yMax = y;
}
}
}
const out = new cv.Mat();
if (xMax < 0) {
gray.copyTo(out);
return out;
}
const view = gray.roi(new cv.Rect(xMin, yMin, xMax - xMin + 1, yMax - yMin + 1));
view.copyTo(out);
view.delete();
return out;
}
export interface RecognizedChar {
char: string;
score: number;
/** segment bounds relative to the crop */
x0: number;
x1: number;
/** segment ink extent relative to the crop (y1 exclusive), for baseline checks */
y0: number;
y1: number;
/** runner-up candidates for this segment (debugging/tuning aid) */
candidates?: {
char: string;
score: number;
ncc: number;
source: "fixture" | "font";
}[];
}
export interface RecognizedText {
text: string;
chars: RecognizedChar[];
/** min char score (0 when ink was present but nothing was recognized) */
confidence: number;
}
export interface RecognizeOptions {
/** binarization threshold for segmentation/masking (white text) */
binThreshold?: number;
/** min white pixels for a column to count as ink */
minColumnPixels?: number;
/** gaps wider than this emit a space; Infinity disables spaces */
spaceGap?: number;
/** discard chars scoring below this */
minCharScore?: number;
}
interface Segment {
x0: number;
x1: number;
}
function columnProfile(binary: Mat): number[] {
const { cols, rows, data } = binary;
const profile = new Array<number>(cols).fill(0);
for (let y = 0; y < rows; y++) {
for (let x = 0; x < cols; x++) {
if (data[y * cols + x]! > 0) profile[x]!++;
}
}
return profile;
}
function segmentColumns(profile: number[], minColumnPixels: number): Segment[] {
const segments: Segment[] = [];
let start = -1;
for (let x = 0; x < profile.length; x++) {
const on = profile[x]! > minColumnPixels;
if (on && start < 0) start = x;
if (!on && start >= 0) {
if (x - start >= 2) segments.push({ x0: start, x1: x });
start = -1;
}
}
if (start >= 0 && profile.length - start >= 2) {
segments.push({ x0: start, x1: profile.length });
}
return segments;
}
/** Distance from `w` to the nearest positive multiple of `unit`. */
function widthError(w: number, unit: number): number {
const n = Math.max(1, Math.round(w / unit));
return Math.abs(w - n * unit);
}
function splitWideSegment(profile: number[], seg: Segment, medianWidth: number): Segment[] {
const maxCharWidth = Math.round(medianWidth * 1.5);
if (seg.x1 - seg.x0 <= maxCharWidth) return [seg];
// Pick the split column by dip depth *relative to the peaks on both sides*,
// not by raw minimum: the raw minimum can land inside a thin-but-real stroke
// (a 'T' top bar profiles as low as the blurred gap to the next glyph),
// while the true boundary is the column lowest relative to its neighbours.
const leftMax = new Array<number>(seg.x1).fill(0);
const rightMax = new Array<number>(seg.x1).fill(0);
let running = 0;
for (let i = seg.x0; i < seg.x1; i++) {
leftMax[i] = running;
running = Math.max(running, profile[i]!);
}
running = 0;
for (let i = seg.x1 - 1; i >= seg.x0; i--) {
rightMax[i] = running;
running = Math.max(running, profile[i]!);
}
let best = -1;
let bestRatio = Infinity;
for (let i = seg.x0 + 3; i < seg.x1 - 3; i++) {
const ratio = profile[i]! / Math.max(1, Math.min(leftMax[i]!, rightMax[i]!));
if (ratio < bestRatio) {
bestRatio = ratio;
best = i;
}
}
if (best < 0 || bestRatio > 0.45) return [seg];
// The dip is often a plateau of equally-low columns (a 'T' arm blurring
// into the next glyph profiles no higher than the gap itself), and the
// profile cannot say which glyph the plateau's ink belongs to. Cut at the
// plateau edge that leaves the left part closest to whole glyph widths.
let lo = best;
let hi = best;
while (lo - 1 >= seg.x0 + 3 && profile[lo - 1]! <= profile[best]!) lo--;
while (hi + 1 < seg.x1 - 3 && profile[hi + 1]! <= profile[best]!) hi++;
const cut =
widthError(lo - seg.x0, medianWidth) <= widthError(hi + 1 - seg.x0, medianWidth) ? lo : hi + 1;
return [
...splitWideSegment(profile, { x0: seg.x0, x1: cut }, medianWidth),
...splitWideSegment(profile, { x0: cut, x1: seg.x1 }, medianWidth),
];
}
interface SegmentInfo extends Segment {
ink: number;
height: number;
/** ink extent rows (y1 exclusive) */
y0: number;
y1: number;
}
function measureSegment(binary: Mat, seg: Segment): SegmentInfo {
const { cols, data, rows } = binary;
let ink = 0;
let yMin = rows;
let yMax = -1;
for (let y = 0; y < rows; y++) {
for (let x = seg.x0; x < seg.x1; x++) {
if (data[y * cols + x]! > 0) {
ink++;
if (y < yMin) yMin = y;
if (y > yMax) yMax = y;
}
}
}
return {
...seg,
ink,
height: Math.max(0, yMax - yMin + 1),
y0: Math.min(yMin, rows),
y1: yMax + 1,
};
}
/**
* Fixture-crop templates are ground-truth pixels; a font-rendered glyph must
* beat them by at least this much to win (stops near-duplicate font glyphs
* like 'I' edging out an exact 'l' crop on noise). Keep this tight: for
* near-duplicate shapes the gap is tiny, while at 0.04 a fixture crop of a
* genuinely different char ('h' vs 'b') can displace a correct font match.
*/
const FIXTURE_TIEBREAK = 0.02;
function classifySegment(
masked: Mat,
seg: SegmentInfo,
set: GlyphSet,
/**
* Exact early-reject floor: a score can never exceed its precomputed
* bound, so a caller that only needs to know whether any glyph can beat
* `scoreFloor` (mergeSplitGlyphs) lets the bound-sorted loop stop as soon
* as none can. Scores that ARE computed come out identical — but the
* returned list omits sub-floor candidates, so pass it only for probes.
*/
scoreFloor = -Infinity,
): { char: string; score: number; ncc: number; source: "fixture" | "font" }[] {
const cv = getCV();
const segWidth = seg.x1 - seg.x0;
const pad = 5;
// NB: Mat dimension/data accessors go through embind (validateThis etc.)
// and are expensive — hoist them out of the per-glyph/per-pixel loops.
const maskedRows = masked.rows;
const x0 = Math.max(0, seg.x0 - pad);
const x1 = Math.min(masked.cols, seg.x1 + pad);
const regionCols = x1 - x0;
// Crop the search region vertically to the segment's ink rows: line crops
// can run far taller than the text (the death tag band is ~2x its glyphs),
// and every extra row multiplies matchTemplate work while only offering
// placements away from the ink no correct match can occupy. The slack
// keeps every hRatio-eligible template (tRows <= 1.3 * seg.height)
// placeable: seg.height + 2 * (0.15 * seg.height + 2) >= 1.3 * seg.height.
const vSlack = Math.ceil(0.15 * seg.height) + 2;
const y0 = Math.max(0, seg.y0 - vSlack);
const y1 = Math.min(maskedRows, seg.y1 + vSlack);
const regionRows = y1 - y0;
const region = masked.roi(new cv.Rect(x0, y0, regionCols, regionRows));
// Both penalty factors below depend only on the glyph and the segment, and
// NCC <= 1, so their product bounds the score a glyph can reach before any
// matching runs. Matching in descending-bound order lets the loop stop as
// soon as no remaining glyph could come within FIXTURE_TIEBREAK of the
// best — those can neither win nor take part in the fixture tie-break.
const eligible: {
glyph: Glyph;
tRows: number;
tCols: number;
r: number;
hr: number;
bound: number;
}[] = [];
for (const glyph of set.glyphs) {
const t = glyph.mat;
const tRows = t.rows;
const tCols = t.cols;
if (tRows > regionRows || tCols > regionCols) continue;
const wRatio = tCols / Math.max(segWidth, 1);
if (wRatio < 0.4 || wRatio > 2.5) continue;
const hRatio = tRows / Math.max(seg.height, 1);
// a template sliding freely inside a taller region can score high on a
// fragment of the segment (an 'l' bar inside a 'c'), so reject templates
// much taller or shorter than the segment's ink extent
if (hRatio < 0.5 || hRatio > 1.3) continue;
// ink-coverage penalty: templates should explain the segment's ink
const r = Math.min(glyph.ink, seg.ink) / Math.max(Math.max(glyph.ink, seg.ink), 1);
// height-mismatch penalty: an x-height template sliding on an ascender/
// descender glyph ('o' on 'b' or 'g') can correlate well on the bowl
// alone; matching heights should outrank it
const hr = Math.min(tRows, seg.height) / Math.max(tRows, Math.max(seg.height, 1));
eligible.push({
glyph,
tRows,
tCols,
r,
hr,
bound: (0.7 + 0.3 * r) * (0.85 + 0.15 * hr),
});
}
eligible.sort((a, b) => b.bound - a.bound);
const result = new cv.Mat();
const candidates: {
char: string;
score: number;
ncc: number;
source: "fixture" | "font";
}[] = [];
// Only consider placements that cover most of the segment: the region
// is padded, so a freely-sliding template can otherwise score a perfect
// match on the *neighboring* glyph inside the pad (an 'l' template next
// to a narrow 'i' segment matches the adjacent real 'l' at 0.97+), or
// on a fragment of the segment (a 't' stem on the left bar of a 'b').
const minOverlap = 0.7 * segWidth;
// In probe mode only the boolean "can any glyph beat the floor" matters,
// so the loop can stop as soon as either answer is certain: no remaining
// bound reaches the floor, or a computed score already cleared it.
const probeMode = scoreFloor !== -Infinity;
let bestScore = scoreFloor;
for (const { glyph, tRows, tCols, r, hr, bound } of eligible) {
if (bound < bestScore - FIXTURE_TIEBREAK) break;
if (probeMode && (bound <= scoreFloor || bestScore > scoreFloor)) break;
cv.matchTemplate(region, glyph.mat, result, cv.TM_CCOEFF_NORMED);
const rCols = regionCols - tCols + 1;
const rRows = regionRows - tRows + 1;
// overlap(sx) is concave in sx, so the valid placements form one
// contiguous rx interval — find its edges, then scan row-major
const overlapAt = (rx: number) => Math.min(x0 + rx + tCols, seg.x1) - Math.max(x0 + rx, seg.x0);
let lo = 0;
while (lo < rCols && overlapAt(lo) < minOverlap) lo++;
let hi = rCols - 1;
while (hi >= lo && overlapAt(hi) < minOverlap) hi--;
if (hi < lo) continue;
let maxVal = -Infinity;
const scores = result.data32F;
for (let ry = 0, rowBase = 0; ry < rRows; ry++, rowBase += rCols) {
for (let rx = lo; rx <= hi; rx++) {
const v = scores[rowBase + rx]!;
if (v > maxVal) maxVal = v;
}
}
const score = maxVal * (0.7 + 0.3 * r) * (0.85 + 0.15 * hr);
if (Number.isFinite(score)) {
if (score > bestScore) bestScore = score;
candidates.push({ char: glyph.char, score, ncc: maxVal, source: glyph.source });
}
}
result.delete();
region.delete();
candidates.sort((a, b) => b.score - a.score);
const top = candidates[0];
if (top && top.source === "font") {
const fixture = candidates.find(
(c) =>
c.source === "fixture" &&
top.score - c.score < FIXTURE_TIEBREAK &&
// ...but only when the fixture's raw correlation is also competitive;
// a fixture crop of a *different* char that merely lands near the top
// score via the ink penalty must not displace a well-matching font glyph
top.ncc - c.ncc < FIXTURE_TIEBREAK,
);
if (fixture && fixture !== top) {
candidates.splice(candidates.indexOf(fixture), 1);
candidates.unshift(fixture);
}
}
return candidates.slice(0, 5);
}
interface ClassifiedSegment {
seg: SegmentInfo;
ranked: ReturnType<typeof classifySegment>;
}
/**
* Multi-stroke glyphs whose strokes never connect (katakana ハ/パ/リ and
* kin) segment as two fragments, and the full-glyph template cannot match
* either one (the width-ratio filter rightly rejects a wide template on a
* narrow fragment). Re-classify close neighbor pairs as a single segment
* and keep the merge only when it beats both fragments by a clear margin —
* genuine letter pairs ("rn", "VV") already read well individually, so a
* lookalike merged template ('m', 'W') cannot clear the margin over them.
*/
/**
* Kana stroke gaps run wide: い's two strokes sit ~0.4 medianWidth apart at
* tag-name scale, so the gap cap must reach past that for the merge to even
* be attempted — while staying below the space gap (0.55 medianWidth), so
* merges never bridge a real word break. Misjoins stay guarded by the
* beat-both-fragments margin below.
*/
const MERGE_MAX_GAP_RATIO = 0.45;
const MERGE_MARGIN = 0.02;
function mergeSplitGlyphs(
items: ClassifiedSegment[],
binary: Mat,
masked: Mat,
set: GlyphSet,
): void {
const maxGap = Math.max(3, Math.round(set.medianWidth * MERGE_MAX_GAP_RATIO));
const maxCharWidth = Math.round(set.medianWidth * 1.5);
for (let i = 0; i + 1 < items.length; ) {
const a = items[i]!;
const b = items[i + 1]!;
const gap = b.seg.x0 - a.seg.x1;
const width = b.seg.x1 - a.seg.x0;
if (gap > maxGap || width > maxCharWidth) {
i++;
continue;
}
const seg = measureSegment(binary, { x0: a.seg.x0, x1: b.seg.x1 });
const fragmentBest = Math.max(a.ranked[0]?.score ?? 0, b.ranked[0]?.score ?? 0);
const floor = fragmentBest + MERGE_MARGIN;
// Probe with the floor first: most neighbor pairs are genuine letter
// pairs whose merge can't win, and the floor lets the bound-sorted
// matching stop almost immediately. The probe's computed scores are
// exact, so "no candidate beats the floor" is a definitive reject.
const probe = classifySegment(masked, seg, set, floor);
let merged = false;
if (probe.some((c) => c.score > floor)) {
// full run (rare): the winning merge's ranked list must also carry
// the sub-floor runner-up candidates downstream consumers see
const ranked = classifySegment(masked, seg, set);
if ((ranked[0]?.score ?? 0) > floor) {
// stay at i: the merged segment may absorb yet another stroke
items.splice(i, 2, { seg, ranked });
merged = true;
}
}
if (!merged) i++;
}
}
/**
* Recognize white-on-dark text in a grayscale crop.
* The crop must be tight vertically (one text line).
*/
export function recognizeText(
gray: Mat,
set: GlyphSet,
options: RecognizeOptions = {},
): RecognizedText {
const cv = getCV();
const {
binThreshold = 150,
minColumnPixels = 1,
spaceGap = Math.max(5, Math.round(set.medianWidth * 0.55)),
minCharScore = 0.4,
} = options;
const binary = new cv.Mat();
cv.threshold(gray, binary, binThreshold, 255, cv.THRESH_BINARY);
// mask the gray crop so background patterns don't take part in matching,
// dilated so antialiased glyph edges survive
const mask = new cv.Mat();
const kernel = cv.getStructuringElement(cv.MORPH_RECT, new cv.Size(3, 3));
cv.dilate(binary, mask, kernel, new cv.Point(-1, -1), 2);
kernel.delete();
const masked = new cv.Mat(gray.rows, gray.cols, cv.CV_8UC1, new cv.Scalar(0));
gray.copyTo(masked, mask);
mask.delete();
const profile = columnProfile(binary);
const segments = segmentColumns(profile, minColumnPixels)
.flatMap((s) => splitWideSegment(profile, s, set.medianWidth))
.map((s) => measureSegment(binary, s));
const items: ClassifiedSegment[] = segments.map((seg) => ({
seg,
ranked: classifySegment(masked, seg, set),
}));
mergeSplitGlyphs(items, binary, masked, set);
const chars: RecognizedChar[] = [];
let text = "";
let prevEnd: number | null = null;
for (const { seg, ranked } of items) {
if (prevEnd !== null && seg.x0 - prevEnd > spaceGap && text.length > 0) {
text += " ";
}
const top = ranked[0];
if (top && top.score >= minCharScore) {
chars.push({
char: top.char,
score: top.score,
x0: seg.x0,
x1: seg.x1,
y0: seg.y0,
y1: seg.y1,
candidates: ranked.map((c) => ({
char: c.char,
score: c.score,
ncc: c.ncc,
source: c.source,
})),
});
text += top.char;
}
prevEnd = seg.x1;
}
binary.delete();
masked.delete();
const confidence =
chars.length > 0 ? Math.min(...chars.map((c) => c.score)) : segments.length > 0 ? 0 : 1;
return { text, chars, confidence };
}

150
app/features/cv/core/image.ts Normal file
View File

@@ -0,0 +1,150 @@
/**
* Environment-agnostic frame representation and helpers.
* A FrameData is RGBA, same layout as browser ImageData — Node code builds it
* from @napi-rs/canvas, browser code from a canvas or VideoFrame.
*/
import { CANONICAL_HEIGHT, CANONICAL_WIDTH, type Roi } from "./canonical";
import { getCV, type Mat, meanOf, minMaxLoc } from "./cv";
export type { Roi };
export interface FrameData {
width: number;
height: number;
/** RGBA, 4 bytes per pixel */
data: Uint8ClampedArray;
}
export function toMat(frame: FrameData): Mat {
const cv = getCV();
// matFromImageData only reads width/height/data, so FrameData is compatible
return cv.matFromImageData(frame as unknown as ImageData);
}
/**
* Normalize any input frame to the canonical 1920x1080 RGBA mat that all ROI
* constants are defined against. Returns a new mat; caller owns both.
*/
export function normalizeFrame(src: Mat): Mat {
const cv = getCV();
const dst = new cv.Mat();
if (src.cols === CANONICAL_WIDTH && src.rows === CANONICAL_HEIGHT) {
src.copyTo(dst);
return dst;
}
const interpolation = src.cols > CANONICAL_WIDTH ? cv.INTER_AREA : cv.INTER_CUBIC;
cv.resize(src, dst, new cv.Size(CANONICAL_WIDTH, CANONICAL_HEIGHT), 0, 0, interpolation);
return dst;
}
/**
* Crop a rect out of a mat. Returns a view: fine as *input* to OpenCV calls
* (matchTemplate, mean, resize, ...) but NEVER read `.data` off it — in this
* opencv.js build both `.data` and `.clone()` mishandle non-continuous views.
* Use copyRoi when pixel access is needed.
*/
export function cropRoi(src: Mat, roi: Roi): Mat {
const cv = getCV();
return src.roi(new cv.Rect(roi.x, roi.y, roi.w, roi.h));
}
/** Crop a rect into a fresh continuous mat (safe for `.data` access). */
export function copyRoi(src: Mat, roi: Roi): Mat {
const view = cropRoi(src, roi);
const out = new (getCV().Mat)();
view.copyTo(out);
view.delete();
return out;
}
/**
* Mean brightness of a ROI: the average of the first three channels on a
* color mat, the single channel's mean on a grayscale (or |Laplacian|) mat.
* The shared probe primitive of every detector gate.
*/
export function meanBrightness(mat: Mat, roi: Roi): number {
const view = cropRoi(mat, roi);
const m = meanOf(view);
view.delete();
return mat.channels() >= 3 ? (m[0]! + m[1]! + m[2]!) / 3 : m[0]!;
}
/** Brightest pixel of a grayscale ROI. */
export function maxBrightness(gray: Mat, roi: Roi): number {
const view = cropRoi(gray, roi);
const { maxVal } = minMaxLoc(view);
view.delete();
return maxVal;
}
function channelExtreme(mat: Mat, roi: Roi | undefined, op: "min" | "max"): Mat {
const cv = getCV();
const view = roi ? cropRoi(mat, roi) : null;
const src = view ?? mat;
const channels = new cv.MatVector();
cv.split(src, channels);
const r = channels.get(0);
const g = channels.get(1);
const b = channels.get(2);
const rg = new cv.Mat();
const out = new cv.Mat();
if (op === "max") {
cv.max(r, g, rg);
cv.max(rg, b, out);
} else {
cv.min(r, g, rg);
cv.min(rg, b, out);
}
rg.delete();
r.delete();
g.delete();
b.delete();
if (mat.channels() === 4) channels.get(3).delete();
channels.delete();
view?.delete();
return out;
}
/** Brightest channel per pixel, so colored text binarizes like white. */
export function maxChannel(mat: Mat, roi?: Roi): Mat {
return channelExtreme(mat, roi, "max");
}
/** Per-pixel min of R/G/B — drops color-tinted brightness, keeps white. */
export function minChannel(mat: Mat, roi?: Roi): Mat {
return channelExtreme(mat, roi, "min");
}
/** |Laplacian| response of a grayscale mat; caller owns the result. */
export function laplacianAbs(gray: Mat): Mat {
const cv = getCV();
const lap = new cv.Mat();
cv.Laplacian(gray, lap, cv.CV_16S, 3, 1, 0, cv.BORDER_DEFAULT);
const abs8 = new cv.Mat();
cv.convertScaleAbs(lap, abs8);
lap.delete();
return abs8;
}
export function matToFrameData(mat: Mat): FrameData {
const cv = getCV();
const rgba = new cv.Mat();
if (mat.type() === cv.CV_8UC4) {
mat.copyTo(rgba);
} else if (mat.type() === cv.CV_8UC3) {
cv.cvtColor(mat, rgba, cv.COLOR_RGB2RGBA);
} else if (mat.type() === cv.CV_8UC1) {
cv.cvtColor(mat, rgba, cv.COLOR_GRAY2RGBA);
} else {
rgba.delete();
throw new Error(`unsupported mat type ${mat.type()}`);
}
const out: FrameData = {
width: rgba.cols,
height: rgba.rows,
data: new Uint8ClampedArray(rgba.data),
};
rgba.delete();
return out;
}

2160
app/features/cv/core/localized-entries.ts Normal file
View File

File diff suppressed because it is too large Load Diff

94
app/features/cv/core/localized.ts Normal file
View File

@@ -0,0 +1,94 @@
/**
* Flattened localized match sets derived from the generated
* localized-entries.ts: detectors snap OCR output against every language's
* strings at once and report the canonical English value, so ingestion
* works no matter which language the player runs the game in. Combos
* (mode+stage, lobby+mode) stay within one language — on-screen text never
* mixes languages — which keeps the cross product from inventing pairings
* no UI would show.
*/
import { LANGUAGE_ENTRIES, type LocalizedText } from "./localized-entries";
import { matchKey } from "./text";
function dedupe<T>(items: T[], keyOf: (item: T) => string): T[] {
const seen = new Set<string>();
return items.filter((item) => {
const k = keyOf(item);
if (seen.has(k)) return false;
seen.add(k);
return true;
});
}
const byText = (e: { text: string }) => matchKey(e.text);
export const ALL_LOBBY_ENTRIES: readonly LocalizedText[] = dedupe(
LANGUAGE_ENTRIES.flatMap((l) => l.lobbies),
byText,
);
/** Single-line mode names plus the intro splash's two-line wrap variants. */
export const ALL_MODE_ENTRIES: readonly LocalizedText[] = dedupe(
LANGUAGE_ENTRIES.flatMap((l) => [...l.modes, ...l.modeWraps]),
byText,
);
export const ALL_STAGE_ENTRIES: readonly LocalizedText[] = dedupe(
LANGUAGE_ENTRIES.flatMap((l) => l.stages),
byText,
);
/** Every language's constant "MODE" intro-splash label. */
export const ALL_MODE_LABELS: readonly string[] = dedupe(
LANGUAGE_ENTRIES.map((l) => l.modeLabel),
matchKey,
);
/** Replay-browser panel tags; canonical is "VICTORY" or "DEFEAT". */
export const RESULT_TAG_ENTRIES: readonly LocalizedText[] = dedupe(
LANGUAGE_ENTRIES.flatMap((l) => [
{ text: l.victory, canonical: "VICTORY" },
{ text: l.defeat, canonical: "DEFEAT" },
]),
byText,
);
export interface ModeStageCombo {
text: string;
mode: string;
stage: string;
}
/** The scoreboard header's "<mode> <stage>" line, per language. */
export const MODE_STAGE_COMBOS: readonly ModeStageCombo[] = dedupe(
LANGUAGE_ENTRIES.flatMap((l) =>
l.modes.flatMap((mode) =>
l.stages.map((stage) => ({
text: `${mode.text} ${stage.text}`,
mode: mode.canonical,
stage: stage.canonical,
})),
),
),
byText,
);
export interface LobbyModeCombo {
text: string;
lobby: string;
mode: string;
}
/** The replay-browser header's "<lobby> <mode>" line, per language. */
export const LOBBY_MODE_COMBOS: readonly LobbyModeCombo[] = dedupe(
LANGUAGE_ENTRIES.flatMap((l) =>
l.lobbies.flatMap((lobby) =>
l.modes.map((mode) => ({
text: `${lobby.text} ${mode.text}`,
lobby: lobby.canonical,
mode: mode.canonical,
})),
),
),
byText,
);

128
app/features/cv/core/replay-time.ts Normal file
View File

@@ -0,0 +1,128 @@
/**
* Turns the replay browser's on-screen recording timestamp (locale-formatted
* by the console, e.g. "3/7/2026 22:28", "7.3.2026 22:28", "2026/3/7 22:28")
* into a UTC epoch. The string carries no timezone or day/month-order
* marker: the wall time is read in the local timezone, and when day vs month
* is ambiguous (both ≤ 12) the order is inferred in three steps. An hour of
* 0 or ≥ 13 proves a 24h clock (a 12h console would render AM/PM, which the
* pattern doesn't match) and 24h locales are overwhelmingly day-first — the
* lone major month-first locale, en-US, uses a 12h clock. Failing that, the
* browser locale's date-part order decides. Finally, because the console's
* locale and the browser's can still disagree (Finnish console, en-US
* browser), a recency check corrects decisive misreads: replays are
* near-always ingested close to when they were recorded, so when the chosen
* reading lands far from now while the day/month swap lands recent, the swap
* wins.
*/
const TIMESTAMP_RE = /^(\d{1,4})[./-](\d{1,4})[./-](\d{1,4})\s+(\d{1,2}):(\d{2})$/;
/** How far in the past a reading may land and still count as "recent". */
const RECENT_PAST_MS = 30 * 24 * 60 * 60 * 1000;
/** Forward tolerance for console-vs-browser clock and timezone skew. */
const FUTURE_SLACK_MS = 24 * 60 * 60 * 1000;
type Ymd = { year: number; month: number; day: number };
/**
* Parses a replay timestamp into UTC epoch milliseconds, or null when the
* string doesn't form a valid date. `locale` defaults to the environment's;
* `now` (default `Date.now()`) anchors the recency disambiguation — pass the
* moment the replay screen was on-screen, not the send time.
*/
export function parseReplayTimestamp(
raw: string,
{ locale, now }: { locale?: string; now?: number } = {},
): number | null {
const m = TIMESTAMP_RE.exec(raw.trim());
if (!m) return null;
const dateParts = [Number(m[1]!), Number(m[2]!), Number(m[3]!)];
const hours = Number(m[4]!);
const minutes = Number(m[5]!);
if (hours > 23 || minutes > 59) return null;
// hour 0 or ≥13 only occurs on a 24h clock, and 24h locales are
// near-universally day-first; only an ambiguous hour falls back to the
// browser locale, which may not match the console's
const is24hClock = hours === 0 || hours >= 13;
const resolved = resolveDateParts(dateParts, is24hClock ? true : dayBeforeMonth(locale));
if (!resolved) return null;
const preferred = toEpoch(resolved.preferred, hours, minutes);
if (preferred === null) return null;
if (resolved.swapped) {
const swapped = toEpoch(resolved.swapped, hours, minutes);
const ref = now ?? Date.now();
if (swapped !== null && isRecent(swapped, ref) && !isRecent(preferred, ref)) {
return swapped;
}
}
return preferred;
}
function isRecent(t: number, ref: number): boolean {
return t >= ref - RECENT_PAST_MS && t <= ref + FUTURE_SLACK_MS;
}
function toEpoch({ year, month, day }: Ymd, hours: number, minutes: number): number | null {
const date = new Date(year, month - 1, day, hours, minutes);
// the Date constructor rolls invalid dates over (31/2 → 2/3); reject those
if (date.getFullYear() !== year || date.getMonth() !== month - 1 || date.getDate() !== day) {
return null;
}
return date.getTime();
}
/**
* Splits the three date segments into year/month/day. The year is the
* segment with ≥3 digits (leading for ja-style "2026/3/7", trailing
* otherwise); with no such segment the last one is a 2-digit year. Day vs
* month resolves by magnitude when one exceeds 12, else by the caller's
* order guess (year-first formats are month-first — no locale writes Y/D/M);
* a guessed split also reports the swapped reading for the recency check.
*/
function resolveDateParts(
parts: number[],
guessDayFirst: boolean,
): { preferred: Ymd; swapped: Ymd | null } | null {
let year: number;
let rest: [number, number];
let dayFirst: boolean;
let orderKnown = false;
if (parts[0]! >= 100) {
year = parts[0]!;
rest = [parts[1]!, parts[2]!];
dayFirst = false;
orderKnown = true;
} else {
year = parts[2]! >= 100 ? parts[2]! : 2000 + parts[2]!;
rest = [parts[0]!, parts[1]!];
dayFirst = guessDayFirst;
}
const [a, b] = rest;
let day: number;
let month: number;
let guessed = false;
if (a > 12 && b > 12) return null;
if (a > 12) [day, month] = [a, b];
else if (b > 12) [month, day] = [a, b];
else {
[day, month] = dayFirst ? [a, b] : [b, a];
guessed = !orderKnown && a !== b;
}
if (month < 1 || month > 12 || day < 1 || day > 31) return null;
return {
preferred: { year, month, day },
swapped: guessed ? { year, month: day, day: month } : null,
};
}
function dayBeforeMonth(locale: string | undefined): boolean {
const parts = new Intl.DateTimeFormat(locale).formatToParts(new Date(2000, 0, 2));
const dayIndex = parts.findIndex((p) => p.type === "day");
const monthIndex = parts.findIndex((p) => p.type === "month");
return dayIndex !== -1 && monthIndex !== -1 && dayIndex < monthIndex;
}

194
app/features/cv/core/resources.ts Normal file
View File

@@ -0,0 +1,194 @@
/**
* Environment-agnostic assembly of ScoreboardResources. Node (filesystem)
* and the worker (HTTP) inject only the four IO primitives; every resource
* key, icon directory, template option set, and atlas name lives here once,
* so adding a resource cannot desync the two loaders.
*
* Icon/atlas decoding happens eagerly (cheap), but template preparation and
* atlas slicing are deferred behind memoized getters: a test process loads
* the full bundle while its one detector only ever touches a slice of it,
* and the unused sets (four weapon-template builds alone are ~7k resizes)
* dominated startup.
*/
import { prepareAbilityTemplates } from "./detectors/death/abilities";
import { BURST_ICON_TEMPLATE_SIZES } from "./detectors/death/rois";
import { prepareMinimapAbilityTemplates } from "./detectors/minimap/abilities";
import {
CARD_WEAPON_BACKGROUND,
MINIMAP_WEAPON_INK_THRESHOLD,
MINIMAP_WEAPON_TEMPLATE_SIZES,
SPECIAL_READY_BACKGROUND,
SPECIAL_READY_INK_THRESHOLD,
SUB_TILE_TEMPLATE_SIZES,
} from "./detectors/minimap/rois";
import type { PlannerStage } from "./detectors/minimap/stage";
import type { ScoreboardResources } from "./detectors/scoreboard/index";
import { prepareSpecialTemplates } from "./detectors/scoreboard/specials";
import { prepareWeaponTemplates } from "./detectors/scoreboard/weapons";
import { prepareOwnAbilityTemplates } from "./detectors/scoreboard-own/abilities";
import type { GlyphSet } from "./glyphs";
import type { FrameData } from "./image";
export interface ResourceIO {
/** ids listed in <assets>/<dir>/manifest.json */
readManifest(dir: string): Promise<string[]>;
/** decoded RGBA of <assets>/<dir>/<id>.png */
readIcon(dir: string, id: string): Promise<FrameData>;
/** glyph atlas by name as a (possibly lazy) getter; () => null when absent */
loadAtlas(name: string): Promise<() => GlyphSet | null>;
/** planner signature atlas as a getter; () => null when absent */
loadPlannerStages(): Promise<() => PlannerStage[] | null>;
}
/** resource key → atlas name under assets/cv/glyphs/ */
const ATLASES = {
paintDigits: "scoreboard-paint-digits",
statDigits: "scoreboard-stat-digits",
teamDigits: "scoreboard-team-digits",
nameGlyphs: "scoreboard-names",
headerLobbyGlyphs: "scoreboard-header-lobby",
headerLineGlyphs: "scoreboard-header-line",
replayCodeGlyphs: "scoreboard-replay-code",
replayResultGlyphs: "scoreboard-replay-result",
deathWeaponGlyphs: "death-weapon",
deathWeaponJaGlyphs: "death-weapon-ja",
deathTagNameGlyphs: "death-tag-name",
mapStartModeGlyphs: "map-start-mode",
mapStartStageGlyphs: "map-start-stage",
} as const;
/** Memoize an expensive template/atlas build for the lazy resource getters. */
function lazy<T>(build: () => T): () => T {
let value: T;
let built = false;
return () => {
if (!built) {
value = build();
built = true;
}
return value;
};
}
/** Requires loadOpenCV() to have resolved. */
export async function assembleScoreboardResources(io: ResourceIO): Promise<ScoreboardResources> {
const icons = async (dir: string) => {
const ids = await io.readManifest(dir);
return Promise.all(ids.map(async (id) => ({ id, image: await io.readIcon(dir, id) })));
};
const [weaponIcons, specialIcons, subIcons, abilityIcons, plannerStages, atlasEntries] =
await Promise.all([
icons("main-weapons"),
icons("specials"),
icons("sub-weapons"),
icons("abilities"),
io.loadPlannerStages(),
Promise.all(
(Object.entries(ATLASES) as [keyof typeof ATLASES, string][]).map(
async ([key, name]) => [key, await io.loadAtlas(name)] as const,
),
),
]);
const atlas = Object.fromEntries(atlasEntries) as Record<
keyof typeof ATLASES,
() => GlyphSet | null
>;
const weapons = lazy(() => prepareWeaponTemplates(weaponIcons));
const deathBurstWeapons = lazy(() =>
prepareWeaponTemplates(weaponIcons, BURST_ICON_TEMPLATE_SIZES),
);
const minimapCardWeapons = lazy(() =>
prepareWeaponTemplates(weaponIcons, MINIMAP_WEAPON_TEMPLATE_SIZES, {
background: CARD_WEAPON_BACKGROUND,
inkThreshold: MINIMAP_WEAPON_INK_THRESHOLD,
cropToArt: true,
}),
);
const minimapLightWeapons = lazy(() =>
prepareWeaponTemplates(weaponIcons, MINIMAP_WEAPON_TEMPLATE_SIZES, {
background: SPECIAL_READY_BACKGROUND,
inkThreshold: SPECIAL_READY_INK_THRESHOLD,
cropToArt: true,
}),
);
const specials = lazy(() => prepareSpecialTemplates(specialIcons));
const minimapSubWeapons = lazy(() => prepareSpecialTemplates(subIcons, SUB_TILE_TEMPLATE_SIZES));
const abilities = lazy(() => prepareAbilityTemplates(abilityIcons));
const ownAbilities = lazy(() => prepareOwnAbilityTemplates(abilityIcons));
const minimapAbilities = lazy(() => prepareMinimapAbilityTemplates(abilityIcons));
return {
get weapons() {
return weapons();
},
get deathBurstWeapons() {
return deathBurstWeapons();
},
get minimapCardWeapons() {
return minimapCardWeapons();
},
get minimapLightWeapons() {
return minimapLightWeapons();
},
get specials() {
return specials();
},
get minimapSubWeapons() {
return minimapSubWeapons();
},
get abilities() {
return abilities();
},
get ownAbilities() {
return ownAbilities();
},
get minimapAbilities() {
return minimapAbilities();
},
get plannerStages() {
return plannerStages();
},
get paintDigits() {
return atlas.paintDigits();
},
get statDigits() {
return atlas.statDigits();
},
get teamDigits() {
return atlas.teamDigits();
},
get nameGlyphs() {
return atlas.nameGlyphs();
},
get headerLobbyGlyphs() {
return atlas.headerLobbyGlyphs();
},
get headerLineGlyphs() {
return atlas.headerLineGlyphs();
},
get replayCodeGlyphs() {
return atlas.replayCodeGlyphs();
},
get replayResultGlyphs() {
return atlas.replayResultGlyphs();
},
get deathWeaponGlyphs() {
return atlas.deathWeaponGlyphs();
},
get deathWeaponJaGlyphs() {
return atlas.deathWeaponJaGlyphs();
},
get deathTagNameGlyphs() {
return atlas.deathTagNameGlyphs();
},
get mapStartModeGlyphs() {
return atlas.mapStartModeGlyphs();
},
get mapStartStageGlyphs() {
return atlas.mapStartStageGlyphs();
},
};
}

137
app/features/cv/core/text.ts Normal file
View File

@@ -0,0 +1,137 @@
/**
* Small text utilities: edit distance and closed-set snapping for OCR output.
*/
function editDistance(a: string, b: string): number {
const dp = Array.from({ length: a.length + 1 }, (_, i) => {
const row = new Array<number>(b.length + 1).fill(0);
row[0] = i;
return row;
});
for (let j = 0; j <= b.length; j++) dp[0]![j] = j;
for (let i = 1; i <= a.length; i++) {
for (let j = 1; j <= b.length; j++) {
dp[i]![j] = Math.min(
dp[i - 1]![j]! + 1,
dp[i]![j - 1]! + 1,
dp[i - 1]![j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1),
);
}
}
return dp[a.length]![b.length]!;
}
/**
* Case-, space- and diacritic-insensitive comparison key (é≈e), so OCR
* confusing an accented glyph with its base form stays a near-match across
* the localized closed sets.
*/
export function matchKey(s: string): string {
return s
.toLowerCase()
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
.replace(/\s+/g, "");
}
export interface ClosestMatch<T> {
entry: T;
/** 1 = exact (ignoring case/spaces/diacritics), 0 = nothing in common */
score: number;
}
/** Snap an OCR reading to the closest of a closed set of arbitrary entries. */
export function closestBy<T>(
reading: string,
entries: readonly T[],
textOf: (entry: T) => string,
): ClosestMatch<T> | null {
const r = matchKey(reading);
let best: ClosestMatch<T> | null = null;
for (const entry of entries) {
const e = matchKey(textOf(entry));
const d = editDistance(r, e);
const score = 1 - d / Math.max(r.length, e.length, 1);
if (!best || score > best.score) best = { entry, score };
}
return best;
}
/** Snap an OCR reading to the closest entry of a closed set of strings. */
export function closestEntry<T extends string>(
reading: string,
entries: readonly T[],
): ClosestMatch<T> | null {
return closestBy(reading, entries, (e) => e);
}
/** Rank every entry of a closed set against an OCR reading, best first. */
export function rankBy<T>(
reading: string,
entries: readonly T[],
textOf: (entry: T) => string,
): ClosestMatch<T>[] {
const r = matchKey(reading);
return entries
.map((entry) => {
const e = matchKey(textOf(entry));
const d = editDistance(r, e);
return { entry, score: 1 - d / Math.max(r.length, e.length, 1) };
})
.sort((a, b) => b.score - a.score);
}
/**
* One recognized text segment with its ranked alternatives — structurally
* compatible with glyphs.ts RecognizedChar, kept minimal here so string
* utilities stay decoupled from the recognition module.
*/
export interface ReadSegment {
candidates?: readonly { char: string; score: number }[];
}
/**
* Rank a closed set against the segment *candidate lists* of a recognized
* line instead of its greedy top-1-per-segment string. On low-fidelity
* captures (720p upscaled) the right glyph often sits at rank 2-3 of a
* segment while the top-1 string is garbage; plain edit distance can't see
* it, this can. Alignment is a weighted edit distance: matching a segment
* to a target char costs 1 minus that char's score in the segment's
* candidate list (1 when absent), insert/delete cost 1. Scores land well
* below rankBy's for the same quality of match (a correct char still costs
* 1 - templateScore), so the two scales must not share thresholds.
*/
export function rankByRead<T>(
segments: readonly ReadSegment[],
entries: readonly T[],
textOf: (entry: T) => string,
): ClosestMatch<T>[] {
// per-segment candidate score by match key (max wins when keys collide,
// e.g. dakuten variants folding onto one base kana)
const segScores = segments.map((seg) => {
const m = new Map<string, number>();
for (const c of seg.candidates ?? []) {
const k = matchKey(c.char);
const s = Math.max(0, Math.min(1, c.score));
if (s > (m.get(k) ?? 0)) m.set(k, s);
}
return m;
});
const n = segScores.length;
return entries
.map((entry) => {
const t = matchKey(textOf(entry));
const m = t.length;
const dp = Array.from({ length: n + 1 }, () => new Array<number>(m + 1).fill(0));
for (let i = 1; i <= n; i++) dp[i]![0] = i;
for (let j = 1; j <= m; j++) dp[0]![j] = j;
for (let i = 1; i <= n; i++) {
for (let j = 1; j <= m; j++) {
const sub = 1 - (segScores[i - 1]!.get(t[j - 1]!) ?? 0);
dp[i]![j] = Math.min(dp[i - 1]![j]! + 1, dp[i]![j - 1]! + 1, dp[i - 1]![j - 1]! + sub);
}
}
return { entry, score: 1 - dp[n]![m]! / Math.max(n, m, 1) };
})
.sort((a, b) => b.score - a.score);
}

87
app/features/cv/core/timeline/index.ts Normal file
View File

@@ -0,0 +1,87 @@
/**
* TimelineBuilder: minimal event stream cleanup for the POC.
* Same-type events within a merge window collapse into one, keeping the
* highest-confidence version; events below a confidence floor are dropped.
*/
import { SCOREBOARD_EVENT_TYPE } from "../detectors/scoreboard/index";
import { SCOREBOARD_REPLAY_EVENT_TYPE } from "../detectors/scoreboard-replay/index";
import type { DetectedEvent } from "../detectors/types";
import { sameScoreboardMatch } from "./same-scoreboard";
export interface TimelineOptions {
/** same-type events closer than this (seconds) merge */
mergeWindow: number;
/**
* per-type mergeWindow overrides: repeatable events need a window shorter
* than the minimum spacing between two real occurrences
*/
mergeWindowByType: Record<string, number>;
/**
* per-type content guard: same-type events inside the window merge only
* when this returns true for their data — screens whose distinct real
* occurrences can appear seconds apart (replay browsing) need content,
* not time, to tell them apart. Absent = merge on time alone.
*/
sameEventDataByType: Record<string, (a: unknown, b: unknown) => boolean>;
/** events below this confidence are dropped */
minConfidence: number;
}
const DEFAULT_TIMELINE_OPTIONS: TimelineOptions = {
mergeWindow: 30,
// the death screen shows for ~5s and respawn takes ~8.5s, so repeat frames
// of one death land within the window while consecutive deaths are outside;
// players flick the map open for 1-3s and each open is a fresh sample
// (slots read differently across opens), so minimap frames merge only
// within one open
mergeWindowByType: { Death: 8, Minimap: 5 },
sameEventDataByType: {
[SCOREBOARD_EVENT_TYPE]: sameScoreboardMatch,
[SCOREBOARD_REPLAY_EVENT_TYPE]: sameScoreboardMatch,
},
minConfidence: 0.6,
};
export type TimelineAction =
| { action: "added"; event: DetectedEvent }
| { action: "replaced"; event: DetectedEvent; replaced: DetectedEvent }
| { action: "merged"; into: DetectedEvent }
| { action: "dropped"; reason: "low-confidence" };
export class TimelineBuilder {
#events: DetectedEvent[] = [];
#options: TimelineOptions;
constructor(options: Partial<TimelineOptions> = {}) {
this.#options = { ...DEFAULT_TIMELINE_OPTIONS, ...options };
}
get events(): readonly DetectedEvent[] {
return this.#events;
}
push(event: DetectedEvent): TimelineAction {
if (event.confidence < this.#options.minConfidence) {
return { action: "dropped", reason: "low-confidence" };
}
const window = this.#options.mergeWindowByType[event.type] ?? this.#options.mergeWindow;
const same = this.#options.sameEventDataByType[event.type];
const near = this.#events.find(
(e) =>
e.type === event.type &&
Math.abs(e.t - event.t) <= window &&
(same?.(e.data, event.data) ?? true),
);
if (!near) {
this.#events.push(event);
this.#events.sort((a, b) => a.t - b.t);
return { action: "added", event };
}
if (event.confidence > near.confidence) {
this.#events[this.#events.indexOf(near)] = event;
this.#events.sort((a, b) => a.t - b.t);
return { action: "replaced", event, replaced: near };
}
return { action: "merged", into: near };
}
}

88
app/features/cv/core/timeline/same-scoreboard.ts Normal file
View File

@@ -0,0 +1,88 @@
/**
* Content guard for merging scoreboard-shaped events (results screen,
* replay-browser detail). The replay browser lets the user flip between
* different matches' detail screens within seconds, so a time window alone
* cannot tell "same board, sampled again" from "next replay opened" — three
* distinct replays browsed back-to-back must not collapse into one event.
*
* Two events are considered the same match unless something decisive says
* otherwise; every check tolerates the read jitter of a static screen
* (glyph misreads, a flipped winner side, fields that only read on some
* frames), so a split only happens on evidence that survives that noise.
*/
import type { ScoreboardData } from "../detectors/scoreboard/index";
import type { ScoreboardReplayData } from "../detectors/scoreboard-replay/index";
/**
* Replay codes of the same replay re-read on a low-fidelity capture differ
* in a few glyphs (U/V, G/C confusions); different replays share almost no
* positions. Split only past this many mismatched characters.
*/
const CODE_DIFF_MIN = 7;
/**
* Paint totals are per-match fingerprints that read reliably (big digits).
* Compared only when both events read at least this many of the 8 rows.
*/
const PAINT_MIN_READ = 6;
/** Positions where two equal-length strings disagree (∞ on length mismatch). */
function charDiff(a: string, b: string): number {
if (a.length !== b.length) return Number.POSITIVE_INFINITY;
let n = 0;
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) n++;
return n;
}
/** Size of the multiset intersection of two number lists. */
function multisetOverlap(a: number[], b: number[]): number {
const counts = new Map<number, number>();
for (const v of a) counts.set(v, (counts.get(v) ?? 0) + 1);
let n = 0;
for (const v of b) {
const c = counts.get(v) ?? 0;
if (c > 0) {
counts.set(v, c - 1);
n++;
}
}
return n;
}
/** The non-null paint values, order-free (winner side can flip between reads). */
function paints(data: Partial<ScoreboardData>): number[] {
return (data.players ?? []).map((p) => p.paint).filter((p): p is number => p !== null);
}
/**
* Whether two scoreboard events plausibly show the same match. Fields are
* only compared when both sides read them, so a null read never splits.
*/
export function sameScoreboardMatch(a: unknown, b: unknown): boolean {
const da = a as Partial<ScoreboardData> & Partial<ScoreboardReplayData>;
const db = b as Partial<ScoreboardData> & Partial<ScoreboardReplayData>;
const stageA = da.stage ?? null;
const stageB = db.stage ?? null;
if (stageA !== null && stageB !== null && stageA !== stageB) return false;
// replay detail screens: the recording timestamp and the replay code
// both identify the replay (timestamp reads are shape-validated, so a
// garbled read comes back null rather than as a different valid time)
const tsA = da.timestamp ?? null;
const tsB = db.timestamp ?? null;
if (tsA !== null && tsB !== null && tsA !== tsB) return false;
const codeA = da.replayCode ?? null;
const codeB = db.replayCode ?? null;
if (codeA !== null && codeB !== null && charDiff(codeA, codeB) >= CODE_DIFF_MIN) return false;
// same lobby, different map: names stay identical, but the paint totals
// are distinctive per match — near-zero overlap means a different board
const pa = paints(da);
const pb = paints(db);
if (pa.length >= PAINT_MIN_READ && pb.length >= PAINT_MIN_READ) {
if (multisetOverlap(pa, pb) < Math.min(pa.length, pb.length) / 2) return false;
}
return true;
}

220
app/features/cv/core/vod-matches.ts Normal file
View File

@@ -0,0 +1,220 @@
/**
* Group a detected-event timeline into per-match rows for sendou.ink's
* /ingest/vod endpoint (contract: sendou-ingest-endpoint.md), which builds a
* CAST-type VoD on /vods out of them.
*
* A VoD match becomes a `VideoMatch`: it needs a mode, a stage, a start
* timestamp to jump to in the YouTube embed, and the two teams' weapons.
* Casted broadcasts run their own between-map graphics (caster desk, stage
* pick, set score) instead of the native results/map-intro screens, so — apart
* from a POV VoD that happens to show them — the only native Splatoon UI is the
* in-match **spectator map screen**. Matches are therefore built primarily from
* the minimap, which shows all eight players' weapons and (via the planner
* signature) the stage.
*
* Because such footage carries no MapStart/Scoreboard events to delimit
* matches, minimaps are split into per-map matches by **stage change** and a
* **time gap** (a Splatoon game is only a few minutes, so minimaps far apart
* belong to different maps). A MapStart still opens a match and a scoreboard
* still closes one when present, and either supplies the mode/weapons then.
*
* The minimap cannot read the **mode**; for this PoC it is hard-coded to Splat
* Zones when no MapStart/Scoreboard supplied one. Weapons are left as the
* detector read them (sendou main-weapon ids, or null for a slot that never
* read); the endpoint validates them and skips any match missing a mode,
* stage, or a full set of weapons.
*/
import { MAP_START_EVENT_TYPE, type MapStartData } from "./detectors/map-start/index";
import { MINIMAP_EVENT_TYPE, type MinimapData } from "./detectors/minimap/index";
import { SCOREBOARD_EVENT_TYPES } from "./detectors/registry";
import type { ScoreboardData } from "./detectors/scoreboard/index";
import type { DetectedEvent } from "./detectors/types";
/**
* PoC: casted broadcasts never expose the mode to any detector, so minimap-only
* matches default to Splat Zones — flagged via `modeAssumed` so downstream can
* tell the guess from a real read. Replace with real mode detection later.
*/
const DEFAULT_MODE = "Splat Zones";
/**
* Two minimaps more than this far apart cannot be the same game (a Splatoon
* match runs a few minutes), so they open separate matches even on the same
* stage — the map-open the caster shows near a game's start and end still fall
* inside it.
*/
const MATCH_GAP_SECONDS = 300;
/** One VoD match as prefilled into sendou.ink's /vods/new form. */
export interface VodMatch {
/** whole seconds into the video the match starts at */
startsAt: number;
/** canonical English mode name; null when no source read it */
mode: string | null;
/**
* true when `mode` is the fabricated PoC default rather than a real
* read — lets the endpoint/form treat it as a guess, not a detection
*/
modeAssumed: boolean;
/** canonical English stage name; null when no source read it */
stage: string | null;
/**
* the match's weapons, alpha team then bravo team: sendou main-weapon
* ids, or null for a slot that never read
*/
weapons: (number | null)[];
}
/** A match being accumulated as the timeline is walked. */
interface OpenMatch {
mapStart: DetectedEvent | null;
firstMinimap: DetectedEvent | null;
minimaps: DetectedEvent[];
scoreboard: DetectedEvent | null;
/**
* per-stage read counts across the match's minimaps (a MapStart's stage
* seeds it); the plurality winner delimits same-vs-next map and is the
* reported stage, so one misread frame can't poison the whole match
*/
stageVotes: Map<string, number>;
/** t of the last minimap added, for the gap check */
lastMinimapT: number | null;
}
/** Plurality stage of the reads so far; insertion order breaks ties. */
function leadingStage(votes: Map<string, number>): string | null {
let winner: string | null = null;
let best = 0;
for (const [stage, count] of votes) {
if (count > best) {
winner = stage;
best = count;
}
}
return winner;
}
/**
* Splits a timeline into VoD matches. MapStart opens a match and a scoreboard
* closes one; between them (or with neither) minimaps are grouped per map by
* stage and time gap.
*/
export function buildVodMatches(events: readonly DetectedEvent[]): VodMatch[] {
const sorted = [...events].sort((a, b) => a.t - b.t);
const matches: VodMatch[] = [];
// For each minimap event, the next minimap's non-null stage read (walked
// backwards). A stage change only splits when the next read doesn't refute
// it: a lone frame disagreeing with both its match's running stage and the
// following read is a misread to fold in as a minority vote, not a match
// boundary. With no later read the change stands.
const nextStage = new Map<DetectedEvent, string | null>();
let carry: string | null = null;
for (let i = sorted.length - 1; i >= 0; i--) {
const event = sorted[i]!;
if (event.type !== MINIMAP_EVENT_TYPE) continue;
nextStage.set(event, carry);
carry = (event.data as MinimapData).stage ?? carry;
}
let open: OpenMatch | null = null;
const start = (): OpenMatch => ({
mapStart: null,
firstMinimap: null,
minimaps: [],
scoreboard: null,
stageVotes: new Map(),
lastMinimapT: null,
});
const vote = (votes: Map<string, number>, stage: string | null): void => {
if (stage !== null) votes.set(stage, (votes.get(stage) ?? 0) + 1);
};
const finalize = (): void => {
if (!open) return;
const match = toVodMatch(open);
if (match) matches.push(match);
open = null;
};
for (const event of sorted) {
if (event.type === MAP_START_EVENT_TYPE) {
finalize();
open = start();
open.mapStart = event;
vote(open.stageVotes, (event.data as MapStartData).stage ?? null);
} else if (SCOREBOARD_EVENT_TYPES.includes(event.type)) {
open ??= start();
open.scoreboard = event;
vote(open.stageVotes, (event.data as ScoreboardData).stage ?? null);
finalize();
} else if (event.type === MINIMAP_EVENT_TYPE) {
const stage = (event.data as MinimapData).stage;
if (open) {
const current = leadingStage(open.stageVotes);
const stageChanged =
current !== null &&
stage !== null &&
stage !== current &&
(nextStage.get(event) ?? stage) === stage;
const gapTooBig =
open.lastMinimapT !== null && event.t - open.lastMinimapT > MATCH_GAP_SECONDS;
if (stageChanged || gapTooBig) finalize();
}
open ??= start();
open.minimaps.push(event);
open.firstMinimap ??= event;
open.lastMinimapT = event.t;
vote(open.stageVotes, stage);
}
}
finalize();
return matches;
}
/** Builds a match, or null when it carries no weapons to show. */
function toVodMatch(open: OpenMatch): VodMatch | null {
const start = open.mapStart?.data as MapStartData | undefined;
const board = open.scoreboard?.data as ScoreboardData | undefined;
const weapons = board
? board.players.map((player) => player.weaponId)
: weaponsFromMinimaps(open.minimaps);
if (weapons.length === 0) return null;
const anchorT = open.mapStart?.t ?? open.firstMinimap?.t ?? open.scoreboard?.t ?? 0;
const readMode = start?.mode ?? board?.mode ?? null;
return {
startsAt: Math.max(0, Math.floor(anchorT)),
mode: readMode ?? DEFAULT_MODE,
modeAssumed: readMode === null,
stage: start?.stage ?? board?.stage ?? leadingStage(open.stageVotes),
weapons,
};
}
/**
* Merges the eight weapon slots (four alpha then four bravo) across a match's
* minimaps, taking the first frame that read each slot — weapons are fixed for
* a match, so a slot missed in one frame is filled from another. Empty when
* there were no minimaps.
*/
function weaponsFromMinimaps(minimaps: DetectedEvent[]): (number | null)[] {
if (minimaps.length === 0) return [];
const datas = minimaps.map((event) => event.data as MinimapData);
const alpha = mergeSlots(datas.map((d) => d.teammates.map((t) => t.weaponId)));
const bravo = mergeSlots(datas.map((d) => d.enemies.map((e) => e.weaponId)));
return [...alpha, ...bravo];
}
/** For each slot index, the first non-null id across frames, else null. */
function mergeSlots(frames: (number | null)[][]): (number | null)[] {
const width = Math.max(0, ...frames.map((frame) => frame.length));
const out: (number | null)[] = [];
for (let i = 0; i < width; i++) {
out.push(frames.map((frame) => frame[i]).find((id) => id != null) ?? null);
}
return out;
}

9
app/features/cv/node/assets-dir.ts Normal file
View File

@@ -0,0 +1,9 @@
/**
* Where Node-side code (tests, atlas builders) reads and writes the CV
* asset sets. Defaults to the sibling sendou-ink/assets checkout — the same
* files the CDN mirrors — and can be overridden with CV_ASSETS_DIR. The
* version segment must match worker-side CV_ASSETS_URL (app/utils/urls.ts).
*/
export const CV_ASSETS_DIR =
process.env.CV_ASSETS_DIR ??
new URL("../../../../../assets/assets/cv/v1", import.meta.url).pathname;

147
app/features/cv/node/fixtures.ts Normal file
View File

@@ -0,0 +1,147 @@
/**
* Fixture discovery and detector execution for tests and tools.
*
* A fixture is a directory under <repo>/tests/fixtures/<detector>/<case-name>/ containing
* frame.png or frame.jpg (raw capture, any resolution — normalization happens
* inside the pipeline under test) and expected.json.
*/
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import type { ScoreboardData } from "../core/detectors/scoreboard/index";
import type { DetectedEvent, Detector, GateResult } from "../core/detectors/types";
import { normalizeFrame, toMat } from "../core/image";
import { readImage } from "./image-io";
export const FIXTURES_DIR = new URL("../../tests/fixtures", import.meta.url).pathname;
interface ExpectedPlayer {
name?: string;
weaponId?: number | null;
paint?: number;
ka?: number;
d?: number;
s?: number;
}
interface ExpectedMinimapTeammate {
slot?: "up" | "left" | "right" | "self";
name?: string | null;
weapon?: string | null;
weaponId?: number | null;
abilities?: (string | null)[];
}
interface ExpectedMinimapEnemy {
/** spectator frames only: the screen shows bravo-team names */
name?: string | null;
weapon?: string | null;
weaponId?: number | null;
abilities?: (string | null)[];
}
interface ExpectedScoreboard {
event:
| "Scoreboard"
| "ScoreboardReplay"
| "ScoreboardOwn"
| "Death"
| "MapStart"
| "Minimap"
| "none";
data?: {
lobby?: string;
mode?: string;
stage?: string;
/** ScoreboardReplay only */
timestamp?: string;
/** ScoreboardReplay only */
replayCode?: string;
scores?: [number, number];
/** ScoreboardReplay only: the "Score:" banner values */
matchScores?: [number, number];
players?: ExpectedPlayer[];
/** index of the yellow POV-arrow row in `players`; null = no arrow */
povIndex?: number | null;
/**
* Death: killer's weapon (English name), its id, and its kind.
* ScoreboardOwn: the player's own main weapon (weaponType unused).
*/
weapon?: string;
weaponId?: number | null;
weaponType?: "MAIN" | "SUB" | "SPECIAL";
/** Death + ScoreboardOwn: 3 gear rows of [main, sub, sub, sub] ability ids */
abilities?: string[][];
/** Death only: killer's splash-tag name */
name?: string;
/** Minimap only: casted 8-player spectator map screen (not parsed yet) */
spectator?: boolean;
/** Minimap only: own-team callout cards in slot order */
teammates?: ExpectedMinimapTeammate[];
/** Minimap only: enemy panel rows, top to bottom */
enemies?: ExpectedMinimapEnemy[];
};
options?: {
/** glob-ish field paths to skip, e.g. "players.*.name", "scores" */
skipFields?: string[];
/** free-form context for humans (why fields are skipped, capture quirks) */
notes?: string;
};
}
export interface Fixture {
name: string;
dir: string;
framePath: string;
expected: ExpectedScoreboard;
}
export function loadFixtures(detector: string): Fixture[] {
const root = join(FIXTURES_DIR, detector);
if (!existsSync(root)) return [];
return readdirSync(root, { withFileTypes: true })
.filter((e) => e.isDirectory())
.map((e) => {
const dir = join(root, e.name);
const framePath = ["frame.png", "frame.jpg", "frame.jpeg"]
.map((f) => join(dir, f))
.find(existsSync);
if (!framePath) throw new Error(`fixture ${e.name}: no frame.png/jpg`);
const expected = JSON.parse(
readFileSync(join(dir, "expected.json"), "utf8"),
) as ExpectedScoreboard;
return { name: e.name, dir, framePath, expected };
})
.sort((a, b) => a.name.localeCompare(b.name));
}
export function isFieldSkipped(fixture: Fixture, field: string): boolean {
const skips = fixture.expected.options?.skipFields ?? [];
return skips.some((pattern) => {
const re = new RegExp(
`^${pattern
.split("*")
.map((s) => s.replace(/[.+?^${}()|[\]\\]/g, "\\$&"))
.join("[^.]*")}$`,
);
return re.test(field);
});
}
export interface FixtureRun<TData = ScoreboardData> {
gate: GateResult;
events: DetectedEvent<TData>[];
}
/** Run gate+parse the way the live pipeline would. Caller must have loaded OpenCV. */
export async function runDetectorOnFixture<TData = ScoreboardData>(
detector: Detector<TData>,
fixture: Fixture,
): Promise<FixtureRun<TData>> {
const src = toMat(await readImage(fixture.framePath));
const frame = normalizeFrame(src);
src.delete();
const gate = detector.gate(frame);
const events = gate.pass ? detector.parse(frame, 0, gate) : [];
frame.delete();
return { gate, events };
}

29
app/features/cv/node/image-io.ts Normal file
View File

@@ -0,0 +1,29 @@
/**
* Node-only image decode/encode (@napi-rs/canvas). Never imported from src/core.
*
* Decoding goes through a canvas, whose backing store is alpha-premultiplied;
* RGB at partial-alpha pixels can shift by ±1. Everything we read is either
* fully opaque (frames, atlases) or consumed premultiplied anyway (weapon
* icons are composited over a background), so this is lossless in practice.
*/
import { readFileSync, writeFileSync } from "node:fs";
import { createCanvas, Image, ImageData } from "@napi-rs/canvas";
import type { FrameData } from "../core/image";
export async function readImage(path: string): Promise<FrameData> {
const img = new Image();
img.src = readFileSync(path);
await img.decode();
const canvas = createCanvas(img.width, img.height);
const ctx = canvas.getContext("2d");
ctx.drawImage(img, 0, 0);
const { width, height, data } = ctx.getImageData(0, 0, img.width, img.height);
return { width, height, data };
}
export function writePng(path: string, frame: FrameData): void {
const canvas = createCanvas(frame.width, frame.height);
const ctx = canvas.getContext("2d");
ctx.putImageData(new ImageData(frame.data, frame.width, frame.height), 0, 0);
writeFileSync(path, canvas.toBuffer("image/png"));
}

55
app/features/cv/node/resources.ts Normal file
View File

@@ -0,0 +1,55 @@
/**
* Node IO for ScoreboardResources: reads the CV asset sets from the local
* sendou-ink/assets checkout (tests and atlas builders never touch the
* CDN). What the bundle contains — every key, template option set, and
* atlas name — lives in core/resources.ts, shared with the worker's HTTP
* loader.
*/
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import {
loadPlannerStages,
type PlannerManifest,
type PlannerStage,
} from "../core/detectors/minimap/stage";
import type { ScoreboardResources } from "../core/detectors/scoreboard/index";
import { type AtlasMeta, type GlyphSet, loadGlyphSet } from "../core/glyphs";
import { assembleScoreboardResources } from "../core/resources";
import { readImage } from "./image-io";
import { CV_ASSETS_DIR as ASSETS_DIR } from "./assets-dir";
/** Decode eagerly, defer the (CPU-heavy) glyph slicing to first access. */
async function loadAtlasLazy(name: string): Promise<() => GlyphSet | null> {
const png = join(ASSETS_DIR, "glyphs", `${name}.png`);
const json = join(ASSETS_DIR, "glyphs", `${name}.json`);
if (!existsSync(png) || !existsSync(json)) return () => null;
const meta = JSON.parse(readFileSync(json, "utf8")) as AtlasMeta;
const image = await readImage(png);
let set: GlyphSet | null = null;
return () => (set ??= loadGlyphSet(image, meta));
}
/** Planner stage signatures; the (CPU) tile slicing runs on first access. */
async function loadPlannerStagesLazy(): Promise<() => PlannerStage[] | null> {
const png = join(ASSETS_DIR, "planner", "signatures.png");
const json = join(ASSETS_DIR, "planner", "manifest.json");
if (!existsSync(png) || !existsSync(json)) return () => null;
const manifest = JSON.parse(readFileSync(json, "utf8")) as PlannerManifest;
const atlas = await readImage(png);
let stages: PlannerStage[] | null = null;
return () => (stages ??= loadPlannerStages(atlas, manifest));
}
/** Requires loadOpenCV() to have resolved. */
export function loadScoreboardResources(): Promise<ScoreboardResources> {
return assembleScoreboardResources({
readManifest: (dir) =>
Promise.resolve(
JSON.parse(readFileSync(join(ASSETS_DIR, dir, "manifest.json"), "utf8")) as string[],
),
readIcon: (dir, id) => readImage(join(ASSETS_DIR, dir, `${id}.png`)),
loadAtlas: loadAtlasLazy,
loadPlannerStages: loadPlannerStagesLazy,
});
}

110
app/features/cv/store/db.ts Normal file
View File

@@ -0,0 +1,110 @@
/**
* Shared IndexedDB handle for the app's stores:
* - `events`: live-tab detections, keyed by auto id (see events.ts)
* - `frames`: the live events' full-res analyzed PNGs, keyed by event id —
* kept out of `events` so listing the feed never deserializes them
* - `vods`: one summary record per fully scanned VoD, keyed by file name
* - `vod-events`: the detections of each saved VoD, indexed by VoD name
* - `vod-frames`: the vod-events' PNGs, keyed by vod-event id
*/
const DB_NAME = "vod-parser";
const DB_VERSION = 3;
export const EVENTS_STORE = "events";
export const FRAMES_STORE = "frames";
export const VODS_STORE = "vods";
export const VOD_EVENTS_STORE = "vod-events";
export const VOD_FRAMES_STORE = "vod-frames";
/**
* Move a store's embedded `frame` blobs into a keyed frame store (v3
* migration), stamping `hasFrame` on the source records.
*/
function extractFrames(source: IDBObjectStore, frames: IDBObjectStore): void {
const req = source.openCursor();
req.onsuccess = () => {
const cursor = req.result;
if (!cursor) return;
const record = cursor.value as { frame?: Blob; hasFrame?: boolean };
if (record.frame) {
frames.put(record.frame, cursor.primaryKey);
record.hasFrame = true;
delete record.frame;
cursor.update(record);
}
cursor.continue();
};
}
/**
* Versioned migrations: each `oldVersion < N` block upgrades a database from
* below version N and runs exactly once per database. Any schema change —
* including one to an EXISTING store (new index, moved field) — must be a new
* block plus a DB_VERSION bump, never an edit to an old block: databases that
* already ran the old block would silently skip the change otherwise.
*/
function migrate(database: IDBDatabase, transaction: IDBTransaction, oldVersion: number): void {
if (oldVersion < 2) {
// v1/v2 era stores; contains() guards absorb the pre-versioned scheme,
// where every creation was unconditionally contains()-gated
if (!database.objectStoreNames.contains(EVENTS_STORE)) {
const store = database.createObjectStore(EVENTS_STORE, {
keyPath: "id",
autoIncrement: true,
});
store.createIndex("t", "t");
store.createIndex("detectedAt", "detectedAt");
}
if (!database.objectStoreNames.contains(VODS_STORE)) {
database.createObjectStore(VODS_STORE, { keyPath: "name" });
}
if (!database.objectStoreNames.contains(VOD_EVENTS_STORE)) {
const store = database.createObjectStore(VOD_EVENTS_STORE, {
keyPath: "id",
autoIncrement: true,
});
store.createIndex("vod", "vod");
}
}
if (oldVersion < 3) {
// frame blobs move out of the event records into keyed frame stores
const frames = database.createObjectStore(FRAMES_STORE);
const vodFrames = database.createObjectStore(VOD_FRAMES_STORE);
extractFrames(transaction.objectStore(EVENTS_STORE), frames);
extractFrames(transaction.objectStore(VOD_EVENTS_STORE), vodFrames);
}
}
function openDb(): Promise<IDBDatabase> {
return new Promise((resolve, reject) => {
const req = indexedDB.open(DB_NAME, DB_VERSION);
req.onupgradeneeded = (event) => {
migrate(req.result, req.transaction!, event.oldVersion);
};
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
});
}
let dbPromise: Promise<IDBDatabase> | null = null;
export function db(): Promise<IDBDatabase> {
dbPromise ??= openDb();
return dbPromise;
}
/** Single-request convenience wrapper over one object store. */
export function tx<T>(
storeName: string,
mode: IDBTransactionMode,
run: (store: IDBObjectStore) => IDBRequest<T>,
): Promise<T> {
return db().then(
(database) =>
new Promise<T>((resolve, reject) => {
const transaction = database.transaction(storeName, mode);
const req = run(transaction.objectStore(storeName));
req.onsuccess = () => resolve(req.result);
req.onerror = () => reject(req.error);
}),
);
}

142
app/features/cv/store/events.ts Normal file
View File

@@ -0,0 +1,142 @@
/**
* IndexedDB event store: one `events` object store, keyed by auto id,
* indexed by timestamp. A small thumbnail of the detection frame is kept
* per event for the feed; the full-res analyzed PNG lives in the separate
* `frames` store under the same id (loaded on demand via loadEventFrame),
* so listing the feed never deserializes megabytes of blobs. The store is
* capped: saving past MAX_EVENTS evicts the oldest events and their frames.
*/
import type { DetectedEvent } from "../core/detectors/types";
import { db, EVENTS_STORE, FRAMES_STORE, tx } from "./db";
/** Oldest events (and their frames) are evicted past this count. */
const MAX_EVENTS = 1000;
/** Where an event stands with sendou.ink /ingest; absent = never attempted. */
export interface SendStatus {
state: "queued" | "sending" | "sent" | "failed";
/** wall-clock time of the last state change */
at: number;
/** failure detail, set when state is "failed" */
error?: string;
}
export interface StoredEvent {
id?: number;
type: string;
t: number;
/** wall-clock time of detection */
detectedAt: number;
confidence: number;
data: unknown;
/** small JPEG data URL of the source frame */
thumbnail?: string;
/** whether a full-res frame exists in the `frames` store under this id */
hasFrame?: boolean;
send?: SendStatus;
}
export function saveEvent(event: DetectedEvent, thumbnail?: string, frame?: Blob): Promise<number> {
const record: StoredEvent = {
type: event.type,
t: event.t,
detectedAt: Date.now(),
confidence: event.confidence,
data: event.data,
thumbnail,
hasFrame: frame !== undefined,
};
return db().then(
(database) =>
new Promise<number>((resolve, reject) => {
const transaction = database.transaction([EVENTS_STORE, FRAMES_STORE], "readwrite");
const events = transaction.objectStore(EVENTS_STORE);
const frames = transaction.objectStore(FRAMES_STORE);
let id: number;
const add = events.add(record) as IDBRequest<number>;
add.onsuccess = () => {
id = add.result;
if (frame) frames.put(frame, id);
evictOldest(events, frames);
};
transaction.oncomplete = () => resolve(id);
transaction.onerror = () => reject(transaction.error);
}),
);
}
/** Delete records (and frames) beyond MAX_EVENTS, oldest ids first. */
function evictOldest(events: IDBObjectStore, frames: IDBObjectStore): void {
const count = events.count();
count.onsuccess = () => {
let excess = count.result - MAX_EVENTS;
if (excess <= 0) return;
const cursor = events.openCursor(); // ascending id = oldest first
cursor.onsuccess = () => {
const c = cursor.result;
if (!c || excess <= 0) return;
frames.delete(c.primaryKey);
c.delete();
excess--;
if (excess > 0) c.continue();
};
};
}
/** Sets (or clears) the send status of the given events in one transaction. */
export function updateEventsSend(ids: number[], send: SendStatus | undefined): Promise<void> {
return db().then(
(database) =>
new Promise<void>((resolve, reject) => {
const transaction = database.transaction(EVENTS_STORE, "readwrite");
const events = transaction.objectStore(EVENTS_STORE);
for (const id of ids) {
const get = events.get(id) as IDBRequest<StoredEvent | undefined>;
get.onsuccess = () => {
const record = get.result;
if (!record) return; // evicted meanwhile
if (send) record.send = send;
else delete record.send;
events.put(record);
};
}
transaction.oncomplete = () => resolve();
transaction.onerror = () => reject(transaction.error);
}),
);
}
export function deleteEvent(id: number): Promise<void> {
return db().then(
(database) =>
new Promise<void>((resolve, reject) => {
const transaction = database.transaction([EVENTS_STORE, FRAMES_STORE], "readwrite");
transaction.objectStore(EVENTS_STORE).delete(id);
transaction.objectStore(FRAMES_STORE).delete(id);
transaction.oncomplete = () => resolve();
transaction.onerror = () => reject(transaction.error);
}),
);
}
export function listEvents(): Promise<StoredEvent[]> {
return tx(EVENTS_STORE, "readonly", (store) => store.getAll() as IDBRequest<StoredEvent[]>);
}
/** The event's full-res analyzed PNG, or undefined when none was stored. */
export function loadEventFrame(id: number): Promise<Blob | undefined> {
return tx(FRAMES_STORE, "readonly", (store) => store.get(id) as IDBRequest<Blob | undefined>);
}
export function clearEvents(): Promise<void> {
return db().then(
(database) =>
new Promise<void>((resolve, reject) => {
const transaction = database.transaction([EVENTS_STORE, FRAMES_STORE], "readwrite");
transaction.objectStore(EVENTS_STORE).clear();
transaction.objectStore(FRAMES_STORE).clear();
transaction.oncomplete = () => resolve();
transaction.onerror = () => reject(transaction.error);
}),
);
}

129
app/features/cv/store/vods.ts Normal file
View File

@@ -0,0 +1,129 @@
/**
* Persistence for completed VoD scans, keyed by the VoD's file name so a
* scanned video can be reinspected later without re-decoding it. The
* summary lives in `vods`; the detections live in `vod-events` under a
* `vod` index, and their full-res analyzed PNGs in `vod-frames` under the
* event id (loaded on demand via loadVodEventFrame), so listing VoDs and
* their events stays cheap. Re-scanning the same file name overwrites the
* previous save.
*/
import { db, tx, VOD_EVENTS_STORE, VOD_FRAMES_STORE, VODS_STORE } from "./db";
export interface VodSummary {
/** VoD file name — primary key */
name: string;
/** wall-clock time the scan finished */
savedAt: number;
/** video duration in seconds */
duration: number;
eventCount: number;
}
export interface StoredVodEvent {
id?: number;
/** owning VoD name (indexed) */
vod: string;
type: string;
t: number;
confidence: number;
data: unknown;
/** small JPEG data URL of the source frame */
thumbnail?: string;
/** whether a full-res frame exists in `vod-frames` under this id */
hasFrame?: boolean;
}
/** A vod-event to persist, with its (separately stored) frame attached. */
export type VodEventToSave = Omit<StoredVodEvent, "id" | "vod" | "hasFrame"> & {
frame?: Blob;
};
/** Delete every vod-event (and frame) of `name` via the index, then run `next`. */
function clearVodEvents(
events: IDBObjectStore,
frames: IDBObjectStore,
name: string,
next: () => void,
): void {
const req = events.index("vod").openCursor(IDBKeyRange.only(name));
req.onsuccess = () => {
const cursor = req.result;
if (cursor) {
frames.delete(cursor.primaryKey);
cursor.delete();
cursor.continue();
} else {
next();
}
};
}
export async function saveVod(
meta: Omit<VodSummary, "eventCount">,
events: VodEventToSave[],
): Promise<void> {
const database = await db();
return new Promise((resolve, reject) => {
const transaction = database.transaction(
[VODS_STORE, VOD_EVENTS_STORE, VOD_FRAMES_STORE],
"readwrite",
);
const eventStore = transaction.objectStore(VOD_EVENTS_STORE);
const frameStore = transaction.objectStore(VOD_FRAMES_STORE);
clearVodEvents(eventStore, frameStore, meta.name, () => {
for (const { frame, ...event } of events) {
const add = eventStore.add({
...event,
vod: meta.name,
hasFrame: frame !== undefined,
}) as IDBRequest<number>;
if (frame) add.onsuccess = () => frameStore.put(frame, add.result);
}
transaction.objectStore(VODS_STORE).put({ ...meta, eventCount: events.length });
});
transaction.oncomplete = () => resolve();
transaction.onerror = () => reject(transaction.error);
});
}
export async function listVods(): Promise<VodSummary[]> {
const vods = await tx(
VODS_STORE,
"readonly",
(store) => store.getAll() as IDBRequest<VodSummary[]>,
);
return vods.sort((a, b) => b.savedAt - a.savedAt);
}
export async function loadVodEvents(name: string): Promise<StoredVodEvent[]> {
const events = await tx(
VOD_EVENTS_STORE,
"readonly",
(store) => store.index("vod").getAll(IDBKeyRange.only(name)) as IDBRequest<StoredVodEvent[]>,
);
return events.sort((a, b) => a.t - b.t);
}
/** The vod-event's full-res analyzed PNG, or undefined when none was stored. */
export function loadVodEventFrame(id: number): Promise<Blob | undefined> {
return tx(VOD_FRAMES_STORE, "readonly", (store) => store.get(id) as IDBRequest<Blob | undefined>);
}
export async function deleteVod(name: string): Promise<void> {
const database = await db();
return new Promise((resolve, reject) => {
const transaction = database.transaction(
[VODS_STORE, VOD_EVENTS_STORE, VOD_FRAMES_STORE],
"readwrite",
);
transaction.objectStore(VODS_STORE).delete(name);
clearVodEvents(
transaction.objectStore(VOD_EVENTS_STORE),
transaction.objectStore(VOD_FRAMES_STORE),
name,
() => {},
);
transaction.oncomplete = () => resolve();
transaction.onerror = () => reject(transaction.error);
});
}

80
app/features/cv/worker/analyzer.worker.ts Normal file
View File

@@ -0,0 +1,80 @@
/**
* AnalyzerWorker: owns OpenCV.js (WASM) and the detector registry.
* The main thread posts ImageBitmaps; results come back as plain JSON.
*/
import { loadOpenCV } from "../core/cv";
import { createAllDetectors } from "../core/detectors/registry";
import { ParseSuppressor } from "../core/detectors/suppressor";
import type { Detector } from "../core/detectors/types";
import { normalizeFrame, toMat } from "../core/image";
import type { AnalyzeRequest, InitRequest, WorkerResponse } from "./protocol";
import { fetchScoreboardResources } from "./resources";
let detectors: Detector<unknown>[] = [];
let suppressor: ParseSuppressor | null = null;
function post(message: WorkerResponse): void {
self.postMessage(message);
}
async function init({ assetsBaseUrl, suppressSteadyFrames = true }: InitRequest): Promise<void> {
try {
await loadOpenCV();
const resources = await fetchScoreboardResources(assetsBaseUrl);
detectors = createAllDetectors(resources);
suppressor = suppressSteadyFrames ? new ParseSuppressor() : null;
post({ kind: "ready" });
} catch (error) {
post({ kind: "error", message: `init failed: ${String(error)}` });
}
}
async function analyze({ bitmap, t }: AnalyzeRequest): Promise<void> {
const width = "displayWidth" in bitmap ? bitmap.displayWidth : bitmap.width;
const height = "displayHeight" in bitmap ? bitmap.displayHeight : bitmap.height;
const canvas = new OffscreenCanvas(width, height);
const ctx = canvas.getContext("2d")!;
ctx.drawImage(bitmap, 0, 0);
bitmap.close();
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
const src = toMat({
width: imageData.width,
height: imageData.height,
data: imageData.data,
});
let frame: ReturnType<typeof normalizeFrame>;
try {
frame = normalizeFrame(src);
} finally {
src.delete();
}
// On detection, ship back the exact analyzed pixels (lossless, at capture
// resolution) so the UI never has to re-grab a later frame — encoded at
// most once per frame, however many detectors fire on it.
let encoded: Promise<Blob> | null = null;
const frameBlob = () => (encoded ??= canvas.convertToBlob({ type: "image/png" }));
try {
for (const detector of detectors) {
const gate = detector.gate(frame);
const runParse = suppressor ? suppressor.shouldParse(detector.id, gate.pass) : gate.pass;
const events = runParse ? detector.parse(frame, t, gate) : [];
if (runParse) suppressor?.recordParse(detector.id, events);
const blob = events.length > 0 ? await frameBlob() : undefined;
post({ kind: "result", detector: detector.id, t, gate, events, frame: blob });
}
} catch (error) {
post({ kind: "error", message: `analyze failed: ${String(error)}` });
} finally {
frame.delete();
post({ kind: "done", t });
}
}
self.onmessage = (e: MessageEvent) => {
const msg = e.data as { kind: string } & Record<string, unknown>;
if (msg.kind === "init") void init(msg as unknown as InitRequest);
else if (msg.kind === "frame") void analyze(msg as unknown as AnalyzeRequest);
};

93
app/features/cv/worker/client.ts Normal file
View File

@@ -0,0 +1,93 @@
/**
* Main-thread wrapper around the AnalyzerWorker: init handshake, one
* in-flight frame at a time (the sampler drops frames while busy). Each
* frame yields one result per registered detector, then a single "done".
*/
import { CV_ASSETS_URL } from "../../../utils/urls";
import type { WorkerResponse } from "./protocol";
export type ResultHandler = (result: Extract<WorkerResponse, { kind: "result" }>) => void;
export type ErrorHandler = (message: string) => void;
export type DoneHandler = (t: number) => void;
export class AnalyzerClient {
#worker: Worker;
#ready = false;
#busy = false;
#onResult: ResultHandler;
#onError: ErrorHandler;
#onDone: DoneHandler | undefined;
#readyPromise: Promise<void>;
constructor(
onResult: ResultHandler,
onError: ErrorHandler = console.error,
onDone?: DoneHandler,
options: { suppressSteadyFrames?: boolean } = {},
) {
this.#onResult = onResult;
this.#onError = onError;
this.#onDone = onDone;
this.#worker = new Worker(new URL("./analyzer.worker.ts", import.meta.url), {
type: "module",
});
let resolveReady!: () => void;
this.#readyPromise = new Promise((resolve) => {
resolveReady = resolve;
});
this.#worker.onmessage = (e: MessageEvent<WorkerResponse>) => {
const msg = e.data;
if (msg.kind === "ready") {
this.#ready = true;
resolveReady();
} else if (msg.kind === "result") {
this.#onResult(msg);
} else if (msg.kind === "done") {
this.#busy = false;
this.#onDone?.(msg.t);
} else if (msg.kind === "error") {
this.#busy = false;
this.#onError(msg.message);
}
};
// A throw outside the worker's own try/catch posts neither "error" nor
// "done"; without these handlers `busy` would stay true forever and the
// sampler / VoD scan would silently freeze.
this.#worker.onerror = (e: ErrorEvent) => {
this.#busy = false;
this.#onError(`worker error: ${e.message || String(e)}`);
};
this.#worker.onmessageerror = () => {
this.#busy = false;
this.#onError("worker message deserialization failed");
};
this.#worker.postMessage({
kind: "init",
assetsBaseUrl: CV_ASSETS_URL,
suppressSteadyFrames: options.suppressSteadyFrames ?? true,
});
}
whenReady(): Promise<void> {
return this.#readyPromise;
}
get busy(): boolean {
return this.#busy || !this.#ready;
}
/** Returns false (and closes the bitmap) if the worker is still busy. */
analyze(bitmap: ImageBitmap | VideoFrame, t: number): boolean {
if (this.busy) {
bitmap.close();
return false;
}
this.#busy = true;
this.#worker.postMessage({ kind: "frame", bitmap, t }, [bitmap]);
return true;
}
dispose(): void {
this.#worker.terminate();
}
}

76
app/features/cv/worker/pool.ts Normal file
View File

@@ -0,0 +1,76 @@
/**
* AnalyzerPool: several AnalyzerClients so frame analysis parallelizes
* across cores. The VoD scan hands each decoded frame to any idle worker
* and never waits; when all workers are busy the frame is dropped — the
* next one is milliseconds of video away, so coverage stays as dense as
* the machine can analyze. Results arrive out of decode order, which the
* consumer must tolerate (TimelineBuilder does: it merges on |Δt| and
* keeps its list sorted, independent of arrival order).
*/
import { AnalyzerClient, type ErrorHandler, type ResultHandler } from "./client";
/** leave cores for the main thread and the video decoder */
export function defaultPoolSize(): number {
return Math.min(4, Math.max(1, (navigator.hardwareConcurrency || 4) - 2));
}
export class AnalyzerPool {
#clients: AnalyzerClient[];
#idleWaiters: (() => void)[] = [];
constructor(size: number, onResult: ResultHandler, onError: ErrorHandler) {
const wake = () => {
const waiters = this.#idleWaiters;
this.#idleWaiters = [];
for (const waiter of waiters) waiter();
};
this.#clients = Array.from(
{ length: size },
() =>
new AnalyzerClient(
onResult,
(message) => {
wake(); // an errored frame is also a finished frame — don't hang whenIdle
onError(message);
},
wake,
),
);
}
whenReady(): Promise<void> {
return Promise.all(this.#clients.map((c) => c.whenReady())).then(() => {});
}
hasIdle(): boolean {
return this.#clients.some((c) => !c.busy);
}
/** Resolves once at least one worker is free. Call whenReady() first. */
async whenAnyIdle(): Promise<void> {
while (!this.hasIdle()) {
await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
}
}
/** Hand the frame to an idle worker; false (and the frame closed) when all are busy. */
tryAnalyze(frame: ImageBitmap | VideoFrame, t: number): boolean {
const idle = this.#clients.find((c) => !c.busy);
if (!idle) {
frame.close();
return false;
}
return idle.analyze(frame, t);
}
/** Resolves once no worker has a frame in flight. Call whenReady() first. */
async whenIdle(): Promise<void> {
while (this.#clients.some((c) => c.busy)) {
await new Promise<void>((resolve) => this.#idleWaiters.push(resolve));
}
}
dispose(): void {
for (const client of this.#clients) client.dispose();
}
}

41
app/features/cv/worker/protocol.ts Normal file
View File

@@ -0,0 +1,41 @@
import type { DetectedEvent, GateResult } from "../core/detectors/types";
export interface InitRequest {
kind: "init";
/**
* base URL the worker fetches atlases/templates from (e.g.
* `${Config.staticAssetsUrl}/cv/v1`); passed in the init message so the
* worker bundle stays free of the app config graph
*/
assetsBaseUrl: string;
/**
* skip parse() for a detector whose gate keeps firing without confidence
* improving (static screen); default true — one-shot consumers like the
* screenshot harness turn it off
*/
suppressSteadyFrames?: boolean;
}
export interface AnalyzeRequest {
kind: "frame";
/** VideoFrame is what VoD decode produces; transferring it directly skips
* a main-thread ImageBitmap conversion */
bitmap: ImageBitmap | VideoFrame;
/** seconds into the stream */
t: number;
}
export type WorkerResponse =
| { kind: "ready" }
| {
kind: "result";
detector: string;
t: number;
gate: GateResult;
events: DetectedEvent<unknown>[];
/** lossless PNG of the exact frame that was analyzed; present when events fired */
frame?: Blob;
}
/** all detectors have reported for frame t */
| { kind: "done"; t: number }
| { kind: "error"; message: string };

77
app/features/cv/worker/resources.ts Normal file
View File

@@ -0,0 +1,77 @@
/**
* Worker/browser IO for ScoreboardResources: fetches the CDN-hosted CV
* assets (assets repo `assets/cv/v1/**`) over HTTP. What the bundle
* contains — every key, template option set, and atlas name — lives in
* core/resources.ts, shared with the Node loader. The base URL arrives via
* the worker init message (see worker/protocol.ts) so this module never
* imports the app config.
*/
import {
loadPlannerStages,
type PlannerManifest,
type PlannerStage,
} from "../core/detectors/minimap/stage";
import type { ScoreboardResources } from "../core/detectors/scoreboard/index";
import { type AtlasMeta, type GlyphSet, loadGlyphSet } from "../core/glyphs";
import type { FrameData } from "../core/image";
import { assembleScoreboardResources } from "../core/resources";
async function fetchImage(url: string): Promise<FrameData> {
const res = await fetch(url);
if (!res.ok) throw new Error(`fetch ${url}: ${res.status}`);
const bitmap = await createImageBitmap(await res.blob());
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
const ctx = canvas.getContext("2d")!;
ctx.drawImage(bitmap, 0, 0);
bitmap.close();
const data = ctx.getImageData(0, 0, canvas.width, canvas.height);
return { width: data.width, height: data.height, data: data.data };
}
function makeFetchAtlas(base: string) {
return async function fetchAtlas(name: string): Promise<() => GlyphSet | null> {
try {
const [meta, image] = await Promise.all([
fetch(`${base}/glyphs/${name}.json`).then((r) => {
if (!r.ok) throw new Error(String(r.status));
return r.json() as Promise<AtlasMeta>;
}),
fetchImage(`${base}/glyphs/${name}.png`),
]);
const set = loadGlyphSet(image, meta);
return () => set;
} catch {
return () => null;
}
};
}
function makeFetchPlannerStages(base: string) {
return async function fetchPlannerStages(): Promise<() => PlannerStage[] | null> {
try {
const [manifest, atlas] = await Promise.all([
fetch(`${base}/planner/manifest.json`).then((r) => {
if (!r.ok) throw new Error(String(r.status));
return r.json() as Promise<PlannerManifest>;
}),
fetchImage(`${base}/planner/signatures.png`),
]);
const stages = loadPlannerStages(atlas, manifest);
return () => stages;
} catch {
return () => null;
}
};
}
/** Requires loadOpenCV() to have resolved. */
export function fetchScoreboardResources(base: string): Promise<ScoreboardResources> {
return assembleScoreboardResources({
readManifest: (dir) =>
fetch(`${base}/${dir}/manifest.json`).then((r) => r.json() as Promise<string[]>),
readIcon: (dir, id) => fetchImage(`${base}/${dir}/${id}.png`),
loadAtlas: makeFetchAtlas(base),
loadPlannerStages: makeFetchPlannerStages(base),
});
}

4
app/utils/urls.ts
View File

@@ -129,6 +129,10 @@ export const FIRST_PLACEMENT_ICON_PATH = `${STATIC_ASSETS_URL}/svg/placements/fi
export const SECOND_PLACEMENT_ICON_PATH = `${STATIC_ASSETS_URL}/svg/placements/second.svg`;
export const THIRD_PLACEMENT_ICON_PATH = `${STATIC_ASSETS_URL}/svg/placements/third.svg`;
/** CV parser atlases/templates; the version segment guards against CDN cache
* skew — bump it together with breaking atlas format changes */
export const CV_ASSETS_URL = `${STATIC_ASSETS_URL}/cv/v1`;
export const APP_ICON_URL = `${STATIC_ASSETS_URL}/img/app-icon.png`;
export const pwaSplashScreenImageUrl = (fileName: string) =>
`${STATIC_ASSETS_URL}/img/splash-screens/${fileName}`;