diff --git a/app/features/scanner/README.md b/app/features/scanner/README.md index c51e69b15..4d65b9a06 100644 --- a/app/features/scanner/README.md +++ b/app/features/scanner/README.md @@ -96,7 +96,23 @@ sequenceDiagram edge fakes the shoulder glow on the overhead map view's badge-less strip), with the same `time` value so the two reads pair downstream; its fixtures - live under `tests/fixtures/player-status/`. The builder additionally + live under `tests/fixtures/player-status/`. Within a side the strip's + slot order is the lobby seating, while the results scoreboard re-sorts + each team per game (attested in the sendou-triton VoD: strip [Planetz, + .52, Neo Splash, Snipewriter] vs rows [.52, Neo Splash, Snipewriter, + Planetz], and the orders differ per game while the seating holds) — so + every 5th counter read also samples a `StripWeapons` evidence event: a + ranked weapon-icon match per alive slot (the squid plate's team ink is + hue-knocked-out to flat grey first; splatted slots grey the render out + and are skipped). Single reads rank the true weapon top-1 only about + half the time; the builder aggregates them across the match — plus the + minimap cards' parsed weapons, whose column order mirrors the strip + seating (attested for the enemy column) — and takes the best-scoring of + the 24 slot→row assignments against the scoreboard's weapons + (`core/slot-row-assignment.ts`), falling back to as-drawn order on thin + or tied evidence. The POV overlay's teammate diamond follows neither + order and maps by card name instead. Strip-weapon fixtures live under + `tests/fixtures/strip-weapons/`. The builder additionally flips sub-2s dead-flag runs flanked by dense opposite reads — a splat outlasts the respawn wait, so those are misread blips (background ink bleeding through a crossed-out icon) — and bridges sub-10s not-ready diff --git a/app/features/scanner/components/EventCard.tsx b/app/features/scanner/components/EventCard.tsx index ba737af77..f4d8251d4 100644 --- a/app/features/scanner/components/EventCard.tsx +++ b/app/features/scanner/components/EventCard.tsx @@ -30,6 +30,10 @@ import { PLAYER_STATUS_EVENT_TYPE, type PlayerStatusData, } from "../core/detectors/objective/player-status"; +import { + STRIP_WEAPONS_EVENT_TYPE, + type StripWeaponsData, +} from "../core/detectors/objective/strip-weapons"; import type { ScoreboardData } from "../core/detectors/scoreboard/index"; import { SCOREBOARD_OWN_EVENT_TYPE, @@ -48,6 +52,7 @@ import { ObjectiveCard } from "./ObjectiveCard"; import { PlayerStatusCard } from "./PlayerStatusCard"; import { ScoreboardCard } from "./ScoreboardCard"; import { ScoreboardOwnCard } from "./ScoreboardOwnCard"; +import { StripWeaponsCard } from "./StripWeaponsCard"; export type GetFrame = () => Promise; @@ -157,6 +162,8 @@ function renderCard( ) : type === PLAYER_STATUS_EVENT_TYPE ? ( + ) : type === STRIP_WEAPONS_EVENT_TYPE ? ( + ) : ( b.sources)); - const ungroupedFeed = feed.filter((e) => !groupedEvents.has(e)); + // strip weapon evidence is assignment input, not a detection worth a card + const ungroupedFeed = feed.filter( + (e) => !groupedEvents.has(e) && e.type !== STRIP_WEAPONS_EVENT_TYPE, + ); const abilityMap = connectAbilities(feed); const stop = () => { @@ -430,7 +434,8 @@ export function LivePage({ const cardEvents = withoutRepeatEvents(built.sources).filter( (e) => e.type !== OBJECTIVE_EVENT_TYPE && - e.type !== PLAYER_STATUS_EVENT_TYPE, + e.type !== PLAYER_STATUS_EVENT_TYPE && + e.type !== STRIP_WEAPONS_EVENT_TYPE, ); const newest = built === builtMatches.at(-1); return ( diff --git a/app/features/scanner/components/StripWeaponsCard.tsx b/app/features/scanner/components/StripWeaponsCard.tsx new file mode 100644 index 000000000..a489ab977 --- /dev/null +++ b/app/features/scanner/components/StripWeaponsCard.tsx @@ -0,0 +1,55 @@ +import { + STRIP_WEAPONS_EVENT_TYPE, + type StripWeaponsData, +} from "../core/detectors/objective/strip-weapons"; +import styles from "./EventCard.module.css"; +import { FrameThumb } from "./FrameThumb"; +import { formatClock, useEventTimeFormatter } from "./format"; +import { mainWeaponLabel } from "./labels"; +import { MetaPills } from "./MetaChips"; + +export function StripWeaponsCard(props: { + t: number; + confidence: number; + data: StripWeaponsData; + thumbnail?: string; + detectedAt?: number; + /** lazy loader for the exact analyzed frame — enables fixture export */ + getFrame?: () => Promise; + onInspect?: () => void; +}) { + const { t, confidence, data, thumbnail, detectedAt, getFrame, onInspect } = + props; + const side = (index: 0 | 1) => + data.slots[index] + .map((candidates) => + candidates === null + ? "✕" + : (mainWeaponLabel(candidates[0]?.weaponId ?? null) ?? "?"), + ) + .join(" | "); + const formatDetectedAt = useEventTimeFormatter(); + return ( +
+
+ + + {data.time !== null ? `${formatClock(data.time)} · ` : null} + {side(0)} vs {side(1)} + + {detectedAt ? {formatDetectedAt(detectedAt)} : null} + +
+
+ ); +} diff --git a/app/features/scanner/components/VodPage.tsx b/app/features/scanner/components/VodPage.tsx index 527202516..9bec20eaf 100644 --- a/app/features/scanner/components/VodPage.tsx +++ b/app/features/scanner/components/VodPage.tsx @@ -27,6 +27,7 @@ import { openSeekScan, probeWebCodecs } from "../capture/vod-frames"; import { connectAbilities } from "../core/ability-harvest"; import { OBJECTIVE_EVENT_TYPE } from "../core/detectors/objective/index"; import { PLAYER_STATUS_EVENT_TYPE } from "../core/detectors/objective/player-status"; +import { STRIP_WEAPONS_EVENT_TYPE } from "../core/detectors/objective/strip-weapons"; import { mergeScanTelemetry, type ScanTelemetry, @@ -177,7 +178,11 @@ export function VodPage({ ); const vodMatchByEvent = new Map(matches.map((m) => [m.event, m] as const)); const groupedEvents = new Set(builtMatches.flatMap((b) => b.sources)); - const ungroupedMatches = matches.filter((m) => !groupedEvents.has(m.event)); + // strip weapon evidence is assignment input, not a detection worth a card + const ungroupedMatches = matches.filter( + (m) => + !groupedEvents.has(m.event) && m.event.type !== STRIP_WEAPONS_EVENT_TYPE, + ); // "Send results" sends the whole scan in one go, so its outcome maps // onto every ingestable card; a partial failure (some chunks sent, some @@ -750,7 +755,8 @@ export function VodPage({ const cardEvents = withoutRepeatEvents(built.sources).filter( (e) => e.type !== OBJECTIVE_EVENT_TYPE && - e.type !== PLAYER_STATUS_EVENT_TYPE, + e.type !== PLAYER_STATUS_EVENT_TYPE && + e.type !== STRIP_WEAPONS_EVENT_TYPE, ); return ( + candidates === null + ? "✕" + : (mainWeaponLabel(candidates[0]?.weaponId ?? null) ?? "?"), + ) + .join(" | "); +} + /** one team's four icons as ✕ splatted / ★ special ready / · alive */ function formatPlayerStatusSide(data: PlayerStatusData, side: 0 | 1): string { return data.dead[side] @@ -257,6 +272,25 @@ function eventCells(event: CsvEvent): Cell[] { "", ]; } + case STRIP_WEAPONS_EVENT_TYPE: { + const d = event.data as StripWeaponsData; + const clock = d.time === null ? "" : `${formatClock(d.time)} · `; + return [ + ...base, + "", + "", + "", + "", + "", + "", + "", + "", + "", + `${clock}${formatStripWeaponsSide(d, 0)} vs ${formatStripWeaponsSide(d, 1)} (${d.layout})`, + "", + "", + ]; + } case SCOREBOARD_EVENT_TYPE: case SCOREBOARD_BATTLE_LOG_EVENT_TYPE: case SCOREBOARD_BATTLE_LOG_REPLAY_EVENT_TYPE: { diff --git a/app/features/scanner/components/fixture-export.ts b/app/features/scanner/components/fixture-export.ts index df13ad2e1..62dc60c87 100644 --- a/app/features/scanner/components/fixture-export.ts +++ b/app/features/scanner/components/fixture-export.ts @@ -24,6 +24,10 @@ import { PLAYER_STATUS_EVENT_TYPE, type PlayerStatusData, } from "../core/detectors/objective/player-status"; +import { + STRIP_WEAPONS_EVENT_TYPE, + type StripWeaponsData, +} from "../core/detectors/objective/strip-weapons"; import type { ScoreboardData } from "../core/detectors/scoreboard/index"; import type { ScoreboardBattleLogReplayData } from "../core/detectors/scoreboard-battle-log-replay/index"; import { @@ -44,7 +48,8 @@ export type FixtureData = | ScoreboardOwnData | MinimapData | ObjectiveData - | PlayerStatusData; + | PlayerStatusData + | StripWeaponsData; function isDeath(_data: FixtureData, eventType: string): _data is DeathData { return eventType === DEATH_EVENT_TYPE; @@ -173,6 +178,29 @@ function buildExpectedJson( 2, )}\n`; } + if (eventType === STRIP_WEAPONS_EVENT_TYPE) { + const strip = data as StripWeaponsData; + return `${JSON.stringify( + { + event: eventType, + data: { + layout: strip.layout, + time: strip.time, + // the top candidate per slot; hand-correct to the true weapons + weapons: strip.slots.map((side) => + side.map((candidates) => candidates?.[0]?.weaponId ?? null), + ), + weaponLabels: strip.slots.map((side) => + side.map((candidates) => + mainWeaponLabel(candidates?.[0]?.weaponId ?? null), + ), + ), + }, + }, + null, + 2, + )}\n`; + } // NB: not a type-predicate helper — CardData is structurally assignable to // MapStartData, so a predicate would narrow the fall-through case to never if (eventType === MAP_START_EVENT_TYPE) { diff --git a/app/features/scanner/core/detectors/objective/index.ts b/app/features/scanner/core/detectors/objective/index.ts index 36f6b4910..82fd5ac64 100644 --- a/app/features/scanner/core/detectors/objective/index.ts +++ b/app/features/scanner/core/detectors/objective/index.ts @@ -67,6 +67,7 @@ import { SCORE_ROIS, SCORE_TEXT_HEIGHTS, STATUS_LAYOUT_STICKY_MAX_GAP_S, + STRIP_WEAPON_SAMPLE_INTERVAL, TIMER_BIN_THRESHOLD, TIMER_DARK_PROBES, TIMER_DIGIT_MIN_CONF, @@ -74,6 +75,7 @@ import { TIMER_DIGIT_ROI, TIMER_TEXT_HEIGHTS, } from "./rois"; +import { parseStripWeapons, type StripWeaponsData } from "./strip-weapons"; export type ObjectiveData = SplatZonesObjectiveData; @@ -137,9 +139,12 @@ interface SideRead { export function createObjectiveDetector( resources: ScoreboardResources, -): Detector { +): Detector { const cv = getCV(); let lastStatus: { layout: PlayerStatusLayout; t: number } | undefined; + // primed so the very first counter read samples — short matches and + // single-frame runs (fixtures) get evidence too + let readsSinceWeaponSample = STRIP_WEAPON_SAMPLE_INTERVAL; const scoreSets: GlyphSet[] = resources.paintDigits ? SCORE_TEXT_HEIGHTS.map((h) => @@ -345,7 +350,7 @@ export function createObjectiveDetector( function parse( frame: Mat, t: number, - ): DetectedEvent[] { + ): DetectedEvent[] { const gray = new cv.Mat(); cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY); @@ -384,6 +389,24 @@ export function createObjectiveDetector( ); lastStatus = { layout: playerStatus.data.layout, t }; + // sampled slot-identity evidence for the strip → scoreboard-row + // assignment; every read would re-measure fixed identities at full + // template-sweep cost + let stripWeapons: DetectedEvent | null = null; + readsSinceWeaponSample++; + if ( + resources.stripWeapons && + readsSinceWeaponSample >= STRIP_WEAPON_SAMPLE_INTERVAL + ) { + readsSinceWeaponSample = 0; + stripWeapons = parseStripWeapons( + frame, + t, + playerStatus.data, + resources.stripWeapons, + ); + } + const confidences = sides.flatMap((side) => [ ...(side.score.value !== null ? [side.score.confidence] : []), ...(side.penalty?.value != null ? [side.penalty.confidence] : []), @@ -416,6 +439,7 @@ export function createObjectiveDetector( }, }, playerStatus, + ...(stripWeapons ? [stripWeapons] : []), ]; } diff --git a/app/features/scanner/core/detectors/objective/rois.ts b/app/features/scanner/core/detectors/objective/rois.ts index 2d312ee45..40a57d2ed 100644 --- a/app/features/scanner/core/detectors/objective/rois.ts +++ b/app/features/scanner/core/detectors/objective/rois.ts @@ -280,6 +280,62 @@ export const STATUS_LAYOUT_STICKY_MARGIN = 0.04; */ export const STATUS_LAYOUT_STICKY_MAX_GAP_S = 30; +// ---- strip weapon-icon evidence (the StripWeapons event) ---- +// +// Each slot draws the player's weapon render over its squid plate; the +// match builder aggregates sampled per-slot candidate rankings across a +// match to solve the strip-slot → scoreboard-row assignment +// (strip-weapons.ts). Calibrated on the sendou-triton VoD (cast geometry +// on 720p footage upscaled to canonical space). + +/** + * Weapon search window relative to a slot center: generous enough to hold + * the render at either strip geometry (the render measures ~65px on the + * cast strip, smaller on POV) without swallowing a neighbor slot's art. + */ +export const STRIP_WEAPON_BOX = { dx: -50, y: 20, w: 100, h: 100 }; + +/** + * Template render heights to try inside the window; the attested cast + * strip draws renders at ~55-70px depending on the weapon's aspect. + */ +export const STRIP_WEAPON_TEMPLATE_SIZES = [44, 52, 60, 68, 76] as const; + +/** + * The flat grey the knocked-out plate pixels become and templates are + * composited over — mid-grey, so both dark barrels and white bodies keep + * contrast against it. + */ +export const STRIP_WEAPON_TEMPLATE_BACKGROUND = 90; + +/** Ink floor for the NCC coverage penalty over that background. */ +export const STRIP_WEAPON_INK_THRESHOLD = 140; + +/** + * A plate pixel: saturated and bright (the plate is drawn in team ink), + * within the hue band of the region's modal saturated hue. The spread and + * value floors sit under the modal-vote floors (+15 in strip-weapons.ts) + * so the knockout reaches the plate's dimmer edge pixels the vote skips. + */ +export const STRIP_WEAPON_KNOCKOUT_MIN_SPREAD = 55; +export const STRIP_WEAPON_KNOCKOUT_MIN_VALUE = 90; +export const STRIP_WEAPON_MAX_PLATE_HUE_DIST = 30; + +/** + * Candidates kept per slot: single reads only rank the true weapon top-1 + * about half the time on attested footage, but it lands in the top 8 in + * enough reads for the cross-match aggregate to decide. + */ +export const STRIP_WEAPON_TOP_K = 8; + +/** + * Every Nth successful counter read samples the strip weapons: identities + * are fixed per match, ~1 read/s makes ~20 samples over a short match — + * attested to assign correctly — and the full-atlas NCC sweep is too + * heavy to run on every read. + */ +export const STRIP_WEAPON_SAMPLE_INTERVAL = 5; + /** * Cast-layout discriminator: the spectator HUD always draws white camera * badges under the right team's icons; nothing fixed sits there on POV diff --git a/app/features/scanner/core/detectors/objective/strip-weapons.ts b/app/features/scanner/core/detectors/objective/strip-weapons.ts new file mode 100644 index 000000000..b02209139 --- /dev/null +++ b/app/features/scanner/core/detectors/objective/strip-weapons.ts @@ -0,0 +1,178 @@ +/** + * StripWeapons: per-slot weapon-icon evidence off the same icon strip the + * PlayerStatus read classifies, emitted by the ObjectiveDetector on a + * sampled cadence (identities are fixed for a match, so every read would be + * waste). The results scoreboard re-sorts each team per game while the + * strip keeps the lobby seating (attested in the sendou-triton VoD: strip + * [Planetz, .52, Neo Splash, Snipewriter] vs scoreboard rows + * [.52, Neo Splash, Snipewriter, Planetz]), so status samples cannot be + * paired with scoreboard rows by position alone — the match builder + * aggregates these candidate lists across the match and solves the + * slot→row assignment against the scoreboard's weapons + * (slot-row-assignment.ts). + * + * A slot's icon is the weapon render over a team-ink squid plate; the + * plate (and scene bleeding through it — the plates are translucent) is + * what drowns template matching, so saturated pixels near the plate's + * modal hue are flattened to the template background before the NCC + * ranking. One read's top-1 is only right about half the time on attested + * footage — the value is in the aggregate, so the event carries a ranked + * candidate list per slot. Splatted slots grey the render out and are + * skipped rather than guessed. + */ +import type { MainWeaponId } from "~/modules/in-game-lists/types"; +import { getCV, type Mat } from "../../cv"; +import { copyRoi } from "../../image"; +import { hueDistance, hueOf } from "../../ink-color"; +import { matchWeapon, type WeaponTemplate } from "../scoreboard/weapons"; +import type { DetectedEvent } from "../types"; +import type { PlayerStatusData, PlayerStatusLayout } from "./player-status"; +import { + STATUS_SLOT_CENTERS_CAST, + STATUS_SLOT_CENTERS_CAST_MIRROR, + STATUS_SLOT_CENTERS_POV, + STRIP_WEAPON_BOX, + STRIP_WEAPON_INK_THRESHOLD, + STRIP_WEAPON_KNOCKOUT_MIN_SPREAD, + STRIP_WEAPON_KNOCKOUT_MIN_VALUE, + STRIP_WEAPON_MAX_PLATE_HUE_DIST, + STRIP_WEAPON_TEMPLATE_BACKGROUND, + STRIP_WEAPON_TOP_K, +} from "./rois"; + +export const STRIP_WEAPONS_EVENT_TYPE = "StripWeapons"; + +export interface StripWeaponCandidate { + weaponId: MainWeaponId; + score: number; +} + +export interface StripWeaponsData { + /** match timer at the read, pairing it with the Objective/PlayerStatus events */ + time: number | null; + /** the icon-strip geometry the paired PlayerStatus read picked */ + layout: PlayerStatusLayout; + /** + * ranked weapon candidates per slot, [left team, right team], slots + * left-to-right as drawn; null = slot skipped (splatted icons grey the + * weapon render out) + */ + slots: [(StripWeaponCandidate[] | null)[], (StripWeaponCandidate[] | null)[]]; +} + +/** + * Match every alive slot's icon against the strip weapon templates. + * `status` is the PlayerStatus read off the same frame — its layout picks + * the slot centers and its dead flags pick which slots are worth reading. + */ +export function parseStripWeapons( + frame: Mat, + t: number, + status: PlayerStatusData, + templates: WeaponTemplate[], +): DetectedEvent { + const centers = slotCenters(status.layout); + const scores: number[] = []; + const slots = centers.map((sideCenters, side) => + sideCenters.map((cx, slot): StripWeaponCandidate[] | null => { + if (status.dead[side as 0 | 1][slot]) return null; + const candidates = matchSlot(frame, cx, templates); + if (candidates.length > 0) scores.push(candidates[0]!.score); + return candidates; + }), + ) as StripWeaponsData["slots"]; + + return { + type: STRIP_WEAPONS_EVENT_TYPE, + t, + // raw NCC peaks on attested footage sit ~0.4-0.6 even for correct + // reads; the aggregate assignment carries the reliability, so the + // event's own confidence only reflects that something matched at all + confidence: scores.length > 0 ? Math.max(...scores) : 0, + data: { + time: status.time, + layout: status.layout, + slots, + }, + }; +} + +function slotCenters( + layout: PlayerStatusLayout, +): readonly [readonly number[], readonly number[]] { + return layout === "pov" + ? STATUS_SLOT_CENTERS_POV + : layout === "cast" + ? STATUS_SLOT_CENTERS_CAST + : STATUS_SLOT_CENTERS_CAST_MIRROR; +} + +function matchSlot( + frame: Mat, + cx: number, + templates: WeaponTemplate[], +): StripWeaponCandidate[] { + const cv = getCV(); + const crop = copyRoi(frame, { + x: cx + STRIP_WEAPON_BOX.dx, + y: STRIP_WEAPON_BOX.y, + w: STRIP_WEAPON_BOX.w, + h: STRIP_WEAPON_BOX.h, + }); + const search = new cv.Mat(); + cv.cvtColor(crop, search, cv.COLOR_RGBA2RGB); + crop.delete(); + knockoutPlate(search); + const match = matchWeapon(search, templates, { + inkThreshold: STRIP_WEAPON_INK_THRESHOLD, + topN: STRIP_WEAPON_TOP_K, + }); + search.delete(); + return match.top.map((candidate) => ({ + weaponId: Number(candidate.id) as MainWeaponId, + score: candidate.score, + })); +} + +/** + * Flatten the squid plate out of the search region: the modal hue of the + * region's saturated pixels is the plate's team ink, and every pixel near + * that hue is replaced with the flat template background, leaving the + * weapon render (grey/white bodies and off-hue accents) to carry the NCC. + */ +function knockoutPlate(search: Mat): void { + const { data } = search; + const n = search.rows * search.cols; + const bins = new Array(36).fill(0); + for (let i = 0; i < n; i++) { + const r = data[i * 3]!; + const g = data[i * 3 + 1]!; + const b = data[i * 3 + 2]!; + const value = Math.max(r, g, b); + const spread = value - Math.min(r, g, b); + if ( + spread >= STRIP_WEAPON_KNOCKOUT_MIN_SPREAD + 15 && + value >= STRIP_WEAPON_KNOCKOUT_MIN_VALUE + 15 + ) { + bins[Math.floor(hueOf({ r, g, b }) / 10)]!++; + } + } + const plateHue = bins.indexOf(Math.max(...bins)) * 10 + 5; + for (let i = 0; i < n; i++) { + const r = data[i * 3]!; + const g = data[i * 3 + 1]!; + const b = data[i * 3 + 2]!; + const value = Math.max(r, g, b); + const spread = value - Math.min(r, g, b); + if ( + spread >= STRIP_WEAPON_KNOCKOUT_MIN_SPREAD && + value >= STRIP_WEAPON_KNOCKOUT_MIN_VALUE && + hueDistance(hueOf({ r, g, b }), plateHue) <= + STRIP_WEAPON_MAX_PLATE_HUE_DIST + ) { + data[i * 3] = STRIP_WEAPON_TEMPLATE_BACKGROUND; + data[i * 3 + 1] = STRIP_WEAPON_TEMPLATE_BACKGROUND; + data[i * 3 + 2] = STRIP_WEAPON_TEMPLATE_BACKGROUND; + } + } +} diff --git a/app/features/scanner/core/detectors/scoreboard/index.ts b/app/features/scanner/core/detectors/scoreboard/index.ts index a7720def2..0e9ae892c 100644 --- a/app/features/scanner/core/detectors/scoreboard/index.ts +++ b/app/features/scanner/core/detectors/scoreboard/index.ts @@ -87,6 +87,12 @@ export interface ScoreboardRowDebug { export interface ScoreboardResources { weapons: WeaponTemplate[]; + /** + * Weapon renders prepared for the in-match icon strip (objective's + * StripWeapons evidence). Optional: without them the strip slot → + * scoreboard row assignment falls back to as-drawn order. + */ + stripWeapons?: WeaponTemplate[] | null; /** * Special-weapon silhouettes (assets/cv/specials). Optional: without * them, near-tied weapon icons (Splash- vs Sploosh-o-matic) stay decided diff --git a/app/features/scanner/core/detectors/scoreboard/weapons.ts b/app/features/scanner/core/detectors/scoreboard/weapons.ts index e4ee26767..59f8f1b4a 100644 --- a/app/features/scanner/core/detectors/scoreboard/weapons.ts +++ b/app/features/scanner/core/detectors/scoreboard/weapons.ts @@ -64,6 +64,7 @@ export interface WeaponTemplate { export interface WeaponMatch { id: string; score: number; + /** best candidates, most likely first (3 unless options.topN says more) */ top: { id: string; score: number }[]; /** true when a scoped/unscoped twin tie was resolved by the unscoped prior */ twinAmbiguous?: boolean; @@ -308,10 +309,11 @@ function coarseShortlist( export function matchWeapon( searchRgb: Mat, templates: WeaponTemplate[], - options: { inkThreshold?: number } = {}, + options: { inkThreshold?: number; topN?: number } = {}, ): WeaponMatch { const cv = getCV(); const inkThreshold = options.inkThreshold ?? INK_THRESHOLD; + const topN = options.topN ?? 3; // icon ink present in the search region (pixel access needs a copy) const cont = new cv.Mat(); @@ -348,7 +350,7 @@ export function matchWeapon( const ranked = [...best.entries()] .map(([id, score]) => ({ id, score })) .sort((a, b) => b.score - a.score); - const top = ranked.slice(0, 3); + const top = ranked.slice(0, topN); let first = top[0] ?? { id: "unknown", score: -1 }; let twinAmbiguous = false; const unscopedId = SCOPED_TWINS.get(first.id); @@ -360,7 +362,7 @@ export function matchWeapon( const i = top.findIndex((t) => t.id === twin.id); if (i >= 0) top.splice(i, 1); top.unshift(twin); - top.length = Math.min(top.length, 3); + top.length = Math.min(top.length, topN); } } return { id: first.id, score: first.score, top, twinAmbiguous }; diff --git a/app/features/scanner/core/match-builder.ts b/app/features/scanner/core/match-builder.ts index 6f317db05..d4af8e88e 100644 --- a/app/features/scanner/core/match-builder.ts +++ b/app/features/scanner/core/match-builder.ts @@ -30,6 +30,10 @@ import { type PlayerStatusData, type PlayerStatusFlags, } from "./detectors/objective/player-status"; +import { + STRIP_WEAPONS_EVENT_TYPE, + type StripWeaponsData, +} from "./detectors/objective/strip-weapons"; import { SCOREBOARD_EVENT_TYPES } from "./detectors/registry"; import type { ScoreboardData } from "./detectors/scoreboard/index"; import { @@ -52,6 +56,13 @@ import type { ScannerMatchPlayerStatusSample, ScannerMatchTeam, } from "./scanner-match"; +import { + applyPermutation, + IDENTITY_PERMUTATION, + nameSlotRowPermutation, + type SlotRowPermutation, + weaponSlotRowPermutation, +} from "./slot-row-assignment"; /** The lobby header value private battles (tournament games) carry. */ const TOURNAMENT_LOBBY = "PRIVATE"; @@ -146,6 +157,7 @@ export function buildScannerMatches( let orphanDeaths: E[] = []; let orphanObjectives: E[] = []; let orphanPlayerStatuses: E[] = []; + let orphanStripWeapons: E[] = []; const finalize = (): void => { if (!open) return; if (open.scoreboard || open.minimaps.length > 0) { @@ -164,6 +176,7 @@ export function buildScannerMatches( orphanDeaths = []; orphanObjectives = []; orphanPlayerStatuses = []; + orphanStripWeapons = []; } else if (SCOREBOARD_EVENT_TYPES.includes(event.type)) { if (!open) { open = startMatch(); @@ -176,6 +189,9 @@ export function buildScannerMatches( open.playerStatuses = orphanPlayerStatuses.filter( (status) => event.t - status.t <= FALLBACK_WINDOW_SECONDS, ); + open.stripWeapons = orphanStripWeapons.filter( + (read) => event.t - read.t <= FALLBACK_WINDOW_SECONDS, + ); } open.scoreboard = event; vote(open.stageVotes, (event.data as ScoreboardData).stage); @@ -183,6 +199,7 @@ export function buildScannerMatches( orphanDeaths = []; orphanObjectives = []; orphanPlayerStatuses = []; + orphanStripWeapons = []; } else if (event.type === MINIMAP_EVENT_TYPE) { const stage = (event.data as MinimapData).stage; if (open) { @@ -210,6 +227,8 @@ export function buildScannerMatches( (open?.objectives ?? orphanObjectives).push(event); } else if (event.type === PLAYER_STATUS_EVENT_TYPE) { (open?.playerStatuses ?? orphanPlayerStatuses).push(event); + } else if (event.type === STRIP_WEAPONS_EVENT_TYPE) { + (open?.stripWeapons ?? orphanStripWeapons).push(event); } } finalize(); @@ -266,7 +285,8 @@ export function invalidObjectiveEvents( b.sources.filter( (event) => event.type === OBJECTIVE_EVENT_TYPE || - event.type === PLAYER_STATUS_EVENT_TYPE, + event.type === PLAYER_STATUS_EVENT_TYPE || + event.type === STRIP_WEAPONS_EVENT_TYPE, ), ); } @@ -338,6 +358,8 @@ interface OpenMatch { objectives: E[]; /** icon-strip reads; become the match's `playerStatus` samples */ playerStatuses: E[]; + /** sampled per-slot weapon evidence for the slot→row assignment */ + stripWeapons: E[]; scoreboard: E | null; /** * per-stage read counts (a MapStart's stage seeds it); the plurality @@ -356,6 +378,7 @@ function startMatch(): OpenMatch { deaths: [], objectives: [], playerStatuses: [], + stripWeapons: [], scoreboard: null, stageVotes: new Map(), lastMinimapT: null, @@ -406,6 +429,7 @@ function toBuiltMatch( ...open.deaths, ...open.objectives, ...open.playerStatuses, + ...open.stripWeapons, ...(open.scoreboard ? [open.scoreboard] : []), ].sort((a, b) => a.t - b.t); @@ -428,6 +452,10 @@ function toBuiltMatch( t: event.t, data: event.data as PlayerStatusData, })); + const stripWeapons = open.stripWeapons.map((event) => ({ + t: event.t, + data: event.data as StripWeaponsData, + })); const minimapReads = open.minimaps.map((event) => ({ t: event.t, data: event.data as MinimapData, @@ -443,6 +471,7 @@ function toBuiltMatch( const progress = buildProgress( counterModeValid ? objectives : [], counterModeValid ? playerStatuses : [], + counterModeValid ? stripWeapons : [], minimapReads, board, minimapTeamColors(minimaps), @@ -515,14 +544,25 @@ function floorOrNull(t: number | undefined): number | null { * Their sides are own/alpha-vs-enemy/bravo — camera-stable, unlike the * plates — so they skip the per-read cluster orientation and map to * `teams` through the same minimap ink anchor the whole match uses - * (identity on a minimap-grouped match by construction). Whether the - * minimap's card order matches the icon strip's slot order within a side - * is unattested so far; both follow the order `teams` players are seated - * in for their respective sources. + * (identity on a minimap-grouped match by construction). + * + * Within a side, the strip's slot order is the lobby seating while a + * results scoreboard re-sorts its rows per game (attested in the + * sendou-triton VoD) — so on a scoreboard-closed match each side's slots + * are reordered into row order via the slot→row assignment + * (slot-row-assignment.ts): weapon votes from the sampled StripWeapons + * evidence plus the minimap's card columns, which mirror the strip's + * seating (attested for the enemy column; the spectator screen's own + * column is assumed symmetric). The POV overlay's teammate diamond + * follows neither order, so diamond-sourced flags map by card name + * instead, and keep their as-drawn order when too few names resolve. A + * minimap-grouped match's `teams` come from the cards themselves, so its + * samples stay as drawn by construction. */ function buildProgress( objectives: readonly { t: number; data: ObjectiveData }[], playerStatuses: readonly { t: number; data: PlayerStatusData }[], + stripWeapons: readonly { t: number; data: StripWeaponsData }[], minimapReads: readonly { t: number; data: MinimapData }[], board: ScoreboardData | undefined, minimapColors: [InkRgb | null, InkRgb | null] | null, @@ -555,6 +595,18 @@ function buildProgress( : minimapAnchorSwap(clusterHues, minimapColors); const minimapSwapped = swap !== minimapAnchorSwap(clusterHues, minimapColors); + const perms = board + ? slotRowPermutations( + board, + stripWeapons, + minimapReads, + live, + swapFlags, + swap, + minimapSwapped, + ) + : null; + const objective = oriented.length === 0 ? null @@ -583,11 +635,20 @@ function buildProgress( ? minimapSwapped : nearestSwapFlag(live, swapFlags, read.t) !== swap; const [a, b] = swapped ? ([1, 0] as const) : ([0, 1] as const); + const arrange = ( + flags: readonly [PlayerStatusFlags, PlayerStatusFlags], + ): [PlayerStatusFlags, PlayerStatusFlags] => + [a, b].map((source, side) => + applyPermutation( + flags[source]!, + readPermutation(perms, read, source, side as 0 | 1), + ), + ) as [PlayerStatusFlags, PlayerStatusFlags]; return { t: Math.max(0, Math.floor(read.t)), time: read.data.time, - special: [read.data.special[a], read.data.special[b]], - dead: [read.data.dead[a], read.data.dead[b]], + special: arrange(read.data.special), + dead: arrange(read.data.dead), }; }), ), @@ -597,6 +658,136 @@ function buildProgress( return { objective, playerStatus }; } +/** The slot→row permutations of a scoreboard-closed match, per source. */ +interface SlotRowPerms { + /** per teams side, for strip-seated slots (the strip and card columns) */ + strip: [SlotRowPermutation, SlotRowPermutation]; + /** for the POV diamond's teammate flags; null = keep as drawn */ + diamond: SlotRowPermutation | null; +} + +/** + * How much one minimap card's parsed weapon counts next to the strip + * evidence's raw NCC scores (~0.3-0.6 per candidate per read): the card + * parser is gated on a clean read, so one card outweighs a single strip + * sample without being able to drown a match's worth of them. + */ +const MINIMAP_CARD_VOTE = 1; + +/** + * Accumulate the match's weapon votes (sampled strip evidence oriented + * read-by-read like the status samples; minimap cards through the match's + * minimap anchor) and solve each side's slot→row assignment against the + * scoreboard's weapons, plus the diamond's name-based assignment for POV + * teammate cards. + */ +function slotRowPermutations( + board: ScoreboardData, + stripWeapons: readonly { t: number; data: StripWeaponsData }[], + minimapReads: readonly { t: number; data: MinimapData }[], + live: readonly { t: number; data: ObjectiveData }[], + swapFlags: readonly boolean[], + swap: boolean, + minimapSwapped: boolean, +): SlotRowPerms { + const votes: Map[][] = [0, 1].map(() => + [0, 1, 2, 3].map(() => new Map()), + ); + const addVote = ( + side: 0 | 1, + slot: number, + weaponId: MainWeaponId, + score: number, + ): void => { + const slotVotes = votes[side]![slot]!; + slotVotes.set(weaponId, (slotVotes.get(weaponId) ?? 0) + score); + }; + + for (const read of stripWeapons) { + const swapped = nearestSwapFlag(live, swapFlags, read.t) !== swap; + for (const side of [0, 1] as const) { + const source = swapped ? ((1 - side) as 0 | 1) : side; + for (const [slot, candidates] of read.data.slots[source].entries()) { + for (const candidate of candidates ?? []) { + addVote(side, slot, candidate.weaponId, candidate.score); + } + } + } + } + + // enemy cards mirror the strip seating (attested); the spectator + // screen's own column is assumed symmetric. The POV diamond is not + // strip-seated and votes for nothing. + const enemySide = minimapSwapped ? 0 : 1; + for (const read of minimapReads) { + for (const [slot, enemy] of read.data.enemies.entries()) { + if (enemy.weaponId !== null) { + addVote(enemySide, slot, enemy.weaponId, MINIMAP_CARD_VOTE); + } + } + if (!read.data.spectator) continue; + for (const [slot, mate] of read.data.teammates.entries()) { + if (mate.weaponId !== null) { + addVote( + (1 - enemySide) as 0 | 1, + slot, + mate.weaponId, + MINIMAP_CARD_VOTE, + ); + } + } + } + + const rowWeapons = (side: 0 | 1) => + board.players + .slice(side * PLAYERS_PER_TEAM, (side + 1) * PLAYERS_PER_TEAM) + .map((player) => player.weaponId); + const strip = [0, 1].map((side) => + weaponSlotRowPermutation(votes[side]!, rowWeapons(side as 0 | 1)), + ) as [SlotRowPermutation, SlotRowPermutation]; + + const friendlySide = minimapSwapped ? 1 : 0; + const cardNames: (string | null)[] = [null, null, null, null]; + for (const read of minimapReads) { + if (read.data.spectator) continue; + for (const [slot, mate] of read.data.teammates.entries()) { + cardNames[slot] ??= mate.name?.trim() || null; + } + } + const diamond = cardNames.some((name) => name !== null) + ? nameSlotRowPermutation( + cardNames, + board.players + .slice( + friendlySide * PLAYERS_PER_TEAM, + (friendlySide + 1) * PLAYERS_PER_TEAM, + ) + .map((player) => player.name.trim() || null), + ) + : null; + + return { strip, diamond }; +} + +/** + * Which permutation a status read's `sourceSide` flags go through on their + * way to teams side `side`: strip-seated sources (the strip itself, card + * columns) take the weapon-vote assignment, the POV diamond its name + * assignment; a minimap-grouped match (no perms) keeps everything as drawn. + */ +function readPermutation( + perms: SlotRowPerms | null, + read: StatusRead, + sourceSide: 0 | 1, + side: 0 | 1, +): SlotRowPermutation { + if (!perms) return IDENTITY_PERMUTATION; + if (read.fromMinimap && sourceSide === 0 && !read.spectator) { + return perms.diamond ?? IDENTITY_PERMUTATION; + } + return perms.strip[side]; +} + /** * Debounce per-slot dead flags across a match's samples: an interior run * whose flanking opposite-state reads sit closer together than the state @@ -702,6 +893,12 @@ interface StatusRead { t: number; /** minimap sides are own/enemy — camera-stable, unlike the HUD plates */ fromMinimap: boolean; + /** + * minimap reads only: the 8-card spectator screen, whose own-side + * column is card-seated like the enemy one — the POV overlay's + * teammate diamond is not (see readPermutation) + */ + spectator?: boolean; data: { time: number | null; special: [PlayerStatusFlags, PlayerStatusFlags]; @@ -726,6 +923,7 @@ function minimapStatusReads( { t: read.t, fromMinimap: true, + spectator: read.data.spectator, data: { time: null, special: [ diff --git a/app/features/scanner/core/resources.ts b/app/features/scanner/core/resources.ts index b94dfe8bc..3e7a9b752 100644 --- a/app/features/scanner/core/resources.ts +++ b/app/features/scanner/core/resources.ts @@ -27,6 +27,11 @@ import { SUB_TILE_TEMPLATE_SIZES, } from "./detectors/minimap/rois"; import type { PlannerStage } from "./detectors/minimap/stage"; +import { + STRIP_WEAPON_INK_THRESHOLD, + STRIP_WEAPON_TEMPLATE_BACKGROUND, + STRIP_WEAPON_TEMPLATE_SIZES, +} from "./detectors/objective/rois"; import type { ScoreboardResources } from "./detectors/scoreboard/index"; import { prepareSpecialTemplates } from "./detectors/scoreboard/specials"; import { prepareWeaponTemplates } from "./detectors/scoreboard/weapons"; @@ -132,6 +137,13 @@ export async function assembleScoreboardResources( cropToArt: true, }), ); + const stripWeapons = lazy(() => + prepareWeaponTemplates(weaponIcons, STRIP_WEAPON_TEMPLATE_SIZES, { + background: STRIP_WEAPON_TEMPLATE_BACKGROUND, + inkThreshold: STRIP_WEAPON_INK_THRESHOLD, + cropToArt: true, + }), + ); const specials = lazy(() => prepareSpecialTemplates(specialIcons)); const minimapSubWeapons = lazy(() => prepareSpecialTemplates(subIcons, SUB_TILE_TEMPLATE_SIZES), @@ -155,6 +167,9 @@ export async function assembleScoreboardResources( get minimapLightWeapons() { return minimapLightWeapons(); }, + get stripWeapons() { + return stripWeapons(); + }, get specials() { return specials(); }, diff --git a/app/features/scanner/core/slot-row-assignment.ts b/app/features/scanner/core/slot-row-assignment.ts new file mode 100644 index 000000000..a80a1211f --- /dev/null +++ b/app/features/scanner/core/slot-row-assignment.ts @@ -0,0 +1,130 @@ +/** + * Strip-slot → scoreboard-row assignment. The in-match icon strip (and the + * minimap's card columns, which mirror it) keeps the lobby seating for the + * whole set, while the results scoreboard re-sorts each team per game — so + * per-slot status series can only be paired with scoreboard rows through + * identity evidence. Two kinds are combined: + * + * - weapon votes: per-slot candidate scores accumulated across a match's + * sampled StripWeapons reads (strip-weapons.ts) plus the minimap cards' + * parsed weapons. The best-scoring of the 24 possible slot→row + * assignments against the scoreboard's four weapons wins — the global + * constraint corrects slots whose own evidence is wrong or missing + * (attested: a slot with zero readable votes still lands right by + * elimination). + * - card names (the POV minimap's teammate diamond): matched directly + * against scoreboard row names. + * + * Ties resolve toward the fewest moved slots, so two rows sharing a weapon + * keep their as-drawn relative order, and thin evidence degrades to the + * as-drawn arrangement rather than a coin flip. + */ +import type { MainWeaponId } from "~/modules/in-game-lists/types"; + +/** A slot→row permutation: `perm[slot]` is the scoreboard row the slot feeds. */ +export type SlotRowPermutation = readonly [number, number, number, number]; + +export const IDENTITY_PERMUTATION: SlotRowPermutation = [0, 1, 2, 3]; + +/** + * Total accumulated vote score the winning assignment needs before it may + * reorder anything, and the lead it needs over the best differing + * assignment. Calibrated on the sendou-triton VoD, where correct + * assignments scored 10-33 with margins 2.2-5.3 over ~20 sampled reads; + * junk evidence (a strip geometry mispick, non-Splatoon lookalikes) + * spreads flat and fails the margin. + */ +const MIN_ASSIGNMENT_SCORE = 1.5; +const MIN_ASSIGNMENT_MARGIN = 0.75; + +/** All 24 permutations, fewest-moved-slots first (ties resolve to earlier). */ +const PERMUTATIONS: SlotRowPermutation[] = (() => { + const all: SlotRowPermutation[] = []; + for (const a of [0, 1, 2, 3]) + for (const b of [0, 1, 2, 3]) + for (const c of [0, 1, 2, 3]) + for (const d of [0, 1, 2, 3]) { + if (new Set([a, b, c, d]).size === 4) all.push([a, b, c, d]); + } + const displaced = (perm: SlotRowPermutation) => + perm.filter((row, slot) => row !== slot).length; + return all.sort((x, y) => displaced(x) - displaced(y)); +})(); + +/** + * The slot→row assignment best supported by one side's accumulated weapon + * votes, against that side's scoreboard row weapons. Falls back to the + * as-drawn order when the evidence is too thin or too close to call (see + * MIN_ASSIGNMENT_SCORE/MARGIN). + */ +export function weaponSlotRowPermutation( + votes: readonly ReadonlyMap[], + rowWeapons: readonly (MainWeaponId | null)[], +): SlotRowPermutation { + const scored = PERMUTATIONS.map((perm) => ({ + perm, + score: perm.reduce((sum, row, slot) => { + const weapon = rowWeapons[row]; + return sum + (weapon === null ? 0 : (votes[slot]?.get(weapon!) ?? 0)); + }, 0), + })); + let best = scored[0]!; + for (const candidate of scored) { + if (candidate.score > best.score) best = candidate; + } + if (best.score < MIN_ASSIGNMENT_SCORE) return IDENTITY_PERMUTATION; + const runnerUp = Math.max( + ...scored + .filter((candidate) => candidate.score < best.score) + .map((candidate) => candidate.score), + 0, + ); + if (best.score - runnerUp < MIN_ASSIGNMENT_MARGIN) { + return IDENTITY_PERMUTATION; + } + return best.perm; +} + +/** + * A card→row assignment from card names (the POV minimap's teammate + * diamond, whose order matches neither the strip nor the scoreboard): + * unique case-insensitive name matches place their cards, the leftovers + * keep their relative as-drawn order. Null — keep the as-drawn order — + * when fewer than two cards resolve, since a single hit cannot attest the + * arrangement is worth disturbing. + */ +export function nameSlotRowPermutation( + cardNames: readonly (string | null)[], + rowNames: readonly (string | null)[], +): SlotRowPermutation | null { + const normalized = (name: string | null) => + name?.trim().toLowerCase() || null; + const rows = rowNames.map(normalized); + const assignment: (number | null)[] = [null, null, null, null]; + const takenRows = new Set(); + let resolved = 0; + for (const [slot, cardName] of cardNames.map(normalized).entries()) { + if (cardName === null) continue; + const matches = rows.flatMap((row, i) => (row === cardName ? [i] : [])); + if (matches.length !== 1 || takenRows.has(matches[0]!)) continue; + assignment[slot] = matches[0]!; + takenRows.add(matches[0]!); + resolved++; + } + if (resolved < 2) return null; + const freeRows = [0, 1, 2, 3].filter((row) => !takenRows.has(row)); + for (const [slot, row] of assignment.entries()) { + if (row === null) assignment[slot] = freeRows.shift()!; + } + return assignment as unknown as SlotRowPermutation; +} + +/** `flags` rearranged so slot `i`'s value lands at `perm[i]`. */ +export function applyPermutation( + flags: readonly T[], + perm: SlotRowPermutation, +): T[] { + const out = [...flags] as T[]; + for (const [slot, row] of perm.entries()) out[row] = flags[slot]!; + return out; +} diff --git a/app/features/scanner/core/timeline/index.ts b/app/features/scanner/core/timeline/index.ts index bad031ae2..3de112f71 100644 --- a/app/features/scanner/core/timeline/index.ts +++ b/app/features/scanner/core/timeline/index.ts @@ -16,6 +16,7 @@ import { PLAYER_STATUS_EVENT_TYPE, samePlayerStatusData, } from "../detectors/objective/player-status"; +import { STRIP_WEAPONS_EVENT_TYPE } from "../detectors/objective/strip-weapons"; import { SCOREBOARD_EVENT_TYPE } from "../detectors/scoreboard/index"; import { SCOREBOARD_BATTLE_LOG_EVENT_TYPE } from "../detectors/scoreboard-battle-log/index"; import { SCOREBOARD_BATTLE_LOG_REPLAY_EVENT_TYPE } from "../detectors/scoreboard-battle-log-replay/index"; @@ -39,6 +40,12 @@ export interface TimelineOptions { sameEventDataByType: Record boolean>; /** events below this confidence are dropped */ minConfidence: number; + /** + * per-type confidence floor overrides: evidence-carrying events whose + * scores live on a different scale than parse confidences (raw NCC + * peaks) opt out of the shared floor + */ + minConfidenceByType: Record; } const DEFAULT_TIMELINE_OPTIONS: TimelineOptions = { @@ -54,11 +61,14 @@ const DEFAULT_TIMELINE_OPTIONS: TimelineOptions = { // stretches into one event per state. Player statuses can revisit an // exact prior state no sooner than a respawn takes (~9s), so their // window must stay under that + // strip weapon evidence is sampled every ~5s and consecutive samples are + // distinct evidence — only same-frame re-reads should collapse mergeWindowByType: { Death: 8, [MINIMAP_EVENT_TYPE]: 5, [OBJECTIVE_EVENT_TYPE]: 10, [PLAYER_STATUS_EVENT_TYPE]: 5, + [STRIP_WEAPONS_EVENT_TYPE]: 2, }, sameEventDataByType: { [SCOREBOARD_EVENT_TYPE]: sameScoreboardMatch, @@ -69,6 +79,9 @@ const DEFAULT_TIMELINE_OPTIONS: TimelineOptions = { [PLAYER_STATUS_EVENT_TYPE]: samePlayerStatusData, }, minConfidence: 0.6, + minConfidenceByType: { + [STRIP_WEAPONS_EVENT_TYPE]: 0, + }, }; export type TimelineAction = @@ -90,7 +103,10 @@ export class TimelineBuilder { } push(event: DetectedEvent): TimelineAction { - if (event.confidence < this.#options.minConfidence) { + const minConfidence = + this.#options.minConfidenceByType[event.type] ?? + this.#options.minConfidence; + if (event.confidence < minConfidence) { return { action: "dropped", reason: "low-confidence" }; } const window = diff --git a/app/features/scanner/node/fixtures.ts b/app/features/scanner/node/fixtures.ts index 12725eab1..9c4ad8996 100644 --- a/app/features/scanner/node/fixtures.ts +++ b/app/features/scanner/node/fixtures.ts @@ -72,6 +72,7 @@ interface ExpectedScoreboard { | "Minimap" | "Objective" | "PlayerStatus" + | "StripWeapons" | "none"; data?: { lobby?: ScannerLobby; @@ -112,8 +113,15 @@ interface ExpectedScoreboard { special?: [boolean[], boolean[]]; /** PlayerStatus only: splatted per slot, [left team, right team] */ dead?: [boolean[], boolean[]]; - /** PlayerStatus only: which icon-strip geometry the frame shows */ + /** PlayerStatus + StripWeapons: which icon-strip geometry the frame shows */ layout?: "pov" | "cast" | "cast-mirror"; + /** + * StripWeapons only: the true weapon per slot, [left team, right + * team], null = slot skipped (splatted icon). weaponLabels is + * informational for the human corrector. + */ + weapons?: [(MainWeaponId | null)[], (MainWeaponId | null)[]]; + weaponLabels?: [(string | null)[], (string | null)[]]; /** Minimap only: casted 8-player spectator map screen (not parsed yet) */ spectator?: boolean; /** Minimap only: own-team callout cards in slot order */ diff --git a/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m33/expected.json b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m33/expected.json new file mode 100644 index 000000000..36ad33739 --- /dev/null +++ b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m33/expected.json @@ -0,0 +1,14 @@ +{ + "event": "StripWeapons", + "data": { + "layout": "cast", + "weapons": [ + [2070, 10, 211, 1010], + [1042, 50, 21, 2070] + ], + "weaponLabels": [ + ["Snipewriter 5H", "Splattershot Jr.", "Custom Blaster", "Splat Roller"], + ["Planetz Big Swig Roller", ".52 Gal", "Neo Splash-o-matic", "Snipewriter 5H"] + ] + } +} diff --git a/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m33/frame.png b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m33/frame.png new file mode 100644 index 000000000..086d6431c Binary files /dev/null and b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m33/frame.png differ diff --git a/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m59/expected.json b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m59/expected.json new file mode 100644 index 000000000..288ce24e0 --- /dev/null +++ b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m59/expected.json @@ -0,0 +1,18 @@ +{ + "event": "StripWeapons", + "data": { + "layout": "cast", + "weapons": [ + [2070, 10, 211, 1010], + [1042, 50, 21, 2070] + ], + "weaponLabels": [ + ["Snipewriter 5H", "Splattershot Jr.", "Custom Blaster", "Splat Roller"], + ["Planetz Big Swig Roller", ".52 Gal", "Neo Splash-o-matic", "Snipewriter 5H"] + ] + }, + "options": { + "skipFields": ["layout"], + "notes": "badge-less broadcast; in isolation the decisiveness score picks pov on this frame (sticky layout corrects it in a real scan)" + } +} diff --git a/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m59/frame.png b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m59/frame.png new file mode 100644 index 000000000..0a5de1d2e Binary files /dev/null and b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-allalive-4m59/frame.png differ diff --git a/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-twodead-4m17/expected.json b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-twodead-4m17/expected.json new file mode 100644 index 000000000..74301477d --- /dev/null +++ b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-twodead-4m17/expected.json @@ -0,0 +1,14 @@ +{ + "event": "StripWeapons", + "data": { + "layout": "cast", + "weapons": [ + [2070, 10, null, 1010], + [1042, 50, 21, null] + ], + "weaponLabels": [ + ["Snipewriter 5H", "Splattershot Jr.", "splatted (Custom Blaster)", "Splat Roller"], + ["Planetz Big Swig Roller", ".52 Gal", "Neo Splash-o-matic", "splatted (Snipewriter 5H)"] + ] + } +} diff --git a/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-twodead-4m17/frame.png b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-twodead-4m17/frame.png new file mode 100644 index 000000000..71bda5bbf Binary files /dev/null and b/app/features/scanner/tests/fixtures/strip-weapons/triton-mahi-twodead-4m17/frame.png differ diff --git a/app/features/scanner/tests/match-builder.test.ts b/app/features/scanner/tests/match-builder.test.ts index 256c7b6bb..616a0b316 100644 --- a/app/features/scanner/tests/match-builder.test.ts +++ b/app/features/scanner/tests/match-builder.test.ts @@ -14,6 +14,7 @@ import type { import { SPECTATOR_SLOTS } from "../core/detectors/minimap/rois"; import type { ObjectiveData } from "../core/detectors/objective/index"; import type { PlayerStatusData } from "../core/detectors/objective/player-status"; +import type { StripWeaponsData } from "../core/detectors/objective/strip-weapons"; import type { ScoreboardData } from "../core/detectors/scoreboard/index"; import type { ScoreboardBattleLogData } from "../core/detectors/scoreboard-battle-log/index"; import type { ScoreboardBattleLogReplayData } from "../core/detectors/scoreboard-battle-log-replay/index"; @@ -858,6 +859,23 @@ function playerStatus( return { type: "PlayerStatus", t, confidence: 0.9, data }; } +function stripWeaponsEvent( + t: number, + slots: [(MainWeaponId | null)[], (MainWeaponId | null)[]], + { score = 0.6, time = (300 - Math.round(t)) as number | null } = {}, +): DetectedEvent { + const data: StripWeaponsData = { + time, + layout: "cast", + slots: slots.map((side) => + side.map((weaponId) => + weaponId === null ? null : [{ weaponId, score }], + ), + ) as StripWeaponsData["slots"], + }; + return { type: "StripWeapons", t, confidence: score, data }; +} + test("player-status reads become teams-order samples on the match", () => { const special = [ [true, false, false, false], @@ -1158,3 +1176,101 @@ test("a losing-side pov swaps minimap-sourced samples into teams order", () => { [true, false, false, false], ]); }); + +// ---- strip-slot → scoreboard-row assignment ---- + +test("strip weapon evidence reorders status slots into scoreboard rows", () => { + // strip seating [2010, 40, 3030, 1001] vs scoreboard rows ALPHA + // [40, 1001, 2010, 3030]: slot0 belongs to row2 + const built = buildScannerMatches([ + mapStart(0), + playerStatus(120, { + dead: [ + [true, false, false, false], + [false, false, false, false], + ], + }), + stripWeaponsEvent(121, [ + [2010, 40, 3030, 1001], + [null, null, null, null], + ]), + scoreboard(300), + ]); + const sample = built[0]!.match.playerStatus!.samples[0]!; + assert.deepEqual(sample.dead, [ + [false, false, true, false], + [false, false, false, false], + ]); +}); + +test("weapon evidence below the assignment floor keeps the as-drawn order", () => { + const dead = [ + [true, false, false, false], + [false, false, false, false], + ] as PlayerStatusData["dead"]; + const built = buildScannerMatches([ + mapStart(0), + playerStatus(120, { dead }), + stripWeaponsEvent( + 121, + [ + [2010, null, null, null], + [null, null, null, null], + ], + { + score: 0.5, + }, + ), + scoreboard(300), + ]); + assert.deepEqual(built[0]!.match.playerStatus!.samples[0]!.dead, dead); +}); + +test("minimap enemy-card weapons vote the strip assignment too", () => { + // enemy cards in strip seating [4010, 50, 8000, 210] vs rows BRAVO + // [50, 210, 4010, 8000]: the strip-sourced side1 slot0 belongs to row2 + const seating: (MainWeaponId | null)[] = [4010, 50, 8000, 210]; + const built = buildScannerMatches([ + mapStart(0), + minimap(60, { bravo: seating }), + minimap(90, { bravo: seating }), + playerStatus(120, { + dead: [ + [false, false, false, false], + [true, false, false, false], + ], + }), + scoreboard(300), + ]); + const strip = built[0]!.match.playerStatus!.samples.at(-1)!; + assert.deepEqual(strip.dead, [ + [false, false, false, false], + [false, false, true, false], + ]); +}); + +test("pov diamond cards map to scoreboard rows by name", () => { + const cards = [ + { ...teammate(ALPHA[1]!, 0), name: "w2", dead: true }, + { ...teammate(ALPHA[0]!, 1), name: "w1" }, + { ...teammate(ALPHA[3]!, 2), name: "w4" }, + { ...teammate(ALPHA[2]!, 3), name: "w3" }, + ]; + const data: MinimapData = { + stage: 0 as StageId, + spectator: false, + teammates: cards, + enemies: BRAVO.map((id) => enemy(id)), + teamColors: [null, null], + }; + const built = buildScannerMatches([ + mapStart(0), + { type: "Minimap", t: 90, confidence: 0.8, data } as DetectedEvent, + scoreboard(300), + ]); + const sample = built[0]!.match.playerStatus!.samples[0]!; + assert.deepEqual(sample.dead, [ + [false, true, false, false], + [false, false, false, false], + ]); +}); diff --git a/app/features/scanner/tests/strip-weapons.test.ts b/app/features/scanner/tests/strip-weapons.test.ts new file mode 100644 index 000000000..f41db5957 --- /dev/null +++ b/app/features/scanner/tests/strip-weapons.test.ts @@ -0,0 +1,125 @@ +/** + * Golden-file tests for the StripWeapons evidence event over every fixture + * in strip-weapons/. Single reads are deliberately weak (the true weapon + * ranks top-1 only about half the time), so per-slot assertions stay + * structural — splatted slots skipped, alive slots ranked — and the + * accuracy assertion is the one production relies on: votes aggregated + * across the fixtures assign every slot to its scoreboard row. The + * fixtures are frames of one match of the sendou-triton VoD, whose results + * screen (and D-column ground truth) attests both sides' row orders. + */ + +import assert from "node:assert/strict"; +import type { MainWeaponId } from "~/modules/in-game-lists/types"; +import { loadOpenCV } from "../core/cv"; +import { createObjectiveDetector } from "../core/detectors/objective/index"; +import { + STRIP_WEAPONS_EVENT_TYPE, + type StripWeaponsData, +} from "../core/detectors/objective/strip-weapons"; +import type { DetectedEvent } from "../core/detectors/types"; +import { weaponSlotRowPermutation } from "../core/slot-row-assignment"; +import { + type Fixture, + isFieldSkipped, + loadFixtures, + runDetectorOnFixture, +} from "../node/fixtures"; +import { loadScoreboardResources } from "../node/resources"; +import test from "./node-test-compat"; + +await loadOpenCV(); +const resources = await loadScoreboardResources(); +const fixtures = loadFixtures("strip-weapons"); + +test("strip-weapons fixtures exist", () => { + assert.ok(fixtures.length > 0, "no fixtures found under strip-weapons/"); +}); + +const parsed = new Map>(); + +for (const fixture of fixtures) { + test(`strip-weapons/${fixture.name}`, async (t) => { + const { gate, events } = await runDetectorOnFixture( + createObjectiveDetector(resources), + fixture, + ); + assert.ok(gate.pass, `objective gate did not fire (${gate.score})`); + const event = events.find((e) => e.type === STRIP_WEAPONS_EVENT_TYPE) as + | DetectedEvent + | undefined; + assert.ok(event, "no StripWeapons event alongside the counter read"); + parsed.set(fixture.name, event); + const expected = fixture.expected.data ?? {}; + + await t.test( + "layout", + { skip: expected.layout === undefined || skip(fixture, "layout") }, + () => { + assert.equal(event.data.layout, expected.layout); + }, + ); + + for (const side of [0, 1] as const) { + for (const slot of [0, 1, 2, 3] as const) { + const truth = expected.weapons?.[side]?.[slot]; + await t.test( + `slot[${side}][${slot}]`, + { + skip: + truth === undefined || skip(fixture, `weapons.${side}.${slot}`), + }, + () => { + const candidates = event.data.slots[side][slot]; + if (truth === null) { + assert.equal(candidates, null, "splatted slot should be skipped"); + } else { + assert.ok(candidates, "alive slot should carry candidates"); + assert.ok(candidates.length > 0, "empty candidate list"); + } + }, + ); + } + } + }); +} + +// The assertion production leans on: aggregated across the match's sampled +// reads, each side's best-of-24 assignment against the results screen's +// row weapons places every slot. Row orders attested on the VoD's results +// screen: left/losing side rows [Snipewriter 5H, Custom Blaster, +// Splattershot Jr., Splat Roller] vs strip seating [Snipewriter, Jr, +// Custom Blaster, Roller]; right/winning side rows [.52 Gal, Neo +// Splash-o-matic, Snipewriter 5H, Planetz Big Swig Roller] vs seating +// [Planetz, .52, Neo Splash, Snipewriter]. +test("aggregated votes assign every slot to its scoreboard row", () => { + assert.ok(parsed.size >= 2, "needs at least two parsed fixtures"); + const votes = [0, 1].map(() => + [0, 1, 2, 3].map(() => new Map()), + ); + for (const event of parsed.values()) { + for (const side of [0, 1] as const) { + for (const [slot, candidates] of event.data.slots[side].entries()) { + for (const candidate of candidates ?? []) { + const slotVotes = votes[side]![slot]!; + slotVotes.set( + candidate.weaponId, + (slotVotes.get(candidate.weaponId) ?? 0) + candidate.score, + ); + } + } + } + } + assert.deepEqual( + weaponSlotRowPermutation(votes[0]!, [2070, 211, 10, 1010]), + [0, 2, 1, 3], + ); + assert.deepEqual( + weaponSlotRowPermutation(votes[1]!, [50, 21, 2070, 1042]), + [3, 0, 1, 2], + ); +}); + +function skip(fixture: Fixture, field: string): boolean | string { + return isFieldSkipped(fixture, field) ? "skipFields" : false; +}