Port the Buildfile and deploy scripts to co-located deploy modules

rules/ holds the shared sets (memoized) and the publish helpers that were
duplicated across the vm scripts; assets/smogon/ps .deploy.ts each declare
their rules and finish. The smogdex sheet is split into two chained rules,
retiring the undeclared spritesheet.png temp.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

assets.deploy.ts

@@ -0,0 +1,92 @@
+
+import {gen6Padded, itemPadded} from './rules/minisprites.ts';
+import {itemspritecopy, type Manifest, newspritecopy, spritecopy, writeManifest} from './rules/publish.ts';
+import {forEachRule, rule} from './tools/build/artifact.ts';
+import {spriteglob} from './tools/build/helpers.ts';
+import {defineDeploy} from './tools/deploy/api.ts';
+
+// Smogdex minisprites (webp)
+
+const minispriteInputs = spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false});
+
+const webpMinisprites = forEachRule(minispriteInputs, {
+    display: "webp minisprite %f",
+    cmds: ["cwebp -z 9 %f -o %o"],
+}, "%B.webp");
+
+// Smogdex spritesheet. The sheet tool bakes sprite ids parsed from the %f
+// filenames into the css, hence nameSensitive.
+
+const [sheetPng, sheetCss] = rule(minispriteInputs, {
+    display: "smogdex sheet",
+    nameSensitive: true,
+    deps: [
+        "data/species.json",
+        "data/items.json",
+        "data/lib/index.ts",
+        "lib/root/index.ts",
+        "tools/smogdexspritesheet/index.ts",
+    ],
+    cmds: ["node tools/smogdexspritesheet/index.ts --image %o1 --stylesheet %o2 -- %f"],
+}, ["spritesheet.png", "spritesheet.css"]);
+
+const [sheetWebp] = rule(sheetPng!, ["cwebp -z 9 %f -o %o"], ["spritesheet.webp"]);
+
+// Forumsprites sources. Rules must be declared at import time (the build
+// runs before finish), so these calls cannot live inside finish().
+const forumItems = itemPadded();
+const forumG6 = gen6Padded();
+
+export default defineDeploy({
+    finish(ctx) {
+        // The upload contract wants __key first in the tar, naming the asset
+        // set; the tar packer emits ops in queue order, so it has to be the
+        // first op.
+        ctx.write("__key", "sprites");
+
+        // Dex spritesheet assets: hash-stamped css + webp. The css suffix
+        // pointer rides in __meta/ for the dex to read.
+        {
+            const wh = ctx.hash(sheetWebp!);
+            ctx.copy(sheetWebp!, `spritesheet-${wh}.webp`);
+            const src = ctx.read(sheetCss!);
+            const css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`);
+            if (css === src) {
+                throw new Error("spritesheet.css: no webp urls rewritten");
+            }
+            // Suffix from source content: the rewritten css is a pure function
+            // of (css, webp), so this changes exactly when the served bytes
+            // change.
+            const ch = ctx.hash(sheetCss!, sheetWebp!);
+            ctx.write(`spritesheet-${ch}.css`, css);
+            ctx.write("__meta/spritesheet_css_suffix.txt", `-${ch}\n`);
+        }
+
+        {
+            const h = ctx.hash(...webpMinisprites);
+            for (const f of webpMinisprites) {
+                newspritecopy(ctx, f, {dir: "minisprites/" + h});
+            }
+            ctx.write("__meta/minisprites-hash.txt", h);
+        }
+
+        {
+            const manifest : Manifest = {};
+            for (const f of forumItems) {
+                itemspritecopy(ctx, f, {dir: "forumsprites"}, manifest);
+            }
+            for (const f of forumG6) {
+                spritecopy(ctx, f, {dir: "forumsprites"}, true, manifest);
+            }
+            writeManifest(ctx, "__meta/forumsprites/manifest.json", manifest);
+        }
+
+        {
+            const manifest : Manifest = {};
+            for (const f of ctx.list("src/pmd")) {
+                spritecopy(ctx, f, {dir: "pmd"}, false, manifest);
+            }
+            writeManifest(ctx, "__meta/pmd/manifest.json", manifest);
+        }
+    },
+});

ps.deploy.ts

@@ -0,0 +1,143 @@
+
+import * as spritedata from '@smogon/sprite-data/index.ts';
+
+import {gen10Modelslike, gen5Gifs} from './rules/modelslike.ts';
+import {type Sprite, toPSID} from './rules/publish.ts';
+import {forEachRule, memo, rule} from './tools/build/artifact.ts';
+import {base, compresspng, pad, spriteglob} from './tools/build/helpers.ts';
+import {type DeployCtx, defineDeploy} from './tools/deploy/api.ts';
+
+// PS spritesheets. The sheet tools readdir the minisprite dirs and resolve
+// ids through the sprite data, baking input names into the sheet layout,
+// hence nameSensitive.
+
+const sheetDeps = [
+    "data/species.json",
+    "data/items.json",
+    "data/lib/index.ts",
+    "lib/root/index.ts",
+    "tools/sheet/index.ts",
+];
+
+rule("ps-pokemon.sheet.mjs", {
+    display: "ps pokemon sheet",
+    nameSensitive: true,
+    deps: ["src/minisprites/pokemon/gen6/*", ...sheetDeps],
+    cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
+}, ["pokemonicons-sheet.png"]);
+
+// TODO: reenable when trainers are moved
+// rule("ps-trainers.sheet.mjs", {
+//     display: "ps trainers sheet",
+//     nameSensitive: true,
+//     cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
+// }, ["trainers-sheet.png"]);
+
+rule("ps-items.sheet.mjs", {
+    display: "ps items sheet",
+    nameSensitive: true,
+    deps: ["src/minisprites/items/*", ...sheetDeps],
+    cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
+}, ["itemicons-sheet.png"]);
+
+// PS pokeball icons; input order is the sheet order.
+
+rule([
+    "src/_uncategorized/noncanonical/ui/battle/Ball-Normal.png",
+    "src/_uncategorized/noncanonical/ui/battle/Ball-Sick.png",
+    "src/_uncategorized/noncanonical/ui/battle/Ball-Null.png",
+], {
+    display: "pokemonicons-pokeball-sheet",
+    cmds: [
+        "magick convert -background transparent -gravity center -extent 40x30 %f +append %o",
+        compresspng({config: "SPRITESHEET"}),
+    ],
+}, ["pokemonicons-pokeball-sheet.png"]);
+
+// Padded Dex, plus missing CAPs backfilled from the gen5/model gifs.
+
+const paddedDex = memo(() => {
+    const dex = forEachRule("src/dex/*", {
+        display: "pad dex %f",
+        cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})],
+    }, "%b");
+
+    const dexSet = new Set(dex.map(base));
+    const dexMissing = [];
+    for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) {
+        if (!dexSet.has(base(file))) {
+            dexMissing.push(file);
+            dexSet.add(base(file));
+        }
+    }
+
+    return dex.concat(forEachRule(dexMissing, {
+        display: "missing dex %B",
+        cmds: [
+            'magick convert "%f[0]" -trim %o',
+            'magick mogrify -background transparent -gravity center -resize "120x120>" -extent 120x120 %o',
+            compresspng({config: "DEX"}),
+        ],
+    }, "%B.png"));
+});
+paddedDex();
+
+const aniChampions = gen10Modelslike();
+gen5Gifs();
+
+// PS ids keep the forme dash, unlike the smogon aliases.
+function psSpritecopy(ctx : DeployCtx, f : Sprite, dir : string) : void {
+    const sn = spritedata.parseFilename(f.name);
+    let name : string;
+
+    // Skip asymmetrical for now
+    if (sn.extra.has("a") || sn.extra.has("b") || sn.extra.has("s")) {
+        return;
+    }
+
+    if (sn.extension) {
+        // Skip this, we don't use Unknown/Substitute
+        return;
+    }
+    const sd = spritedata.get(sn.id);
+    if (sd.type !== 'specie') {
+        throw new Error(`Not a specie sprite: ${f.name}`);
+    }
+    name = toPSID(sd.base);
+    if (sd.forme) {
+        name += `-${toPSID(sd.forme)}`;
+    }
+    if (sn.extra.has("f")) {
+        name += "-f";
+    }
+    if (sn.extra.has("g")) {
+        name += "-gmax";
+    }
+
+    if (f.ext === null) {
+        throw new Error(`Sprite ${f.name} has no extension`);
+    }
+    ctx.copy(f, `${dir}/${name}.${f.ext}`);
+}
+
+export default defineDeploy({
+    finish(ctx) {
+        const seenModels = new Set<string>();
+
+        for (const f of ctx.list("src/models")) {
+            seenModels.add(f.name);
+            psSpritecopy(ctx, f, "ani");
+        }
+
+        for (const f of aniChampions) {
+            if (seenModels.has(f.name)) {
+                continue;
+            }
+            seenModels.add(f.name);
+            psSpritecopy(ctx, f, "ani");
+        }
+
+        // TODO: ship the padded dex, sheets, trainers, types/categories when
+        // the PS deploy is revived; the rules above keep them building.
+    },
+});

rules/minisprites.ts

@@ -0,0 +1,25 @@
+
+import {forEachRule, memo} from '../tools/build/artifact.ts';
+import {compresspng, pad, trimimg} from '../tools/build/helpers.ts';
+
+// Uniform size minisprites
+
+export const gen6Padded = memo(() => forEachRule("src/minisprites/pokemon/gen6/*.png", {
+    display: "pad g6 minisprite %f",
+    cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})],
+}, "%b"));
+
+export const itemPadded = memo(() => forEachRule("src/minisprites/items/*.png", {
+    display: "pad item minisprite %f",
+    cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})],
+}, "%b"));
+
+export const gen6Trimmed = memo(() => forEachRule("src/minisprites/pokemon/gen6/*.png", {
+    display: "trim g6 minisprite %f",
+    cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
+}, "%b"));
+
+export const itemTrimmed = memo(() => forEachRule("src/minisprites/items/*.png", {
+    display: "trim item minisprite %f",
+    cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
+}, "%b"));

rules/modelslike.ts

@@ -0,0 +1,35 @@
+
+import {forEachRule, memo} from '../tools/build/artifact.ts';
+
+// Gen 9
+
+export const gen9Modelslike = memo(() => forEachRule("src/gen9species/*.png", {
+    display: "96x96 %f",
+    // TODO, add customizable compression for gif
+    // ... or investigate using webp instead of both png/gif here
+    cmds: [
+        "magick convert %f -trim +repage -resize 90x90 %o",
+        "gifsicle -O3 -b %o",
+    ],
+}, "%B.gif"));
+
+// Gen 10
+
+export const gen10Modelslike = memo(() => forEachRule("src/champions/*.png", {
+    display: "96x96 %f",
+    // TODO, add customizable compression for gif
+    // ... or investigate using webp instead of both png/gif here
+    cmds: [
+        "magick convert %f -trim +repage -resize 90x90 %o",
+        "gifsicle -O3 -b %o",
+    ],
+}, "%B.gif"));
+
+// Gen 5 CAPs...
+
+export const gen5Gifs = memo(() => forEachRule("src/sprites/gen5/*.png", [
+    // TODO, add customizable compression for gif
+    // ... or investigate using webp instead of both png/gif here
+    "magick convert %f %o",
+    "gifsicle -O3 -b %o",
+], "%B.gif"));

rules/publish.ts

@@ -0,0 +1,137 @@
+
+import * as spritedata from '@smogon/sprite-data/index.ts';
+
+import type {Artifact} from '../tools/build/artifact.ts';
+import type {DeployCtx, SrcFile} from '../tools/deploy/api.ts';
+
+// Anything with a sprite-id nominal name and an extension: a built artifact
+// or a raw source file.
+export type Sprite = Artifact | SrcFile;
+
+export type Manifest = Record<string, string>;
+
+export interface Dest {
+    dir : string;
+    ext? : string;
+}
+
+function extOf(f : Sprite, ext? : string) : string {
+    const result = ext ?? f.ext;
+    if (result === null) {
+        throw new Error(`Sprite ${f.name} has no extension`);
+    }
+    return result;
+}
+
+export function toSmogonAlias(name : string) : string {
+    return name.toLowerCase().
+        replace(/[ _]+/, "-").
+        replace(/[^a-z0-9-]+/g, '');
+}
+
+export function toPSID(name : string) : string {
+    return name.toLowerCase().replace(/[^a-z0-9]+/g, '');
+}
+
+// Copy with a content-hash-stamped name and record the unhashed -> hashed
+// mapping in `manifest`.
+export function stampcopy(ctx : DeployCtx, f : Sprite, {dir, ext}: Dest, name : string,
+                          manifest : Manifest) : void {
+    const key = `${name}.${extOf(f, ext)}`;
+    // ActionQueue only dedups final dsts; hashed dsts differ even when
+    // unhashed names collide, so check the manifest key explicitly.
+    if (manifest[key] !== undefined) {
+        throw new Error(`duplicate sprite name ${key}`);
+    }
+    const h = ctx.hash(f);
+    manifest[key] = `${name}-${h}.${extOf(f, ext)}`;
+    ctx.copy(f, `${dir}/${name}-${h}.${extOf(f, ext)}`);
+}
+
+export function writeManifest(ctx : DeployCtx, dst : string, manifest : Manifest) : void {
+    const sorted : Manifest = {};
+    for (const k of Object.keys(manifest).sort()) {
+        sorted[k] = manifest[k]!;
+    }
+    ctx.write(dst, JSON.stringify(sorted, null, 4) + "\n");
+}
+
+function emit(ctx : DeployCtx, f : Sprite, dest : Dest, name : string,
+              manifest : Manifest | null) : void {
+    if (manifest) {
+        stampcopy(ctx, f, dest, name, manifest);
+    } else {
+        ctx.copy(f, `${dest.dir}/${name}.${extOf(f, dest.ext)}`);
+    }
+}
+
+export function spritecopy(ctx : DeployCtx, f : Sprite, dest : Dest,
+                           allowUnknown = false, manifest : Manifest | null = null) : void {
+    const sn = spritedata.parseFilename(f.name);
+    let name : string;
+
+    // Skip asymmetrical for now
+    if (sn.extra.has("a") || sn.extra.has("b") || sn.extra.has("s")) {
+        return;
+    }
+
+    if (sn.extension) {
+        if (allowUnknown && sn.name === "Unknown") {
+            name = "unknown";
+        } else {
+            // Skip this, we don't use Unknown/Substitute
+            return;
+        }
+    } else {
+        const sd = spritedata.get(sn.id);
+        if (sd.type !== 'specie') {
+            throw new Error(`Not a specie sprite: ${f.name}`);
+        }
+        name = toSmogonAlias(sd.base);
+        if (sd.forme) {
+            name += `-${toSmogonAlias(sd.forme)}`;
+        }
+    }
+    if (sn.extra.has("f")) {
+        name += "-f";
+    }
+    if (sn.extra.has("g")) {
+        name += "-gmax";
+    }
+
+    emit(ctx, f, dest, name, manifest);
+}
+
+// TODO: merge with above
+export function itemspritecopy(ctx : DeployCtx, f : Sprite, dest : Dest,
+                               manifest : Manifest | null = null) : void {
+    const sn = spritedata.parseFilename(f.name);
+    if (sn.extension) {
+        throw new Error(`Not an item sprite: ${f.name}`);
+    }
+    const sd = spritedata.get(sn.id);
+    if (sd.type !== 'item') {
+        throw new Error(`Not an item sprite: ${f.name}`);
+    }
+    for (const n of sd.names) {
+        emit(ctx, f, dest, toSmogonAlias(n), manifest);
+    }
+}
+
+export function newspritecopy(ctx : DeployCtx, f : Sprite, dest : Dest) : void {
+    const sn = spritedata.parseFilename(f.name);
+    if (sn.extension) {
+        return;
+    }
+    const sd = spritedata.get(sn.id);
+    for (const n of sd.type === 'item' ? sd.names : [sd.base + sd.forme]) {
+        let name = toPSID(n);
+        if (sn.extra.has("f")) {
+            name += "f";
+        }
+        if (sn.extra.has("g")) {
+            name += "gmax";
+        }
+        ctx.copy(f, `${dest.dir}/${name}.${extOf(f, dest.ext)}`);
+    }
+}

rules/social.ts

@@ -0,0 +1,34 @@
+
+import {forEachRule, memo} from '../tools/build/artifact.ts';
+import {base, compresspng, spriteglob} from '../tools/build/helpers.ts';
+
+// Smogdex social images: models, backfilled with gen9 species not yet in
+// models (first source wins).
+
+const socialInputs = memo(() => {
+    const social = spriteglob(["src/models/*"], {b: false, s: false});
+    const socialSeen = new Set(social.map(base));
+    for (const file of spriteglob(["src/gen9species/*"], {b: false, s: false})) {
+        if (!socialSeen.has(base(file))) {
+            social.push(file);
+            socialSeen.add(base(file));
+        }
+    }
+    return social;
+});
+
+export const fbSprites = memo(() => forEachRule(socialInputs(), {
+    display: "fbsprite %f",
+    cmds: [
+        'magick convert "%f[0]" -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o',
+        compresspng({config: "MODELS"}),
+    ],
+}, "%B.png"));
+
+export const twitterSprites = memo(() => forEachRule(socialInputs(), {
+    display: "twittersprite %f",
+    cmds: [
+        'magick convert "%f[0]" -trim -resize 115x115 -background white -gravity center -extent 120x120 %o',
+        compresspng({config: "MODELS"}),
+    ],
+}, "%B.png"));

smogon.deploy.ts

@@ -0,0 +1,69 @@
+
+import {gen6Trimmed, itemTrimmed} from './rules/minisprites.ts';
+import {gen10Modelslike, gen5Gifs, gen9Modelslike} from './rules/modelslike.ts';
+import {itemspritecopy, type Manifest, type Sprite, spritecopy, writeManifest} from './rules/publish.ts';
+import {fbSprites, twitterSprites} from './rules/social.ts';
+import {defineDeploy} from './tools/deploy/api.ts';
+
+const xyModels = gen9Modelslike();
+const xyChampions = gen10Modelslike();
+const xyGen5 = gen5Gifs();
+const xyIcons = gen6Trimmed();
+const xyItems = itemTrimmed();
+const fb = fbSprites();
+const twitter = twitterSprites();
+
+export default defineDeploy({
+    finish(ctx) {
+        // xy/ animations: first source wins per sprite name.
+        const seenModels = new Set<string>();
+        const xyManifest : Manifest = {};
+        const xycopy = (f : Sprite) => {
+            if (seenModels.has(f.name)) {
+                return;
+            }
+            seenModels.add(f.name);
+            spritecopy(ctx, f, {dir: "xy"}, false, xyManifest);
+        };
+
+        for (const f of ctx.list("src/models")) {
+            xycopy(f);
+        }
+        for (const f of xyModels) {
+            xycopy(f);
+        }
+        for (const f of xyChampions) {
+            xycopy(f);
+        }
+        // Non-model CAPs
+        for (const f of ctx.list("src/sprites/gen5")) {
+            if (f.ext === 'gif') {
+                xycopy(f);
+            }
+        }
+        for (const f of xyGen5) {
+            xycopy(f);
+        }
+        writeManifest(ctx, "xy/manifest.json", xyManifest);
+
+        {
+            const manifest : Manifest = {};
+            for (const f of xyIcons) {
+                spritecopy(ctx, f, {dir: "xyicons"}, false, manifest);
+            }
+            writeManifest(ctx, "xyicons/manifest.json", manifest);
+        }
+
+        for (const f of xyItems) {
+            itemspritecopy(ctx, f, {dir: "xyitems"});
+        }
+
+        for (const f of fb) {
+            spritecopy(ctx, f, {dir: "fbsprites/xy"});
+        }
+
+        for (const f of twitter) {
+            spritecopy(ctx, f, {dir: "twittersprites/xy"});
+        }
+    },
+});

tools/build/store.ts

@@ -162,6 +162,20 @@ export class Store {
     }
 }
 
+// Schema version of an existing db, without opening it for writing (the
+// Store constructor migrates); null if there is no db.
+export function dbVersion(dbPath : string) : number | null {
+    if (!fs.existsSync(dbPath)) {
+        return null;
+    }
+    const db = new Database(dbPath, {readonly: true});
+    try {
+        return Number(db.pragma('user_version', {simple: true}));
+    } finally {
+        db.close();
+    }
+}
+
 // Guards against two concurrent builds; the exclusive transaction is released
 // by the OS on any crash, so there are no stale lock files.
 export function acquireLock(lockPath : string) : () => void {

tsconfig.json

@@ -1,7 +1,9 @@
 {
     "extends": "./tsconfig-base",
-    "files": ["Buildfile.ts"],
+    "include": ["Buildfile.ts", "*.deploy.ts", "rules/*.ts"],
     "references": [
-        {"path": "tools/build"}
+        {"path": "tools/build"},
+        {"path": "tools/deploy"},
+        {"path": "data/lib"}
     ]
 }