From e3d3b1c7f0c7d6dda90ad664b434b834c2e88e2e Mon Sep 17 00:00:00 2001 From: Christopher Monsanto Date: Sun, 23 Aug 2026 15:22:45 -0400 Subject: [PATCH] Merge the two build files, and stamp the smogon sets f278ed4d gave up the stamps on xy/ and xyicons/ because the smogdex composes sprites/xy/charizard.gif itself and reads no manifest, so a stamped tree was one it couldn't read. A mirror of the tree in links answers that without asking the dex to change: the files land under stamped names with a manifest beside them, and every bare name is a link naming the stamped file. The mirror rides in __meta/links/ rather than the served tree, because the tree is add-only -- a name in it is promised never to change -- and a link is repointed on every upload; the upload diverts __meta/ to assets-meta/ and retargets each link at where its asset landed on that side. That leaves nothing separating the two build files, so they become one. Both already imported rules/publish.ts, and the coverage rule is per deploy name over a whole buildFile's outputs, so one file is also one deploy: everything ships through the asset upload, and the two rsync entries go with the untracked deploy.json5 they lived in. Manifest keeps where each copy landed rather than the composed url, which is what a link needs and what write() composes from anyway, and takes the tree it publishes into rather than just that tree's url, so `sprites` is written once and a Dest names a place under it. That is also what keeps the mirror from repeating it: the far side files a key's metadata under a directory of its own, and assets-meta/sprites/links/sprites/ says it twice. The url-less Manifest mode the rsync sets used goes, having no caller left, and xyicons's byName map was only ever a duplicate check, which Manifest.copy performs, so it collapses into spritecopy. 3614 published names are untouched, 3207 move to a stamped name carrying the same bytes, and 3209 are new: the mirror and the two manifests. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 32 +++++++--- assets.build.ts | 85 -------------------------- rules/publish.ts | 75 ++++++++++++++++------- smogon.build.ts | 152 +++++++++++++++++++++++++++++++++++------------ 4 files changed, 191 insertions(+), 153 deletions(-) delete mode 100644 assets.build.ts diff --git a/README.md b/README.md index 8ab05470..1bf68f96 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ reuses the build's digests. All state lives in `.build/`. ``` $ pnpm build # build every deploy's rules, GC stale state $ pnpm deploy # list the deploys in deploy.json5 -$ pnpm deploy assets # run a named deploy +$ pnpm deploy smogon # run a named deploy $ node tools/deploy/index.ts build smogon.build.ts # build one deploy's rules $ node tools/deploy/index.ts run smogon.build.ts -o deploy/smogon $ node tools/deploy/index.ts inspect src/minisprites/items/ileftovers.png -o /tmp/out @@ -106,26 +106,24 @@ repo ships to are written down. ```json5 { - assets: { - buildFile: "assets.build.ts", - deploy: [ - {subset: ["**"], cmd: "smogonctl assets upload sprites"}, - ], - }, smogon: { buildFile: "smogon.build.ts", deploy: [ - {subset: ["xy/**"], dir: true, cmd: "rsync -a --delete-after %d/xy/ :/xy"}, + {subset: ["**"], cmd: "smogonctl assets upload sprites"}, ], }, } ``` +Note that the coverage rule is per deploy name, over the whole buildFile's +outputs: two names on one buildFile can't split its tree between them, since +each of them has to cover all of it. + ### The asset upload's tar layout `smogonctl assets upload` publishes a tar into a served tree under a prefix named in the receiving home's `services.toml`, which this side can't read. So -`assets.build.ts` writes that prefix itself -- everything served ships under +`smogon.build.ts` writes that prefix itself -- everything served ships under `sprites/` -- and the upload rejects a tar whose tree disagrees. The two are checked against each other instead of each guessing, which is what lets the manifests and pointers in `__meta/` name whole urls (`/__assets/sprites/...`) @@ -136,6 +134,22 @@ and their readers hold no configuration at all. name carries a content hash and something un-stamped has to say which name to ask for. +Three kinds of thing ride there. A manifest, `name` -> the whole url of the +stamped file, for a reader that looks one up. A pointer file naming a single +url, for the one-file sets. And `__meta/links/`, the served tree recreated as +symlinks under the un-stamped names, which is the same mapping said in names +instead of a file. It lives out here rather than in the tree because the tree +is add-only -- a name in it is promised never to change -- and a link is +repointed on every upload. + +A link says where in the tar its file is, not what to write into the link: +`__meta/links/xy/charizard.gif` names `sprites/xy/charizard-.gif` and +what is packed is the path between the two. The two halves land in different +trees on the far side (`assets-meta//` and `assets/`), so the upload +retargets every link at where its asset actually went, and refuses one naming +anything that tar didn't carry. The mirror doesn't repeat the `sprites/` +prefix, since the directory it lands in is already this set's. + ## Configuration Build settings are configurable in `build.config` (not tracked by git). diff --git a/assets.build.ts b/assets.build.ts deleted file mode 100644 index 40767972..00000000 --- a/assets.build.ts +++ /dev/null @@ -1,85 +0,0 @@ - -import {Manifest, itemspritecopy, spritecopy} from './rules/publish.ts'; -import {forEachRule, rule} from './tools/build/artifact.ts'; -import {compresspng, pad, spriteglob} from './tools/build/helpers.ts'; -import {deploy} from './tools/deploy/api.ts'; - -// The tar root maps onto the served tree: sprites/x is served at -// /__assets/sprites/x. The upload rejects a tar whose tree disagrees with the -// prefix in services.toml, so the two are checked against each other rather -// than each guessing -- which is what lets the pointers below name whole urls -// and their readers hold no configuration. __meta/ is the exception: the -// upload diverts it to assets-meta/, beside the served tree and out of it. - -let ASSETS = 'sprites'; -let SERVED = '/__assets'; - -let minispriteInputs = spriteglob(['src/minisprites/pokemon/gen6/*', 'src/minisprites/items/*'], {a: false}); - -// Smogdex spritesheet. The sheet tool bakes the names parsed from the %f -// filenames into the css, hence nameSensitive. The png is declared only so -// cwebp has something to read; only the css and the webp are published. - -let [, sheetCss, sheetWebp] = rule(minispriteInputs, { - display: 'smogdex sheet', - nameSensitive: true, - deps: [ - 'data/lib/index.ts', - 'tools/smogdexspritesheet/index.ts', - ], - cmds: [ - 'node tools/smogdexspritesheet/index.ts --image %o1 --stylesheet %o2 -- %f', - 'cwebp -z 9 %o1 -o %o3', - ], -}, ['spritesheet.png', 'spritesheet.css', 'spritesheet.webp']); - -// Hash-stamped css + webp. The css url rides in __meta/ for the dex to read. -deploy(async ctx => { - let wh = await ctx.hash(sheetWebp); - ctx.copy(sheetWebp, `${ASSETS}/spritesheet-${wh}.webp`); - let src = await ctx.read(sheetCss); - let css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`); - if (css === src) { - throw new Error('spritesheet.css: no webp urls rewritten'); - } - // Suffix from source content: the rewritten css is a pure function - // of (css, webp), so this changes exactly when the served bytes - // change. - let ch = await ctx.hash(sheetCss, sheetWebp); - ctx.write(`${ASSETS}/spritesheet-${ch}.css`, css); - ctx.write('__meta/spritesheet-css-url.txt', `${SERVED}/${ASSETS}/spritesheet-${ch}.css\n`); -}); - -// Forumsprites: uniform-size minisprites under stamped names, with the -// unhashed -> url mapping in a manifest. - -let forumItems = forEachRule('src/minisprites/items/*.png', { - display: 'pad item minisprite %f', - cmds: [pad({w: 24, h: 24}), compresspng({config: 'MINISPRITE'})], -}, '%b'); - -let forumG6 = forEachRule('src/minisprites/pokemon/gen6/*.png', { - display: 'pad g6 minisprite %f', - cmds: [pad({w: 40, h: 30}), compresspng({config: 'MINISPRITE'})], -}, '%b'); - -deploy(async ctx => { - let manifest = new Manifest(ctx, SERVED); - for (let f of forumItems) { - await itemspritecopy(manifest, f, {dir: `${ASSETS}/forumsprites`}); - } - for (let f of forumG6) { - await spritecopy(manifest, f, {dir: `${ASSETS}/forumsprites`}, {allowUnknown: true, icons: true}); - } - manifest.write('__meta/forumsprites/manifest.json'); -}); - -// PMD sprites ship as-is, stamped. - -deploy(async ctx => { - let manifest = new Manifest(ctx, SERVED); - for (let f of await ctx.list('src/pmd')) { - await spritecopy(manifest, f, {dir: `${ASSETS}/pmd`}); - } - manifest.write('__meta/pmd/manifest.json'); -}); diff --git a/rules/publish.ts b/rules/publish.ts index d16030f2..c5fd3fff 100644 --- a/rules/publish.ts +++ b/rules/publish.ts @@ -8,19 +8,34 @@ import type {DeployCtx, SrcFile} from '../tools/deploy/api.ts'; // or a raw source file. export type Sprite = Artifact | SrcFile; -// The unhashed name -> served url mapping published beside a stamped set. -// `base` is the url this deploy's tree is served at, so an entry is just that -// plus the copy's destination and no consumer needs a prefix of its own. The -// rsync sets don't know their url yet and pass none, keeping the older -// `name.ext` -> stamped filename shape. +// The published tree a stamped set goes into: where it sits in the output, and +// the url that root is served at. Both live here rather than in a `dir` so +// that they are written once and everything a set publishes -- the copies, the +// urls, the mirror -- is spelled from the same place. +export type Tree = { + root: string, + served: string, +}; + +// One copy this manifest made: where under the tree root it landed, and the +// un-stamped filename it answers to. +type Entry = { + dir: string, + stamped: string, + filename: string, +}; + +// The unhashed name -> served url mapping published beside a stamped set. An +// entry is the tree's url plus the copy's place in it, so no consumer needs a +// prefix of its own. export class Manifest { #ctx: DeployCtx; - #base: string | null; - #entries = new Map(); + #tree: Tree; + #entries = new Map(); - constructor(ctx: DeployCtx, base: string | null = null) { + constructor(ctx: DeployCtx, tree: Tree) { this.#ctx = ctx; - this.#base = base; + this.#tree = tree; } // Queue a copy of `f` under `dir` with a content-stamped name and record @@ -28,30 +43,46 @@ export class Manifest { async copy(f: Sprite, {dir, ext}: Dest, name: string): Promise { let h = await this.#ctx.hash(f); let e = extOf(f, ext); - let stamped = `${name}-${h}.${e}`; - let dst = `${dir}/${stamped}`; - // A url names the whole path, so its key has nothing to disambiguate - // with an extension. - let key = this.#base === null ? `${name}.${e}` : name; - // ActionQueue only dedups final dsts; hashed dsts differ even when - // unhashed names collide, so check the key explicitly. - if (this.#entries.has(key)) { - throw new Error(`duplicate sprite name ${key}`); + // Keyed on the name alone, since a url names the whole path and has + // nothing to disambiguate with an extension. ActionQueue only dedups + // final dsts, and hashed dsts differ even where unhashed names + // collide, so the collision is caught here or not at all. + if (this.#entries.has(name)) { + throw new Error(`duplicate sprite name ${name}`); } - this.#entries.set(key, this.#base === null ? stamped : `${this.#base}/${dst}`); - this.#ctx.copy(f, dst); + let stamped = `${name}-${h}.${e}`; + this.#entries.set(name, {dir, stamped, filename: `${name}.${e}`}); + this.#ctx.copy(f, `${this.#tree.root}/${dir}/${stamped}`); } write(dst: string): void { let sorted: Record = {}; - for (let [k, v] of [...this.#entries].sort((a, b) => a[0] < b[0] ? -1 : 1)) { - sorted[k] = v; + for (let [k, e] of this.#sorted()) { + sorted[k] = `${this.#tree.served}/${e.dir}/${e.stamped}`; } this.#ctx.write(dst, JSON.stringify(sorted, null, 4) + '\n'); } + + // The same set again under its un-stamped names, as links to the stamped + // files. `dst` is a root of its own rather than the tree's, so what a link + // says is a path and not a name: the two ends land apart, and the reader + // that follows one composed its half from nothing but the sprite. The + // tree's own root is not repeated under it, since the mirror is already + // one set's worth of it. + links(dst: string): void { + for (let [, e] of this.#sorted()) { + this.#ctx.symlink(`${dst}/${e.dir}/${e.filename}`, + `${this.#tree.root}/${e.dir}/${e.stamped}`); + } + } + + #sorted(): [string, Entry][] { + return [...this.#entries].sort((a, b) => a[0] < b[0] ? -1 : 1); + } } export type Dest = { + // Under the tree root, not from the output root. dir: string, ext?: string, }; diff --git a/smogon.build.ts b/smogon.build.ts index d6cb0649..02559970 100644 --- a/smogon.build.ts +++ b/smogon.build.ts @@ -1,11 +1,35 @@ import {gen10Modelslike} from './rules/modelslike.ts'; -import {type Sprite, publishedNames} from './rules/publish.ts'; -import {forEachRule} from './tools/build/artifact.ts'; -import {compresspng, trimimg} from './tools/build/helpers.ts'; -import {type DeployCtx, deploy} from './tools/deploy/api.ts'; +import {Manifest, type Sprite, type Tree, itemspritecopy, publishedNames, spritecopy} from './rules/publish.ts'; +import {forEachRule, rule} from './tools/build/artifact.ts'; +import {compresspng, pad, spriteglob, trimimg} from './tools/build/helpers.ts'; +import {deploy} from './tools/deploy/api.ts'; -// xy/ animations: first source wins per sprite name. +// The tar root maps onto the served tree: sprites/x is served at +// /__assets/sprites/x. The upload rejects a tar whose tree disagrees with the +// prefix in services.toml, so the two are checked against each other rather +// than each guessing -- which is what lets the pointers below name whole urls +// and their readers hold no configuration. __meta/ is the exception: the +// upload diverts it to assets-meta/, beside the served tree and out of it. + +let ASSETS = 'sprites'; +let SERVED = '/__assets'; +let TREE: Tree = {root: ASSETS, served: `${SERVED}/${ASSETS}`}; + +// Where the un-stamped names go. A served name carries a content hash and so +// can be cached forever, which is exactly why it can't be composed by a reader +// that knows only the sprite: the smogdex asks for sprites/xy/charizard.gif +// and reads no manifest. So the sets it composes paths into publish the tree a +// second time, as links under the un-stamped names naming the stamped file. +// They ride in __meta/ rather than the served tree because the tree is +// add-only -- a name in it is promised never to change -- and a link is +// repointed on every upload; the upload retargets each one at where its asset +// landed on its side, under a directory it already names for this set. +let LINKS = '__meta/links'; + +let minispriteInputs = spriteglob(['src/minisprites/pokemon/gen6/*', 'src/minisprites/items/*'], {a: false}); + +// sprites/xy/ animations: first source wins per sprite name. let xyModels = forEachRule('src/gen9species/*.png', { display: '96x96 %f', @@ -31,81 +55,135 @@ let xyGen5 = forEachRule('src/sprites/gen5/*.png', [ ], '%B.gif'); deploy(async ctx => { + let manifest = new Manifest(ctx, TREE); let seen = new Set(); // First source wins per published name rather than per filename, because // the later sources are backfills and one name can be spelled several // ways. gen 5 carries a sprite per forme slot, so its six Minior meteors // and its two Zygarde Power Construct slots all want the name the models - // already published, and two copies to one path is an invalid queue. - let xycopy = (f: Sprite) => { + // already published, which the manifest would refuse as a duplicate. + let xycopy = async (f: Sprite) => { let names = publishedNames(f); if (names.some(n => seen.has(n))) { return; } for (let name of names) { seen.add(name); + await manifest.copy(f, {dir: 'xy'}, name); } - smogonSpritecopy(ctx, f, 'xy', names); }; for (let f of await ctx.list('src/models')) { - xycopy(f); + await xycopy(f); } for (let f of xyModels) { - xycopy(f); + await xycopy(f); } for (let f of xyChampions) { - xycopy(f); + await xycopy(f); } for (let f of await ctx.list('src/sprites/gen5')) { if (f.ext === 'gif') { - xycopy(f); + await xycopy(f); } } for (let f of xyGen5) { - xycopy(f); + await xycopy(f); } + manifest.write('__meta/xy/manifest.json'); + manifest.links(LINKS); }); -// xyicons/: trimmed gen6 minisprites. +// sprites/xyicons/: trimmed gen6 minisprites. let xyIcons = forEachRule('src/minisprites/pokemon/gen6/*.png', { display: 'trim g6 minisprite %f', cmds: [trimimg(), compresspng({config: 'MINISPRITE'})], }, '%b'); -deploy(ctx => { +deploy(async ctx => { // icons: the gen 6 set has no art for some formes and lends them another's, - // which the smogdex sheet and forumsprites do off the same directory. - let byName = new Map(); + // which the smogdex sheet and forumsprites do off the same directory. Two + // formes lent the same icon are a duplicate name the manifest refuses. + let manifest = new Manifest(ctx, TREE); for (let f of xyIcons) { - for (let name of publishedNames(f, {icons: true})) { - if (byName.has(name)) { - throw new Error(`Two icons published as ${name}`); - } - byName.set(name, f); - } - } - for (let [name, f] of byName) { - smogonSpritecopy(ctx, f, 'xyicons', [name]); + await spritecopy(manifest, f, {dir: 'xyicons'}, {icons: true}); } + manifest.write('__meta/xyicons/manifest.json'); + manifest.links(LINKS); }); -// The smogon side asks for a fixed path, /sprites/xy/charizard.gif, and reads -// no manifest yet, so these copies carry no content stamp and the published -// name is the whole filename. -function smogonSpritecopy(ctx: DeployCtx, f: Sprite, dir: string, names: string[]): void { - if (f.ext === null) { - throw new Error(`Sprite ${f.name} has no extension`); +// Smogdex spritesheet. The sheet tool bakes the names parsed from the %f +// filenames into the css, hence nameSensitive. The png is declared only so +// cwebp has something to read; only the css and the webp are published. + +let [, sheetCss, sheetWebp] = rule(minispriteInputs, { + display: 'smogdex sheet', + nameSensitive: true, + deps: [ + 'data/lib/index.ts', + 'tools/smogdexspritesheet/index.ts', + ], + cmds: [ + 'node tools/smogdexspritesheet/index.ts --image %o1 --stylesheet %o2 -- %f', + 'cwebp -z 9 %o1 -o %o3', + ], +}, ['spritesheet.png', 'spritesheet.css', 'spritesheet.webp']); + +// Hash-stamped css + webp. The css url rides in __meta/ for the dex to read. +deploy(async ctx => { + let wh = await ctx.hash(sheetWebp); + ctx.copy(sheetWebp, `${ASSETS}/spritesheet-${wh}.webp`); + let src = await ctx.read(sheetCss); + let css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`); + if (css === src) { + throw new Error('spritesheet.css: no webp urls rewritten'); } - for (let name of names) { - ctx.copy(f, `${dir}/${name}.${f.ext}`); + // Suffix from source content: the rewritten css is a pure function + // of (css, webp), so this changes exactly when the served bytes + // change. + let ch = await ctx.hash(sheetCss, sheetWebp); + ctx.write(`${ASSETS}/spritesheet-${ch}.css`, css); + ctx.write('__meta/spritesheet-css-url.txt', `${SERVED}/${ASSETS}/spritesheet-${ch}.css\n`); +}); + +// Forumsprites: uniform-size minisprites under stamped names, with the +// unhashed -> url mapping in a manifest. + +let forumItems = forEachRule('src/minisprites/items/*.png', { + display: 'pad item minisprite %f', + cmds: [pad({w: 24, h: 24}), compresspng({config: 'MINISPRITE'})], +}, '%b'); + +let forumG6 = forEachRule('src/minisprites/pokemon/gen6/*.png', { + display: 'pad g6 minisprite %f', + cmds: [pad({w: 40, h: 30}), compresspng({config: 'MINISPRITE'})], +}, '%b'); + +deploy(async ctx => { + let manifest = new Manifest(ctx, TREE); + for (let f of forumItems) { + await itemspritecopy(manifest, f, {dir: 'forumsprites'}); } -} + for (let f of forumG6) { + await spritecopy(manifest, f, {dir: 'forumsprites'}, {allowUnknown: true, icons: true}); + } + manifest.write('__meta/forumsprites/manifest.json'); +}); + +// PMD sprites ship as-is, stamped. + +deploy(async ctx => { + let manifest = new Manifest(ctx, TREE); + for (let f of await ctx.list('src/pmd')) { + await spritecopy(manifest, f, {dir: 'pmd'}); + } + manifest.write('__meta/pmd/manifest.json'); +}); // Deprecated, unstamped sets. Reviving one also means importing what it -// uses (PNG_DETERMINISTIC, base, spriteglob, itemspritecopy) and giving the -// copies a Manifest, as the stamped deploys above do. +// uses (PNG_DETERMINISTIC, base) and giving the copies a Manifest, as the +// deploys above do. // // let xyItems = forEachRule('src/minisprites/items/*.png', { // display: 'trim item minisprite %f',