diff --git a/README.md b/README.md index 8ab05470..1bf68f96 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ reuses the build's digests. All state lives in `.build/`. ``` $ pnpm build # build every deploy's rules, GC stale state $ pnpm deploy # list the deploys in deploy.json5 -$ pnpm deploy assets # run a named deploy +$ pnpm deploy smogon # run a named deploy $ node tools/deploy/index.ts build smogon.build.ts # build one deploy's rules $ node tools/deploy/index.ts run smogon.build.ts -o deploy/smogon $ node tools/deploy/index.ts inspect src/minisprites/items/ileftovers.png -o /tmp/out @@ -106,26 +106,24 @@ repo ships to are written down. ```json5 { - assets: { - buildFile: "assets.build.ts", - deploy: [ - {subset: ["**"], cmd: "smogonctl assets upload sprites"}, - ], - }, smogon: { buildFile: "smogon.build.ts", deploy: [ - {subset: ["xy/**"], dir: true, cmd: "rsync -a --delete-after %d/xy/ :/xy"}, + {subset: ["**"], cmd: "smogonctl assets upload sprites"}, ], }, } ``` +Note that the coverage rule is per deploy name, over the whole buildFile's +outputs: two names on one buildFile can't split its tree between them, since +each of them has to cover all of it. + ### The asset upload's tar layout `smogonctl assets upload` publishes a tar into a served tree under a prefix named in the receiving home's `services.toml`, which this side can't read. So -`assets.build.ts` writes that prefix itself -- everything served ships under +`smogon.build.ts` writes that prefix itself -- everything served ships under `sprites/` -- and the upload rejects a tar whose tree disagrees. The two are checked against each other instead of each guessing, which is what lets the manifests and pointers in `__meta/` name whole urls (`/__assets/sprites/...`) @@ -136,6 +134,22 @@ and their readers hold no configuration at all. name carries a content hash and something un-stamped has to say which name to ask for. +Three kinds of thing ride there. A manifest, `name` -> the whole url of the +stamped file, for a reader that looks one up. A pointer file naming a single +url, for the one-file sets. And `__meta/links/`, the served tree recreated as +symlinks under the un-stamped names, which is the same mapping said in names +instead of a file. It lives out here rather than in the tree because the tree +is add-only -- a name in it is promised never to change -- and a link is +repointed on every upload. + +A link says where in the tar its file is, not what to write into the link: +`__meta/links/xy/charizard.gif` names `sprites/xy/charizard-.gif` and +what is packed is the path between the two. The two halves land in different +trees on the far side (`assets-meta//` and `assets/`), so the upload +retargets every link at where its asset actually went, and refuses one naming +anything that tar didn't carry. The mirror doesn't repeat the `sprites/` +prefix, since the directory it lands in is already this set's. + ## Configuration Build settings are configurable in `build.config` (not tracked by git). diff --git a/assets.build.ts b/assets.build.ts deleted file mode 100644 index 40767972..00000000 --- a/assets.build.ts +++ /dev/null @@ -1,85 +0,0 @@ - -import {Manifest, itemspritecopy, spritecopy} from './rules/publish.ts'; -import {forEachRule, rule} from './tools/build/artifact.ts'; -import {compresspng, pad, spriteglob} from './tools/build/helpers.ts'; -import {deploy} from './tools/deploy/api.ts'; - -// The tar root maps onto the served tree: sprites/x is served at -// /__assets/sprites/x. The upload rejects a tar whose tree disagrees with the -// prefix in services.toml, so the two are checked against each other rather -// than each guessing -- which is what lets the pointers below name whole urls -// and their readers hold no configuration. __meta/ is the exception: the -// upload diverts it to assets-meta/, beside the served tree and out of it. - -let ASSETS = 'sprites'; -let SERVED = '/__assets'; - -let minispriteInputs = spriteglob(['src/minisprites/pokemon/gen6/*', 'src/minisprites/items/*'], {a: false}); - -// Smogdex spritesheet. The sheet tool bakes the names parsed from the %f -// filenames into the css, hence nameSensitive. The png is declared only so -// cwebp has something to read; only the css and the webp are published. - -let [, sheetCss, sheetWebp] = rule(minispriteInputs, { - display: 'smogdex sheet', - nameSensitive: true, - deps: [ - 'data/lib/index.ts', - 'tools/smogdexspritesheet/index.ts', - ], - cmds: [ - 'node tools/smogdexspritesheet/index.ts --image %o1 --stylesheet %o2 -- %f', - 'cwebp -z 9 %o1 -o %o3', - ], -}, ['spritesheet.png', 'spritesheet.css', 'spritesheet.webp']); - -// Hash-stamped css + webp. The css url rides in __meta/ for the dex to read. -deploy(async ctx => { - let wh = await ctx.hash(sheetWebp); - ctx.copy(sheetWebp, `${ASSETS}/spritesheet-${wh}.webp`); - let src = await ctx.read(sheetCss); - let css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`); - if (css === src) { - throw new Error('spritesheet.css: no webp urls rewritten'); - } - // Suffix from source content: the rewritten css is a pure function - // of (css, webp), so this changes exactly when the served bytes - // change. - let ch = await ctx.hash(sheetCss, sheetWebp); - ctx.write(`${ASSETS}/spritesheet-${ch}.css`, css); - ctx.write('__meta/spritesheet-css-url.txt', `${SERVED}/${ASSETS}/spritesheet-${ch}.css\n`); -}); - -// Forumsprites: uniform-size minisprites under stamped names, with the -// unhashed -> url mapping in a manifest. - -let forumItems = forEachRule('src/minisprites/items/*.png', { - display: 'pad item minisprite %f', - cmds: [pad({w: 24, h: 24}), compresspng({config: 'MINISPRITE'})], -}, '%b'); - -let forumG6 = forEachRule('src/minisprites/pokemon/gen6/*.png', { - display: 'pad g6 minisprite %f', - cmds: [pad({w: 40, h: 30}), compresspng({config: 'MINISPRITE'})], -}, '%b'); - -deploy(async ctx => { - let manifest = new Manifest(ctx, SERVED); - for (let f of forumItems) { - await itemspritecopy(manifest, f, {dir: `${ASSETS}/forumsprites`}); - } - for (let f of forumG6) { - await spritecopy(manifest, f, {dir: `${ASSETS}/forumsprites`}, {allowUnknown: true, icons: true}); - } - manifest.write('__meta/forumsprites/manifest.json'); -}); - -// PMD sprites ship as-is, stamped. - -deploy(async ctx => { - let manifest = new Manifest(ctx, SERVED); - for (let f of await ctx.list('src/pmd')) { - await spritecopy(manifest, f, {dir: `${ASSETS}/pmd`}); - } - manifest.write('__meta/pmd/manifest.json'); -}); diff --git a/rules/publish.ts b/rules/publish.ts index d16030f2..c5fd3fff 100644 --- a/rules/publish.ts +++ b/rules/publish.ts @@ -8,19 +8,34 @@ import type {DeployCtx, SrcFile} from '../tools/deploy/api.ts'; // or a raw source file. export type Sprite = Artifact | SrcFile; -// The unhashed name -> served url mapping published beside a stamped set. -// `base` is the url this deploy's tree is served at, so an entry is just that -// plus the copy's destination and no consumer needs a prefix of its own. The -// rsync sets don't know their url yet and pass none, keeping the older -// `name.ext` -> stamped filename shape. +// The published tree a stamped set goes into: where it sits in the output, and +// the url that root is served at. Both live here rather than in a `dir` so +// that they are written once and everything a set publishes -- the copies, the +// urls, the mirror -- is spelled from the same place. +export type Tree = { + root: string, + served: string, +}; + +// One copy this manifest made: where under the tree root it landed, and the +// un-stamped filename it answers to. +type Entry = { + dir: string, + stamped: string, + filename: string, +}; + +// The unhashed name -> served url mapping published beside a stamped set. An +// entry is the tree's url plus the copy's place in it, so no consumer needs a +// prefix of its own. export class Manifest { #ctx: DeployCtx; - #base: string | null; - #entries = new Map(); + #tree: Tree; + #entries = new Map(); - constructor(ctx: DeployCtx, base: string | null = null) { + constructor(ctx: DeployCtx, tree: Tree) { this.#ctx = ctx; - this.#base = base; + this.#tree = tree; } // Queue a copy of `f` under `dir` with a content-stamped name and record @@ -28,30 +43,46 @@ export class Manifest { async copy(f: Sprite, {dir, ext}: Dest, name: string): Promise { let h = await this.#ctx.hash(f); let e = extOf(f, ext); - let stamped = `${name}-${h}.${e}`; - let dst = `${dir}/${stamped}`; - // A url names the whole path, so its key has nothing to disambiguate - // with an extension. - let key = this.#base === null ? `${name}.${e}` : name; - // ActionQueue only dedups final dsts; hashed dsts differ even when - // unhashed names collide, so check the key explicitly. - if (this.#entries.has(key)) { - throw new Error(`duplicate sprite name ${key}`); + // Keyed on the name alone, since a url names the whole path and has + // nothing to disambiguate with an extension. ActionQueue only dedups + // final dsts, and hashed dsts differ even where unhashed names + // collide, so the collision is caught here or not at all. + if (this.#entries.has(name)) { + throw new Error(`duplicate sprite name ${name}`); } - this.#entries.set(key, this.#base === null ? stamped : `${this.#base}/${dst}`); - this.#ctx.copy(f, dst); + let stamped = `${name}-${h}.${e}`; + this.#entries.set(name, {dir, stamped, filename: `${name}.${e}`}); + this.#ctx.copy(f, `${this.#tree.root}/${dir}/${stamped}`); } write(dst: string): void { let sorted: Record = {}; - for (let [k, v] of [...this.#entries].sort((a, b) => a[0] < b[0] ? -1 : 1)) { - sorted[k] = v; + for (let [k, e] of this.#sorted()) { + sorted[k] = `${this.#tree.served}/${e.dir}/${e.stamped}`; } this.#ctx.write(dst, JSON.stringify(sorted, null, 4) + '\n'); } + + // The same set again under its un-stamped names, as links to the stamped + // files. `dst` is a root of its own rather than the tree's, so what a link + // says is a path and not a name: the two ends land apart, and the reader + // that follows one composed its half from nothing but the sprite. The + // tree's own root is not repeated under it, since the mirror is already + // one set's worth of it. + links(dst: string): void { + for (let [, e] of this.#sorted()) { + this.#ctx.symlink(`${dst}/${e.dir}/${e.filename}`, + `${this.#tree.root}/${e.dir}/${e.stamped}`); + } + } + + #sorted(): [string, Entry][] { + return [...this.#entries].sort((a, b) => a[0] < b[0] ? -1 : 1); + } } export type Dest = { + // Under the tree root, not from the output root. dir: string, ext?: string, }; diff --git a/smogon.build.ts b/smogon.build.ts index d6cb0649..02559970 100644 --- a/smogon.build.ts +++ b/smogon.build.ts @@ -1,11 +1,35 @@ import {gen10Modelslike} from './rules/modelslike.ts'; -import {type Sprite, publishedNames} from './rules/publish.ts'; -import {forEachRule} from './tools/build/artifact.ts'; -import {compresspng, trimimg} from './tools/build/helpers.ts'; -import {type DeployCtx, deploy} from './tools/deploy/api.ts'; +import {Manifest, type Sprite, type Tree, itemspritecopy, publishedNames, spritecopy} from './rules/publish.ts'; +import {forEachRule, rule} from './tools/build/artifact.ts'; +import {compresspng, pad, spriteglob, trimimg} from './tools/build/helpers.ts'; +import {deploy} from './tools/deploy/api.ts'; -// xy/ animations: first source wins per sprite name. +// The tar root maps onto the served tree: sprites/x is served at +// /__assets/sprites/x. The upload rejects a tar whose tree disagrees with the +// prefix in services.toml, so the two are checked against each other rather +// than each guessing -- which is what lets the pointers below name whole urls +// and their readers hold no configuration. __meta/ is the exception: the +// upload diverts it to assets-meta/, beside the served tree and out of it. + +let ASSETS = 'sprites'; +let SERVED = '/__assets'; +let TREE: Tree = {root: ASSETS, served: `${SERVED}/${ASSETS}`}; + +// Where the un-stamped names go. A served name carries a content hash and so +// can be cached forever, which is exactly why it can't be composed by a reader +// that knows only the sprite: the smogdex asks for sprites/xy/charizard.gif +// and reads no manifest. So the sets it composes paths into publish the tree a +// second time, as links under the un-stamped names naming the stamped file. +// They ride in __meta/ rather than the served tree because the tree is +// add-only -- a name in it is promised never to change -- and a link is +// repointed on every upload; the upload retargets each one at where its asset +// landed on its side, under a directory it already names for this set. +let LINKS = '__meta/links'; + +let minispriteInputs = spriteglob(['src/minisprites/pokemon/gen6/*', 'src/minisprites/items/*'], {a: false}); + +// sprites/xy/ animations: first source wins per sprite name. let xyModels = forEachRule('src/gen9species/*.png', { display: '96x96 %f', @@ -31,81 +55,135 @@ let xyGen5 = forEachRule('src/sprites/gen5/*.png', [ ], '%B.gif'); deploy(async ctx => { + let manifest = new Manifest(ctx, TREE); let seen = new Set(); // First source wins per published name rather than per filename, because // the later sources are backfills and one name can be spelled several // ways. gen 5 carries a sprite per forme slot, so its six Minior meteors // and its two Zygarde Power Construct slots all want the name the models - // already published, and two copies to one path is an invalid queue. - let xycopy = (f: Sprite) => { + // already published, which the manifest would refuse as a duplicate. + let xycopy = async (f: Sprite) => { let names = publishedNames(f); if (names.some(n => seen.has(n))) { return; } for (let name of names) { seen.add(name); + await manifest.copy(f, {dir: 'xy'}, name); } - smogonSpritecopy(ctx, f, 'xy', names); }; for (let f of await ctx.list('src/models')) { - xycopy(f); + await xycopy(f); } for (let f of xyModels) { - xycopy(f); + await xycopy(f); } for (let f of xyChampions) { - xycopy(f); + await xycopy(f); } for (let f of await ctx.list('src/sprites/gen5')) { if (f.ext === 'gif') { - xycopy(f); + await xycopy(f); } } for (let f of xyGen5) { - xycopy(f); + await xycopy(f); } + manifest.write('__meta/xy/manifest.json'); + manifest.links(LINKS); }); -// xyicons/: trimmed gen6 minisprites. +// sprites/xyicons/: trimmed gen6 minisprites. let xyIcons = forEachRule('src/minisprites/pokemon/gen6/*.png', { display: 'trim g6 minisprite %f', cmds: [trimimg(), compresspng({config: 'MINISPRITE'})], }, '%b'); -deploy(ctx => { +deploy(async ctx => { // icons: the gen 6 set has no art for some formes and lends them another's, - // which the smogdex sheet and forumsprites do off the same directory. - let byName = new Map(); + // which the smogdex sheet and forumsprites do off the same directory. Two + // formes lent the same icon are a duplicate name the manifest refuses. + let manifest = new Manifest(ctx, TREE); for (let f of xyIcons) { - for (let name of publishedNames(f, {icons: true})) { - if (byName.has(name)) { - throw new Error(`Two icons published as ${name}`); - } - byName.set(name, f); - } - } - for (let [name, f] of byName) { - smogonSpritecopy(ctx, f, 'xyicons', [name]); + await spritecopy(manifest, f, {dir: 'xyicons'}, {icons: true}); } + manifest.write('__meta/xyicons/manifest.json'); + manifest.links(LINKS); }); -// The smogon side asks for a fixed path, /sprites/xy/charizard.gif, and reads -// no manifest yet, so these copies carry no content stamp and the published -// name is the whole filename. -function smogonSpritecopy(ctx: DeployCtx, f: Sprite, dir: string, names: string[]): void { - if (f.ext === null) { - throw new Error(`Sprite ${f.name} has no extension`); +// Smogdex spritesheet. The sheet tool bakes the names parsed from the %f +// filenames into the css, hence nameSensitive. The png is declared only so +// cwebp has something to read; only the css and the webp are published. + +let [, sheetCss, sheetWebp] = rule(minispriteInputs, { + display: 'smogdex sheet', + nameSensitive: true, + deps: [ + 'data/lib/index.ts', + 'tools/smogdexspritesheet/index.ts', + ], + cmds: [ + 'node tools/smogdexspritesheet/index.ts --image %o1 --stylesheet %o2 -- %f', + 'cwebp -z 9 %o1 -o %o3', + ], +}, ['spritesheet.png', 'spritesheet.css', 'spritesheet.webp']); + +// Hash-stamped css + webp. The css url rides in __meta/ for the dex to read. +deploy(async ctx => { + let wh = await ctx.hash(sheetWebp); + ctx.copy(sheetWebp, `${ASSETS}/spritesheet-${wh}.webp`); + let src = await ctx.read(sheetCss); + let css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`); + if (css === src) { + throw new Error('spritesheet.css: no webp urls rewritten'); } - for (let name of names) { - ctx.copy(f, `${dir}/${name}.${f.ext}`); + // Suffix from source content: the rewritten css is a pure function + // of (css, webp), so this changes exactly when the served bytes + // change. + let ch = await ctx.hash(sheetCss, sheetWebp); + ctx.write(`${ASSETS}/spritesheet-${ch}.css`, css); + ctx.write('__meta/spritesheet-css-url.txt', `${SERVED}/${ASSETS}/spritesheet-${ch}.css\n`); +}); + +// Forumsprites: uniform-size minisprites under stamped names, with the +// unhashed -> url mapping in a manifest. + +let forumItems = forEachRule('src/minisprites/items/*.png', { + display: 'pad item minisprite %f', + cmds: [pad({w: 24, h: 24}), compresspng({config: 'MINISPRITE'})], +}, '%b'); + +let forumG6 = forEachRule('src/minisprites/pokemon/gen6/*.png', { + display: 'pad g6 minisprite %f', + cmds: [pad({w: 40, h: 30}), compresspng({config: 'MINISPRITE'})], +}, '%b'); + +deploy(async ctx => { + let manifest = new Manifest(ctx, TREE); + for (let f of forumItems) { + await itemspritecopy(manifest, f, {dir: 'forumsprites'}); } -} + for (let f of forumG6) { + await spritecopy(manifest, f, {dir: 'forumsprites'}, {allowUnknown: true, icons: true}); + } + manifest.write('__meta/forumsprites/manifest.json'); +}); + +// PMD sprites ship as-is, stamped. + +deploy(async ctx => { + let manifest = new Manifest(ctx, TREE); + for (let f of await ctx.list('src/pmd')) { + await spritecopy(manifest, f, {dir: 'pmd'}); + } + manifest.write('__meta/pmd/manifest.json'); +}); // Deprecated, unstamped sets. Reviving one also means importing what it -// uses (PNG_DETERMINISTIC, base, spriteglob, itemspritecopy) and giving the -// copies a Manifest, as the stamped deploys above do. +// uses (PNG_DETERMINISTIC, base) and giving the copies a Manifest, as the +// deploys above do. // // let xyItems = forEachRule('src/minisprites/items/*.png', { // display: 'trim item minisprite %f',