diff --git a/Buildfile.ts b/Buildfile.ts deleted file mode 100644 index ff48aec3..00000000 --- a/Buildfile.ts +++ /dev/null @@ -1,200 +0,0 @@ - -import {base, compresspng, forEachRule, pad, rule, spriteglob, trimimg} from './tools/build/api.ts'; - -// Generate uniform size minisprites - -forEachRule("src/minisprites/pokemon/gen6/*.png", { - display: "pad g6 minisprite %f", - cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})], -}, "build/gen6-minisprites-padded/%b"); - -forEachRule("src/minisprites/items/*.png", { - display: "pad item minisprite %f", - cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})], -}, "build/item-minisprites-padded/%b"); - -forEachRule("src/minisprites/pokemon/gen6/*.png", { - display: "trim g6 minisprite %f", - cmds: [trimimg(), compresspng({config: "MINISPRITE"})], -}, "build/gen6-minisprites-trimmed/%b"); - -forEachRule("src/minisprites/items/*.png", { - display: "trim item minisprite %f", - cmds: [trimimg(), compresspng({config: "MINISPRITE"})], -}, "build/item-minisprites-trimmed/%b"); - -// Gen 9 - -forEachRule("src/gen9species/*.png", { - display: "96x96 %f", - // TODO, add customizable compression for gif - // ... or investigate using webp instead of both png/gif here - cmds: [ - "magick convert %f -trim +repage -resize 90x90 %o", - "gifsicle -O3 -b %o", - ], -}, "build/gen9-modelslike/%B.gif"); - -// Gen 10 - -forEachRule("src/champions/*.png", { - display: "96x96 %f", - // TODO, add customizable compression for gif - // ... or investigate using webp instead of both png/gif here - cmds: [ - "magick convert %f -trim +repage -resize 90x90 %o", - "gifsicle -O3 -b %o", - ], -}, "build/gen10-modelslike/%B.gif"); - -// Gen 5 CAPs... - -forEachRule("src/sprites/gen5/*.png", [ - // TODO, add customizable compression for gif - // ... or investigate using webp instead of both png/gif here - "magick convert %f %o", - "gifsicle -O3 -b %o", -], "build/gen5-gif/%B.gif"); - -// PS spritesheet - -rule("ps-pokemon.sheet.mjs", { - display: "ps pokemon sheet", - // The sheet tool reads the minisprites (via readdir, which tup couldn't - // track) and the sprite data; declare them so changes rebuild the sheet. - deps: [ - "src/minisprites/pokemon/gen6/*", - "data/species.json", - "data/items.json", - "data/lib/index.ts", - "lib/root/index.ts", - "tools/sheet/index.ts", - ], - cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], -}, "build/ps/pokemonicons-sheet.png"); - -// TODO: reenable when trainers are moved -// rule("ps-trainers.sheet.mjs", { -// display: "ps trainers sheet", -// cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], -// }, "build/ps/trainers-sheet.png"); - -rule("ps-items.sheet.mjs", { - display: "ps items sheet", - deps: [ - "src/minisprites/items/*", - "data/species.json", - "data/items.json", - "data/lib/index.ts", - "lib/root/index.ts", - "tools/sheet/index.ts", - ], - cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], -}, "build/ps/itemicons-sheet.png"); - -// PS pokeball icons - -const balls = [ - "src/_uncategorized/noncanonical/ui/battle/Ball-Normal.png", - "src/_uncategorized/noncanonical/ui/battle/Ball-Sick.png", - "src/_uncategorized/noncanonical/ui/battle/Ball-Null.png", -]; - -rule(balls, { - display: "pokemonicons-pokeball-sheet", - cmds: [ - "magick convert -background transparent -gravity center -extent 40x30 %f +append %o", - compresspng({config: "SPRITESHEET"}), - ], -}, "build/ps/pokemonicons-pokeball-sheet.png"); - -// Smogdex minisprites (webp) - -forEachRule(spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false}), { - display: "webp minisprite %f", - cmds: ["cwebp -z 9 %f -o %o"], -}, "build/smogon/minisprites/%B.webp"); - -// Smogdex spritesheet - -rule(spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false}), { - display: "smogdex sheet", - deps: [ - "data/species.json", - "data/items.json", - "data/lib/index.ts", - "lib/root/index.ts", - "tools/smogdexspritesheet/index.ts", - ], - // build/smogon/spritesheet.png is an undeclared temporary; it is removed - // before the rule finishes. - cmds: [ - "node tools/smogdexspritesheet/index.ts --image build/smogon/spritesheet.png --stylesheet build/smogon/spritesheet.css -- %f", - "cwebp -z 9 build/smogon/spritesheet.png -o build/smogon/spritesheet.webp", - "rm build/smogon/spritesheet.png", - ], -}, ["build/smogon/spritesheet.webp", "build/smogon/spritesheet.css"]); - -// Smogdex social images - -const social = spriteglob(["src/models/*"], {b: false, s: false}); - -const socialSeen = new Set(social.map(base)); -for (const file of spriteglob(["src/gen9species/*"], {b: false, s: false})) { - if (!socialSeen.has(base(file))) { - social.push(file); - socialSeen.add(base(file)); - } -} - -forEachRule(social, { - display: "fbsprite %f", - cmds: [ - 'magick convert "%f[0]" -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o', - compresspng({config: "MODELS"}), - ], -}, "build/smogon/fbsprites/xy/%B.png"); - -forEachRule(social, { - display: "twittersprite %f", - cmds: [ - 'magick convert "%f[0]" -trim -resize 115x115 -background white -gravity center -extent 120x120 %o', - compresspng({config: "MODELS"}), - ], -}, "build/smogon/twittersprites/xy/%B.png"); - -// Trainers - -// TODO: reenable when trainers are moved -// forEachRule("src/canonical/trainers/*", { -// display: "pad trainer %f", -// cmds: [pad({w: 80, h: 80}), compresspng({config: "TRAINERS"})], -// }, "build/padded-trainers/canonical/%b"); - -// Padded Dex - -const dexOutput = forEachRule("src/dex/*", { - display: "pad dex %f", - cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})], -}, "build/padded-dex/%b"); - -// Build missing CAP dex - -const dexSet = new Set(dexOutput.map(base)); - -const dexMissing = []; -for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) { - if (!dexSet.has(base(file))) { - dexMissing.push(file); - dexSet.add(base(file)); - } -} - -forEachRule(dexMissing, { - display: "missing dex %B", - cmds: [ - 'magick convert "%f[0]" -trim %o', - 'magick mogrify -background transparent -gravity center -resize "120x120>" -extent 120x120 %o', - compresspng({config: "DEX"}), - ], -}, "build/padded-dex/%B.png"); diff --git a/README.md b/README.md index 4f06445d..39871ee6 100644 --- a/README.md +++ b/README.md @@ -46,21 +46,33 @@ Using [`brew`](https://brew.sh/) on a macOS: $ brew install imagemagick gifsicle advancecomp optipng pngquant webp ``` -## Building +## Building and deploying -Install dependencies once with `pnpm install`. Then, to build: +Install dependencies once with `pnpm install`. + +Each deploy is a root `*.deploy.ts` module: it declares its build rules +(shared sets live in `rules/`) and a `finish` function that maps the built +artifacts to their published names. Build outputs are content-addressed: +rules declare nominal output filenames but the store names every object by +the hash of its bytes (under `.build/cas/`), so incrementality keys on +content, same-byte renames rebuild nothing, and hash-stamped publishing +reuses the build's digests. All state lives in `.build/`. ``` -$ pnpm build +$ pnpm build # build every deploy's rules, GC stale state +$ pnpm deploy # assets.deploy.ts -> tar -> DEPLOY_COMMAND (.env) +$ node tools/deploy/index.ts build ps.deploy.ts # build one deploy's rules +$ node tools/deploy/index.ts run smogon.deploy.ts -o deploy/smogon +$ node tools/deploy/index.ts inspect src/minisprites/items/i1.png -o /tmp/out ``` -The rules live in `Buildfile.ts`. Build state (content hashes, rule records) -is kept in `.build/`; outputs of removed rules are deleted automatically, and -renamed sources are detected and their outputs copied instead of rebuilt. +`run` materializes a deploy to a directory (`--link` hardlinks, `--tar` +writes a tar file) without uploading anything. `inspect` builds every rule +that consumes the given source paths (and their transitive consumers) and +copies the outputs out under readable names for eyeballing. Useful flags: `-j ` parallelism, `-n` dry run, `-v` verbose, -`--adopt` record already-existing `build/` outputs as up to date instead of -rebuilding them (useful when `build/` was produced elsewhere). +`--fail-fast` stop after the first failure. ## Configuration @@ -88,7 +100,13 @@ DEFAULT_ADVPNG=-z4 -i5000 - The build tool only tracks the inputs a rule declares. If a build tool reads files that aren't on its command line (e.g. it does a `readdir()`), declare - them with the rule's `deps:` in `Buildfile.ts` so changes are detected. + them with the rule's `deps:` so changes are detected. +- Rule identity is content-only by default: renaming a source without + changing its bytes rebuilds nothing. If a tool bakes input *names* into + its output bytes (the spritesheet builders do), the rule must set + `nameSensitive: true` or renames will leave its output silently stale. +- Rules must be declared when a deploy module is imported (top level), not + inside `finish` — the build runs before finish does. ## License diff --git a/assets.deploy.js b/assets.deploy.js deleted file mode 100644 index 6abe5d40..00000000 --- a/assets.deploy.js +++ /dev/null @@ -1,149 +0,0 @@ - -// The upload contract wants __key first in the tar, naming the asset set; -// the tar packer emits ops in queue order, so it has to be the first op. -write("__key", "sprites"); - -function toSmogonAlias(name) { - return name.toLowerCase(). - replace(/[ _]+/, "-"). - replace(/[^a-z0-9-]+/g, ''); -} - -function toPSID(name) { - return name.toLowerCase().replace(/[^a-z0-9]+/g, ''); -} - -// Copy with a content-hash-stamped name and record the unhashed -> hashed -// mapping in `manifest`. -function stampcopy(f, {dir, ext, name}, manifest) { - const key = `${name}.${ext ?? f.ext}`; - // ActionQueue only dedups final dsts; hashed dsts differ even when - // unhashed names collide, so check the manifest key explicitly. - if (manifest[key] !== undefined) { - throw new Error(`duplicate sprite name ${key}`); - } - const h = hash(f); - manifest[key] = `${name}-${h}.${ext ?? f.ext}`; - copy(f, {dir, ext, name: `${name}-${h}`}); -} - -function writeManifest(dst, manifest) { - const sorted = {}; - for (const k of Object.keys(manifest).sort()) { - sorted[k] = manifest[k]; - } - write(dst, JSON.stringify(sorted, null, 4) + "\n"); -} - -function spritecopy(f, {dir, ext}, allowUnknown=false, manifest=null) { - const sn = spritedata.parseFilename(f.name); - let name; - - // Skip asymmetrical for now - if (sn.extra.has("a") || sn.extra.has("b") || sn.extra.has("s")) { - return; - } - - if (sn.extension) { - if (allowUnknown && sn.extension && sn.name === "Unknown") { - name = "unknown" - } else { - // Skip this, we don't use Unknown/Substitute - return; - } - } else { - const sd = spritedata.get(sn.id); - name = toSmogonAlias(sd.base); - if (sd.forme) { - name += `-${toSmogonAlias(sd.forme)}`; - } - } - if (sn.extra.has("f")) { - name += "-f"; - } - if (sn.extra.has("g")) { - name += "-gmax"; - } - - if (manifest) { - stampcopy(f, {dir, ext, name}, manifest); - } else { - copy(f, {dir, ext, name}); - } -} - -// TODO: merge with above -function itemspritecopy(f, {dir, ext}, manifest=null) { - const sn = spritedata.parseFilename(f.name); - const sd = spritedata.get(sn.id); - for (const n of sd.names) { - const name = toSmogonAlias(n); - if (manifest) { - stampcopy(f, {dir, ext, name}, manifest); - } else { - copy(f, {dir, ext, name}); - } - } -} - -function newspritecopy(f, {dir, ext}) { - const sn = spritedata.parseFilename(f.name); - if (sn.extension) { - return - } - const sd = spritedata.get(sn.id); - for (const n of sd.type === 'item' ? sd.names : [sd.base + sd.forme]) { - let name = toPSID(n); - if (sn.extra.has("f")) { - name += "f"; - } - if (sn.extra.has("g")) { - name += "gmax"; - } - copy(f, {dir, ext, name}); - } -} - -// Dex spritesheet assets: hash-stamped css + webp. The css suffix pointer -// rides in __meta/ for the dex to read. -{ - const wh = hash("build/smogon/spritesheet.webp"); - copy("build/smogon/spritesheet.webp", `spritesheet-${wh}.webp`); - const src = read("build/smogon/spritesheet.css"); - const css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`); - if (css === src) { - throw new Error("spritesheet.css: no webp urls rewritten"); - } - // Suffix from source content: the rewritten css is a pure function of - // (css, webp), so this changes exactly when the served bytes change. - const ch = hash("build/smogon/spritesheet.css", "build/smogon/spritesheet.webp"); - write(`spritesheet-${ch}.css`, css); - write("__meta/spritesheet_css_suffix.txt", `-${ch}\n`); -} - -{ - const h = hash(...list("build/smogon/minisprites")); - for (const f of list("build/smogon/minisprites")) { - newspritecopy(f, {dir: "minisprites/" + h}); - } - write("__meta/minisprites-hash.txt", h); -} - -{ - const manifest = {}; - for (const f of list("build/item-minisprites-padded")) { - itemspritecopy(f, {dir: "forumsprites"}, manifest); - } - for (const f of list("build/gen6-minisprites-padded")) { - spritecopy(f, {dir: "forumsprites"}, true, manifest); - } - writeManifest("__meta/forumsprites/manifest.json", manifest); -} - -{ - const manifest = {}; - for (const f of list("src/pmd")) { - spritecopy(f, {dir: "pmd"}, false, manifest); - } - writeManifest("__meta/pmd/manifest.json", manifest); -} diff --git a/package.json b/package.json index 396f215b..4e8ee918 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "type": "module", "scripts": { - "build": "node tools/build/index.ts", - "deploy": "node tools/deploy/index.ts deploy assets.deploy.js", + "build": "node tools/deploy/index.ts build", + "deploy": "node tools/deploy/index.ts deploy assets.deploy.ts", "check": "tsc --build tsconfig-workspace.json" }, "dependencies": { diff --git a/ps.deploy.js b/ps.deploy.js deleted file mode 100644 index 1ff45db4..00000000 --- a/ps.deploy.js +++ /dev/null @@ -1,120 +0,0 @@ - -// function toID(name) { -// return name.toLowerCase().replace(/[^a-z0-9]+/g, ''); -// } - -// function spritecopy(f, {dir, ext}) { -// const sn = spritedata.parseFilename(f.name); -// let name; -// if (sn.extension) { -// name = toID(sn.name); -// } else { -// const sd = spritedata.get(sn.id); -// debug(sd); -// name = toID(sd.base); -// if (sd.forme) { -// name += `-${toID(sd.forme)}`; -// } -// } -// if (sn.extra.has("f")) { -// name += "-f"; -// } -// if (sn.extra.has("g")) { -// name += "-gmax"; -// } -// if (sn.extra.has("b")) { -// dir += "-back"; -// } -// if (sn.extra.has("s")) { -// dir += "-shiny"; -// } -// copy(f, {dir, ext, name}); -// } - -function toPSID(name) { - return name.toLowerCase().replace(/[^a-z0-9]+/g, ''); -} - -function spritecopy(f, {dir, ext}, allowUnknown=false) { - const sn = spritedata.parseFilename(f.name); - let name; - - // Skip asymmetrical for now - if (sn.extra.has("a") || sn.extra.has("b") || sn.extra.has("s")) { - return; - } - - if (sn.extension) { - if (allowUnknown && sn.extension && sn.name === "Unknown") { - name = "unknown" - } else { - // Skip this, we don't use Unknown/Substitute - return; - } - } else { - const sd = spritedata.get(sn.id); - name = toPSID(sd.base); - if (sd.forme) { - name += `-${toPSID(sd.forme)}`; - } - } - if (sn.extra.has("f")) { - name += "-f"; - } - if (sn.extra.has("g")) { - name += "-gmax"; - } - - copy(f, {dir, ext, name}); -} - -let seenModels = new Set; - -for (const f of list("src/models")) { - seenModels.add(f.name); - spritecopy(f, {dir: "ani"}); -} - -for (const f of list("build/gen10-modelslike")) { - if (seenModels.has(f.name)) continue; - seenModels.add(f.name); - spritecopy(f, {dir: "ani"}); -} - -// for (const f of list("src/sprites/gen5")) { -// spritecopy(f, {dir: "gen5ani"}); -// } - -// for (const f of list("src/afd")) { -// spritecopy(f, {dir: "afd"}); -// } - -// for (const f of list("build/padded-dex")) { -// spritecopy(f, {dir: "dex"}); -// } - -// function fixType(name) { -// return name.replace("Unknown", "???"); -// } - -// for (const f of list("src/canonical/ui/types/gen4").concat(list("src/noncanonical/ui/types/gen4"))) { -// copy(f, {dir: "types", name: fixType(f.name)}); -// } - -// for (const f of list("src/canonical/ui/categories/gen4")) { -// copy(f, {dir: "categories"}); -// } -// copy("src/noncanonical/ui/categories/undefined.png", {dir: "categories"}); - -// copy("src/canonical/ui/battle/Alpha.png", {dir: "misc"}); -// copy("src/canonical/ui/battle/Mega.png", {dir: "misc"}); -// copy("src/canonical/ui/battle/Omega.png", {dir: "misc"}); - -// // TODO: reenable when trainers are moved -// // dest("trainers"); -// // sel("build/padded-trainers/canonical"); - -// copy("build/ps/pokemonicons-pokeball-sheet.png", {dir: "."}); -// copy("build/ps/pokemonicons-sheet.png", {dir: "."}); -// //copy("build/ps/trainers-sheet.png", {dir: "."}); -// copy("build/ps/itemicons-sheet.png", {dir: "."}); diff --git a/smogon.deploy.js b/smogon.deploy.js deleted file mode 100644 index a7ddb453..00000000 --- a/smogon.deploy.js +++ /dev/null @@ -1,134 +0,0 @@ - -function toSmogonAlias(name) { - return name.toLowerCase(). - replace(/[ _]+/, "-"). - replace(/[^a-z0-9-]+/g, ''); -} - -// Copy with a content-hash-stamped name and record the unhashed -> hashed -// mapping in `manifest`. -function stampcopy(f, {dir, ext, name}, manifest) { - const key = `${name}.${ext ?? f.ext}`; - // ActionQueue only dedups final dsts; hashed dsts differ even when - // unhashed names collide, so check the manifest key explicitly. - if (manifest[key] !== undefined) { - throw new Error(`duplicate sprite name ${key}`); - } - const h = hash(f); - manifest[key] = `${name}-${h}.${ext ?? f.ext}`; - copy(f, {dir, ext, name: `${name}-${h}`}); -} - -function writeManifest(dst, manifest) { - const sorted = {}; - for (const k of Object.keys(manifest).sort()) { - sorted[k] = manifest[k]; - } - write(dst, JSON.stringify(sorted, null, 4) + "\n"); -} - -function spritecopy(f, {dir, ext}, allowUnknown=false, manifest=null) { - const sn = spritedata.parseFilename(f.name); - let name; - - // Skip asymmetrical for now - if (sn.extra.has("a") || sn.extra.has("b") || sn.extra.has("s")) { - return; - } - - if (sn.extension) { - if (allowUnknown && sn.extension && sn.name === "Unknown") { - name = "unknown" - } else { - // Skip this, we don't use Unknown/Substitute - return; - } - } else { - const sd = spritedata.get(sn.id); - name = toSmogonAlias(sd.base); - if (sd.forme) { - name += `-${toSmogonAlias(sd.forme)}`; - } - } - if (sn.extra.has("f")) { - name += "-f"; - } - if (sn.extra.has("g")) { - name += "-gmax"; - } - - if (manifest) { - stampcopy(f, {dir, ext, name}, manifest); - } else { - copy(f, {dir, ext, name}); - } -} - -// TODO: merge with above -function itemspritecopy(f, {dir, ext}, manifest=null) { - const sn = spritedata.parseFilename(f.name); - const sd = spritedata.get(sn.id); - for (const n of sd.names) { - const name = toSmogonAlias(n); - if (manifest) { - stampcopy(f, {dir, ext, name}, manifest); - } else { - copy(f, {dir, ext, name}); - } - } -} - -let seenModels = new Set; -const xyManifest = {}; - -for (const f of list("src/models")) { - seenModels.add(f.name); - spritecopy(f, {dir: "xy"}, false, xyManifest); -} - -for (const f of list("build/gen9-modelslike")) { - if (seenModels.has(f.name)) continue; - seenModels.add(f.name); - spritecopy(f, {dir: "xy"}, false, xyManifest); -} - -for (const f of list("build/gen10-modelslike")) { - if (seenModels.has(f.name)) continue; - seenModels.add(f.name); - spritecopy(f, {dir: "xy"}, false, xyManifest); -} - -// Non-model CAPs -for (const f of list("src/sprites/gen5")) { - if (f.ext !== 'gif' || seenModels.has(f.name)) continue; - seenModels.add(f.name); - spritecopy(f, {dir: "xy"}, false, xyManifest); -} - -for (const f of list("build/gen5-gif")) { - if (seenModels.has(f.name)) continue; - seenModels.add(f.name); - spritecopy(f, {dir: "xy"}, false, xyManifest); -} - -writeManifest("xy/manifest.json", xyManifest); - -{ - const manifest = {}; - for (const f of list("build/gen6-minisprites-trimmed")) { - spritecopy(f, {dir: "xyicons"}, false, manifest); - } - writeManifest("xyicons/manifest.json", manifest); -} - -for (const f of list("build/item-minisprites-trimmed")) { - itemspritecopy(f, {dir: "xyitems"}); -} - -for (const f of list("build/smogon/fbsprites/xy")) { - spritecopy(f, {dir: "fbsprites/xy"}); -} - -for (const f of list("build/smogon/twittersprites/xy")) { - spritecopy(f, {dir: "twittersprites/xy"}); -} diff --git a/tools/build/api.ts b/tools/build/api.ts deleted file mode 100644 index cb305d79..00000000 --- a/tools/build/api.ts +++ /dev/null @@ -1,110 +0,0 @@ - -import pathlib from 'path'; -import {createHash} from 'crypto'; - -import {astable, glob} from './helpers.ts'; -import {type Cmd, flattenCmds, substitute} from './subst.ts'; - -export type {Cmd}; -export {base, compresspng, getconfig, glob, pad, setConfig, - spritedata, spriteglob, trimimg, type SpriteData} from './helpers.ts'; - -export interface CmdSpec { - display? : string; - // Tracked-but-not-substituted inputs: hashed and part of rule identity, - // but never expanded into %f. Use for files a tool reads on its own - // (e.g. tools/sheet readdirs the minisprite directories). - deps? : string | string[]; - cmds : Cmd[]; -} - -export interface RuleDecl { - inputs : string[]; // %f source, ordered - deps : string[]; // hashed, never substituted - outputs : string[]; // post-substitution paths - command : string; // final ' && '-joined shell command - display : string | null; // post-substitution; cosmetic, not part of identity - template : string; // pre-substitution cmds + output basename template(s) - key : string; // identity for incremental skip -} - -let rules : RuleDecl[] = []; - -export function getRules() : RuleDecl[] { - return rules; -} - -export function resetRules() : void { - rules = []; -} - -function normalizeSpec(spec : CmdSpec | Cmd[]) : CmdSpec { - return Array.isArray(spec) ? {cmds: spec} : spec; -} - -// The rename-detection template deliberately excludes output directories -// (basename only) so that content-preserving moves across directories with -// identical processing still match. Input extensions are included because -// tools like magick pick their output format from file extensions, so an -// extension-only rename must not match. -function templateOf(cmds : string[], outputTemplates : string[], inputs : string[]) : string { - return [ - cmds.join('\n'), - outputTemplates.map(t => pathlib.basename(t)).join('\0'), - inputs.map(p => pathlib.extname(p)).join('\0'), - ].join('\x01'); -} - -function keyOf(command : string, inputs : string[], deps : string[], outputs : string[]) : string { - const h = createHash('sha256'); - h.update([command, inputs.join('\0'), deps.join('\0'), outputs.join('\0')].join('\x01')); - return h.digest('hex'); -} - -function makeRule(inputs : string[], deps : string[], spec : CmdSpec, - outputs : string[], outputTemplates : string[]) : RuleDecl { - const cmds = flattenCmds(spec.cmds).map(c => substitute(c, inputs, outputs)); - if (cmds.length === 0) { - throw new Error(`Rule with no commands (outputs: ${outputs.join(' ')})`); - } - const command = cmds.join(' && '); - const decl : RuleDecl = { - inputs, - deps, - outputs, - command, - display: spec.display !== undefined ? substitute(spec.display, inputs, outputs) : null, - template: templateOf(flattenCmds(spec.cmds), outputTemplates, inputs), - key: keyOf(command, inputs, deps, outputs), - }; - rules.push(decl); - return decl; -} - -export function rule(input : string | string[], spec : CmdSpec | Cmd[], - output : string | string[]) : string[] { - const s = normalizeSpec(spec); - const outputs = astable(output); - for (const out of outputs) { - if (out.includes('%')) { - throw new Error(`rule() outputs are literal paths, no substitutions: ${out}`); - } - } - const decl = makeRule(glob(input), glob(astable(s.deps)), s, outputs, outputs); - return decl.outputs; -} - -export function forEachRule(input : string | string[], spec : CmdSpec | Cmd[], - output : string) : string[] { - const s = normalizeSpec(spec); - if (/%[fo]/.test(output)) { - throw new Error(`forEachRule output template may only use %b/%B: ${output}`); - } - const deps = glob(astable(s.deps)); - const outputs = []; - for (const file of glob(input)) { - const decl = makeRule([file], deps, s, [substitute(output, [file], [])], [output]); - outputs.push(...decl.outputs); - } - return outputs; -} diff --git a/tools/build/db.ts b/tools/build/db.ts deleted file mode 100644 index 1eec899e..00000000 --- a/tools/build/db.ts +++ /dev/null @@ -1,256 +0,0 @@ - -import fs from 'fs'; -import pathlib from 'path'; -import Database from 'better-sqlite3'; - -import type {RuleDecl} from './api.ts'; -import type {FileStat} from './hash.ts'; -import {BuildError} from './graph.ts'; - -export interface StoredRuleInput { - path : string; - isDep : boolean; - hash : Buffer; -} - -export interface StoredRuleOutput { - path : string; - size : bigint | null; // null when the rule last failed - mtimeNs : bigint | null; -} - -export interface StoredRule { - id : bigint; - key : string; - command : string; - display : string | null; - template : string; - inputSig : Buffer; - ok : boolean; - inputs : StoredRuleInput[]; // ordered: inputs (in %f order), then deps - outputs : StoredRuleOutput[]; // ordered: position-mapped for renames -} - -export interface RecordedOutput { - path : string; - size : bigint; - mtimeNs : bigint; -} - -const DDL = ` -CREATE TABLE IF NOT EXISTS file_cache ( - path TEXT PRIMARY KEY, - size INTEGER NOT NULL, - mtime_ns INTEGER NOT NULL, - hash BLOB NOT NULL -) WITHOUT ROWID; - -CREATE TABLE IF NOT EXISTS rules ( - id INTEGER PRIMARY KEY, - key TEXT NOT NULL UNIQUE, - command TEXT NOT NULL, - display TEXT, - template TEXT NOT NULL, - input_sig BLOB NOT NULL, - ok INTEGER NOT NULL DEFAULT 0 -); -CREATE INDEX IF NOT EXISTS rules_rename ON rules(template, input_sig); - -CREATE TABLE IF NOT EXISTS rule_inputs ( - rule_id INTEGER NOT NULL REFERENCES rules(id) ON DELETE CASCADE, - ord INTEGER NOT NULL, - path TEXT NOT NULL, - is_dep INTEGER NOT NULL DEFAULT 0, - hash BLOB NOT NULL, - PRIMARY KEY (rule_id, ord) -); -CREATE INDEX IF NOT EXISTS rule_inputs_path ON rule_inputs(path); - -CREATE TABLE IF NOT EXISTS rule_outputs ( - rule_id INTEGER NOT NULL REFERENCES rules(id) ON DELETE CASCADE, - ord INTEGER NOT NULL, - path TEXT NOT NULL, - size INTEGER, - mtime_ns INTEGER, - PRIMARY KEY (rule_id, ord) -); -CREATE INDEX IF NOT EXISTS rule_outputs_path ON rule_outputs(path); -`; - -export class BuildDb { - private db : Database.Database; - - constructor(dbPath : string) { - fs.mkdirSync(pathlib.dirname(dbPath), {recursive: true}); - this.db = new Database(dbPath); - this.db.defaultSafeIntegers(true); - this.db.pragma('journal_mode = WAL'); - this.db.pragma('foreign_keys = ON'); - this.db.pragma('synchronous = NORMAL'); - this.migrate(); - } - - private migrate() : void { - const version = Number(this.db.pragma('user_version', {simple: true})); - if (version === 0) { - this.db.exec('BEGIN;' + DDL + 'PRAGMA user_version = 1; COMMIT;'); - } else if (version !== 1) { - throw new BuildError( - `Unknown build db schema version ${version}; delete .build/ and re-run with --adopt`); - } - } - - loadFileCache() : Map { - const result = new Map(); - const rows = this.db.prepare<[], {path : string, size : bigint, mtime_ns : bigint, hash : Buffer}>( - 'SELECT path, size, mtime_ns, hash FROM file_cache').all(); - for (const row of rows) { - result.set(row.path, {size: row.size, mtimeNs: row.mtime_ns, hash: row.hash}); - } - return result; - } - - saveFileCache(entries : Map) : void { - const upsert = this.db.prepare(` - INSERT INTO file_cache (path, size, mtime_ns, hash) VALUES (?, ?, ?, ?) - ON CONFLICT(path) DO UPDATE SET - size = excluded.size, mtime_ns = excluded.mtime_ns, hash = excluded.hash`); - this.db.transaction(() => { - for (const [path, stat] of entries) { - upsert.run(path, stat.size, stat.mtimeNs, stat.hash); - } - })(); - } - - pruneFileCache(live : Set) : void { - const paths = this.db.prepare<[], {path : string}>('SELECT path FROM file_cache').all(); - const del = this.db.prepare('DELETE FROM file_cache WHERE path = ?'); - this.db.transaction(() => { - for (const {path} of paths) { - if (!live.has(path)) { - del.run(path); - } - } - })(); - } - - loadStoredRules() : StoredRule[] { - // One transaction: the three queries must see a single snapshot, or a - // concurrent writer (e.g. a build racing a dry run) tears the view. - return this.db.transaction(() => this.loadStoredRulesInner())(); - } - - private loadStoredRulesInner() : StoredRule[] { - const byId = new Map(); - const ruleRows = this.db.prepare<[], { - id : bigint, key : string, command : string, display : string | null, - template : string, input_sig : Buffer, ok : bigint, - }>('SELECT id, key, command, display, template, input_sig, ok FROM rules').all(); - for (const row of ruleRows) { - byId.set(row.id, { - id: row.id, - key: row.key, - command: row.command, - display: row.display, - template: row.template, - inputSig: row.input_sig, - ok: row.ok !== 0n, - inputs: [], - outputs: [], - }); - } - const inputRows = this.db.prepare<[], {rule_id : bigint, path : string, is_dep : bigint, hash : Buffer}>( - 'SELECT rule_id, path, is_dep, hash FROM rule_inputs ORDER BY rule_id, ord').all(); - for (const row of inputRows) { - byId.get(row.rule_id)!.inputs.push({path: row.path, isDep: row.is_dep !== 0n, hash: row.hash}); - } - const outputRows = this.db.prepare<[], {rule_id : bigint, path : string, size : bigint | null, mtime_ns : bigint | null}>( - 'SELECT rule_id, path, size, mtime_ns FROM rule_outputs ORDER BY rule_id, ord').all(); - for (const row of outputRows) { - byId.get(row.rule_id)!.outputs.push({path: row.path, size: row.size, mtimeNs: row.mtime_ns}); - } - return [...byId.values()]; - } - - // One transaction per completed rule: an interrupted build only ever - // contains fully-recorded rules. outputs === null records a failure (ok=0, - // output paths kept for GC, stats nulled so the rule stays dirty). - recordRuleResult(decl : RuleDecl, inputHashes : Map, sig : Buffer, - outputs : RecordedOutput[] | null) : void { - const ok = outputs !== null; - const upsert = this.db.prepare<[string, string, string | null, string, Buffer, bigint], {id : bigint}>(` - INSERT INTO rules (key, command, display, template, input_sig, ok) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(key) DO UPDATE SET - display = excluded.display, input_sig = excluded.input_sig, ok = excluded.ok - RETURNING id`); - const delInputs = this.db.prepare('DELETE FROM rule_inputs WHERE rule_id = ?'); - const delOutputs = this.db.prepare('DELETE FROM rule_outputs WHERE rule_id = ?'); - const insInput = this.db.prepare( - 'INSERT INTO rule_inputs (rule_id, ord, path, is_dep, hash) VALUES (?, ?, ?, ?, ?)'); - const insOutput = this.db.prepare( - 'INSERT INTO rule_outputs (rule_id, ord, path, size, mtime_ns) VALUES (?, ?, ?, ?, ?)'); - this.db.transaction(() => { - const {id} = upsert.get(decl.key, decl.command, decl.display, decl.template, - sig, ok ? 1n : 0n)!; - delInputs.run(id); - delOutputs.run(id); - let ord = 0; - for (const path of decl.inputs) { - insInput.run(id, ord++, path, 0, inputHashes.get(path)!); - } - for (const path of decl.deps) { - insInput.run(id, ord++, path, 1, inputHashes.get(path)!); - } - const outputRows = outputs ?? decl.outputs.map(path => ({path, size: null, mtimeNs: null})); - outputRows.forEach((o, i) => insOutput.run(id, i, o.path, o.size, o.mtimeNs)); - })(); - } - - deleteRule(id : bigint) : void { - this.db.prepare('DELETE FROM rules WHERE id = ?').run(id); - } - - // template/display are pure functions of the rule declaration but are not - // part of the rule key; refresh stored values that have drifted (e.g. the - // template format changed in a newer version of this tool), otherwise - // rename detection quietly stops matching older records. - refreshRuleMeta(entries : {id : bigint, template : string, display : string | null}[]) : void { - if (entries.length === 0) { - return; - } - const update = this.db.prepare('UPDATE rules SET template = ?, display = ? WHERE id = ?'); - this.db.transaction(() => { - for (const e of entries) { - update.run(e.template, e.display, e.id); - } - })(); - } - - close() : void { - this.db.pragma('wal_checkpoint(TRUNCATE)'); - this.db.close(); - } -} - -// Guards against two concurrent builds; the exclusive transaction is released -// by the OS on any crash, so there are no stale lock files. -export function acquireLock(lockPath : string) : () => void { - fs.mkdirSync(pathlib.dirname(lockPath), {recursive: true}); - const lock = new Database(lockPath, {timeout: 0}); - try { - lock.exec('BEGIN EXCLUSIVE'); - } catch (err) { - lock.close(); - if ((err as {code? : string}).code === 'SQLITE_BUSY') { - throw new BuildError('Another build is already running.'); - } - throw err; - } - return () => { - try { - lock.exec('COMMIT'); - } catch {} - lock.close(); - }; -} diff --git a/tools/build/exec.ts b/tools/build/exec.ts index f2cf4266..e8a35f80 100644 --- a/tools/build/exec.ts +++ b/tools/build/exec.ts @@ -79,25 +79,3 @@ export function runShell(command : string, opts : {cwd : string, signal : AbortS }); } -// A worker that throws stops its own loop, but the pool always waits for -// every other worker to finish before rethrowing: failing fast here would -// return control (and e.g. close the database) while rules are still running. -export async function workerPool(items : readonly T[], jobs : number, - fn : (item : T, index : number) => Promise) : Promise { - let next = 0; - const workers = []; - for (let i = 0; i < Math.max(1, Math.min(jobs, items.length)); i++) { - workers.push((async () => { - while (next < items.length) { - const index = next++; - await fn(items[index]!, index); - } - })()); - } - const results = await Promise.allSettled(workers); - for (const result of results) { - if (result.status === 'rejected') { - throw result.reason; - } - } -} diff --git a/tools/build/graph.ts b/tools/build/graph.ts deleted file mode 100644 index ba25dad7..00000000 --- a/tools/build/graph.ts +++ /dev/null @@ -1,63 +0,0 @@ - -import type {RuleDecl} from './api.ts'; -import {BuildError} from './errors.ts'; - -export {BuildError}; - -export interface GraphResult { - order : RuleDecl[]; // topological, stable w.r.t. declaration order - generated : Set; // every path produced by some rule -} - -export function checkGraph(rules : RuleDecl[]) : GraphResult { - const owner = new Map(); - for (const rule of rules) { - for (const output of rule.outputs) { - const other = owner.get(output); - if (other !== undefined) { - throw new BuildError( - `Output ${output} produced by multiple rules:\n ${other.command}\n ${rule.command}`); - } - owner.set(output, rule); - } - } - - const consumers = new Map(); - const indegree = new Map(); - for (const rule of rules) { - indegree.set(rule, 0); - } - for (const rule of rules) { - for (const input of [...rule.inputs, ...rule.deps]) { - const producer = owner.get(input); - if (producer !== undefined && producer !== rule) { - let list = consumers.get(producer); - if (list === undefined) { - consumers.set(producer, list = []); - } - list.push(rule); - indegree.set(rule, indegree.get(rule)! + 1); - } - } - } - - const queue = rules.filter(r => indegree.get(r) === 0); - const order = []; - for (let i = 0; i < queue.length; i++) { - const rule = queue[i]!; - order.push(rule); - for (const consumer of consumers.get(rule) ?? []) { - const deg = indegree.get(consumer)! - 1; - indegree.set(consumer, deg); - if (deg === 0) { - queue.push(consumer); - } - } - } - if (order.length !== rules.length) { - const stuck = rules.filter(r => indegree.get(r)! > 0); - throw new BuildError(`Dependency cycle involving:\n ${stuck.map(r => r.command).join('\n ')}`); - } - - return {order, generated: new Set(owner.keys())}; -} diff --git a/tools/build/index.ts b/tools/build/index.ts deleted file mode 100644 index 3020a2e6..00000000 --- a/tools/build/index.ts +++ /dev/null @@ -1,355 +0,0 @@ - -import fs from 'fs'; -import os from 'os'; -import pathlib from 'path'; -import {fileURLToPath, pathToFileURL} from 'url'; -import {program} from 'commander'; -import debugfn from 'debug'; - -import {getRules, type RuleDecl, setConfig} from './api.ts'; -import {loadConfig} from './config.ts'; -import {acquireLock, BuildDb, type RecordedOutput} from './db.ts'; -import {killAllProcessGroups, runShell, workerPool} from './exec.ts'; -import {BuildError, checkGraph} from './graph.ts'; -import {reconcileHashes} from './hash.ts'; -import {computePlan, type OutputStat, ruleInputSig} from './plan.ts'; - -const debug = debugfn('build'); - -program - .option('-j, --jobs ', 'number of parallel jobs', String(os.availableParallelism())) - .option('-n, --dry-run', 'print the plan without changing anything') - .option('--adopt', 'record existing outputs as up to date instead of running (migration)') - .option('--fail-fast', 'stop scheduling new rules after the first failure') - .option('--config ', 'config file', 'build.config') - .option('-v, --verbose', 'print more detail'); -program.parse(process.argv); -const opts = program.opts(); - -const root = pathlib.resolve(fileURLToPath(import.meta.url), '../../..'); -process.chdir(root); - -function statPath(path : string) : OutputStat | null { - try { - const st = fs.statSync(path, {bigint: true}); - return st.isFile() ? {size: st.size, mtimeNs: st.mtimeNs} : null; - } catch { - return null; - } -} - -function label(decl : RuleDecl) : string { - return decl.display ?? decl.command.split(' && ')[0]!; -} - -function indent(text : string) : string { - return text.replace(/\n$/, '').split('\n').map(l => ' ' + l).join('\n'); -} - -async function main() : Promise { - const jobs = Number(opts.jobs); - if (!Number.isInteger(jobs) || jobs < 1) { - throw new BuildError(`Invalid --jobs value: ${opts.jobs}`); - } - const dryRun = Boolean(opts.dryRun); - const releaseLock = dryRun ? null : acquireLock('.build/lock.sqlite'); - // A dry run must not create state; without an existing db it reads from - // an empty in-memory one. - const dbPath = dryRun && !fs.existsSync('.build/db.sqlite') ? ':memory:' : '.build/db.sqlite'; - const db = new BuildDb(dbPath); - try { - // Phase 1: evaluate the rule set - setConfig(loadConfig(opts.config)); - await import(pathToFileURL(pathlib.join(root, 'Buildfile.ts')).href); - const rules = getRules(); - const {order, generated} = checkGraph(rules); - for (const rule of rules) { - for (const path of [...rule.inputs, ...rule.deps]) { - if (generated.has(path)) { - // The planner assumes all inputs are hashable before - // execution; support this when a rule needs it. - throw new BuildError(`Rules consuming generated files are not yet supported: ${path}`); - } - } - } - debug('%d rules', rules.length); - - // Phase 2: hash source files (stat-cached) - const sources = new Set(); - for (const rule of rules) { - for (const path of [...rule.inputs, ...rule.deps]) { - sources.add(path); - } - } - const {hashes, updated, missing} = reconcileHashes(sources, db.loadFileCache()); - if (missing.length > 0) { - throw new BuildError(`Missing input files:\n ${missing.slice(0, 20).join('\n ')}` - + (missing.length > 20 ? `\n ... and ${missing.length - 20} more` : '')); - } - if (!dryRun && updated.size > 0) { - db.saveFileCache(updated); - } - debug('hashed %d files (%d cached)', sources.size, sources.size - updated.size); - - // Phase 3: plan - const statMemo = new Map(); - const statOutput = (path : string) => { - let st = statMemo.get(path); - if (st === undefined) { - statMemo.set(path, st = statPath(path)); - } - return st; - }; - const stored = db.loadStoredRules(); - const plan = computePlan({ - current: rules, - stored, - hashes, - statOutput, - adopt: Boolean(opts.adopt), - }); - - // Keep stored template/display in sync for key-matched rules; they are - // outside the key, so e.g. a template format change in this tool would - // otherwise silently disable rename detection for old records. - if (!dryRun) { - const storedByKey = new Map(stored.map(s => [s.key, s])); - const staleMeta = []; - for (const decl of rules) { - const s = storedByKey.get(decl.key); - if (s !== undefined && (s.template !== decl.template || s.display !== decl.display)) { - staleMeta.push({id: s.id, template: decl.template, display: decl.display}); - } - } - db.refreshRuleMeta(staleMeta); - } - - const staleOutputs = []; - const currentOutputs = new Set(rules.flatMap(r => r.outputs)); - for (const s of plan.stale) { - for (const o of s.outputs) { - if (!currentOutputs.has(o.path)) { - staleOutputs.push(o.path); - } - } - } - - if (plan.run.length + plan.renames.length + plan.adopt.length + plan.stale.length === 0) { - console.log(`${plan.clean.length} rules up to date.`); - return 0; - } - if (opts.verbose || dryRun) { - for (const {decl, reason} of plan.run) { - console.log(`run (${reason}): ${label(decl)}`); - } - for (const {decl, from} of plan.renames) { - console.log(`rename: ${from.outputs.map(o => o.path).join(' ')} -> ${decl.outputs.join(' ')}`); - } - for (const decl of plan.adopt) { - console.log(`adopt: ${label(decl)}`); - } - for (const path of staleOutputs) { - console.log(`delete: ${path}`); - } - } - if (dryRun) { - console.log(`would run ${plan.run.length}, rename ${plan.renames.length}, ` - + `adopt ${plan.adopt.length}, delete ${staleOutputs.length} outputs ` - + `(${plan.clean.length} up to date)`); - return 0; - } - - // Phase 4: renames (before stale deletion: sources must still exist) - let renamed = 0; - for (const {decl, from} of plan.renames) { - // An earlier copy in this loop may have overwritten this rename's - // source (rename destinations can collide with rename sources); - // re-verify every source against its recorded stat before copying - // so only verified bytes ever propagate. - const intact = from.outputs.every(o => { - const st = statPath(o.path); - return o.size !== null && st !== null - && st.size === o.size && st.mtimeNs === o.mtimeNs; - }); - if (!intact) { - plan.run.push({decl, reason: 'new'}); - continue; - } - const recorded : RecordedOutput[] = []; - for (let i = 0; i < decl.outputs.length; i++) { - const src = from.outputs[i]!.path; - const dst = decl.outputs[i]!; - if (src !== dst) { - fs.mkdirSync(pathlib.dirname(dst), {recursive: true}); - fs.copyFileSync(src, dst); - } - const st = statPath(dst); - if (st === null) { - throw new BuildError(`Rename copy failed: ${src} -> ${dst}`); - } - recorded.push({path: dst, size: st.size, mtimeNs: st.mtimeNs}); - } - db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded); - renamed++; - } - - // Phase 5: delete outputs of removed rules, prune empty dirs - const staleDirs = new Set(); - for (const s of plan.stale) { - for (const o of s.outputs) { - if (!currentOutputs.has(o.path)) { - try { - fs.unlinkSync(o.path); - } catch (err) { - if ((err as {code? : string}).code !== 'ENOENT') { - throw err; - } - } - staleDirs.add(pathlib.dirname(o.path)); - } - } - db.deleteRule(s.id); - } - for (let dir of staleDirs) { - while (dir.startsWith('build/')) { - try { - fs.rmdirSync(dir); - } catch { - break; - } - dir = pathlib.dirname(dir); - } - } - db.pruneFileCache(sources); - - // Phase 6: adoption (migration): trust existing outputs - for (const decl of plan.adopt) { - const recorded = decl.outputs.map(path => { - const st = statPath(path)!; - return {path, size: st.size, mtimeNs: st.mtimeNs}; - }); - db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded); - } - - // Phase 7: execute - // The worker pool does not serialize producers before consumers; that - // is safe because rules consuming generated files are rejected above. - // `order` is used for a stable, declaration-ordered schedule. - const orderIndex = new Map(order.map((r, i) => [r, i])); - const runList = [...plan.run].sort((a, b) => orderIndex.get(a.decl)! - orderIndex.get(b.decl)!); - const ac = new AbortController(); - let interrupted = false; - const onSignal = () => { - if (interrupted) { - killAllProcessGroups(); - process.exit(130); - } - interrupted = true; - console.error('\nInterrupted; waiting for running rules to stop...'); - ac.abort(); - }; - process.on('SIGINT', onSignal); - process.on('SIGTERM', onSignal); - - const failures : RuleDecl[] = []; - let done = 0; - await workerPool(runList, jobs, async ({decl}) => { - if (ac.signal.aborted) { - return; - } - try { - for (const out of decl.outputs) { - fs.mkdirSync(pathlib.dirname(out), {recursive: true}); - } - const result = await runShell(decl.command, {cwd: root, signal: ac.signal}); - if (ac.signal.aborted && result.code !== 0) { - return; // killed by the abort, not a real failure; stays dirty - } - let recorded : RecordedOutput[] | null = null; - const missingOutputs = []; - if (result.code === 0) { - recorded = []; - for (const path of decl.outputs) { - const st = statPath(path); - if (st === null) { - missingOutputs.push(path); - recorded = null; - break; - } - recorded.push({path, size: st.size, mtimeNs: st.mtimeNs}); - } - } - db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded); - done++; - if (recorded !== null) { - console.log(`[${done}/${runList.length}] ${label(decl)}`); - if (result.output !== '') { - console.log(indent(result.output)); - } - } else { - failures.push(decl); - console.error(`[${done}/${runList.length}] FAILED: ${label(decl)}`); - console.error(` command: ${decl.command}`); - if (result.output !== '') { - console.error(indent(result.output)); - } - if (missingOutputs.length > 0) { - console.error(` command succeeded but did not produce: ${missingOutputs.join(' ')}`); - } - if (opts.failFast) { - ac.abort(); - } - } - } catch (err) { - // Unexpected (infrastructure) error: count the rule failed and - // stop scheduling; something systemic is wrong. - failures.push(decl); - console.error(`FAILED (internal error): ${label(decl)}`); - console.error(indent(err instanceof Error ? err.stack ?? err.message : String(err))); - ac.abort(); - } - }); - process.off('SIGINT', onSignal); - process.off('SIGTERM', onSignal); - - // Phase 8: summary - const parts = [`${plan.clean.length} up to date`]; - if (runList.length > 0) { - parts.push(`${done - failures.length} ran`); - } - if (renamed > 0) { - parts.push(`${renamed} renamed`); - } - if (plan.adopt.length > 0) { - parts.push(`${plan.adopt.length} adopted`); - } - if (plan.stale.length > 0) { - parts.push(`${plan.stale.length} removed`); - } - if (failures.length > 0) { - parts.push(`${failures.length} FAILED`); - } - console.log(parts.join(', ') + '.'); - if (failures.length > 0) { - console.error('Failed rules:'); - for (const decl of failures) { - console.error(` ${label(decl)}`); - } - } - return interrupted ? 130 : failures.length > 0 ? 1 : 0; - } finally { - db.close(); - releaseLock?.(); - } -} - -try { - process.exitCode = await main(); -} catch (err) { - if (err instanceof BuildError) { - console.error(err.message); - process.exitCode = 1; - } else { - throw err; - } -} diff --git a/tools/build/plan.ts b/tools/build/plan.ts deleted file mode 100644 index ae8a6282..00000000 --- a/tools/build/plan.ts +++ /dev/null @@ -1,139 +0,0 @@ - -import {createHash} from 'crypto'; - -import type {RuleDecl} from './api.ts'; -import type {StoredRule} from './db.ts'; - -export type DirtyReason = 'new' | 'failed-last-run' | 'input-changed' - | 'output-missing' | 'output-tampered'; - -export interface OutputStat { - size : bigint; - mtimeNs : bigint; -} - -export interface Plan { - clean : RuleDecl[]; - run : {decl : RuleDecl, reason : DirtyReason}[]; - renames : {decl : RuleDecl, from : StoredRule}[]; - stale : StoredRule[]; - adopt : RuleDecl[]; -} - -export function inputSig(hashes : Buffer[]) : Buffer { - const h = createHash('sha256'); - for (const hash of hashes) { - h.update(hash); - } - return h.digest(); -} - -export function ruleInputSig(decl : RuleDecl, hashes : Map) : Buffer { - return inputSig([...decl.inputs, ...decl.deps].map(p => hashes.get(p)!)); -} - -function renameKey(template : string, sig : Buffer) : string { - return template + '\0' + sig.toString('hex'); -} - -export function computePlan(opts : { - current : RuleDecl[], - stored : StoredRule[], - hashes : Map, - statOutput : (path : string) => OutputStat | null, - adopt : boolean, -}) : Plan { - const {current, stored, hashes, statOutput, adopt} = opts; - - const storedByKey = new Map(); - const renameIndex = new Map(); - for (const s of stored) { - storedByKey.set(s.key, s); - // Rename detection is restricted to single-input, no-deps rules: - // sheet-style rules embed input *names* in their output bytes, so - // identical input content does not imply identical outputs there. - if (s.ok && s.inputs.length === 1 && !s.inputs[0]!.isDep) { - const key = renameKey(s.template, s.inputSig); - let list = renameIndex.get(key); - if (list === undefined) { - renameIndex.set(key, list = []); - } - list.push(s); - } - } - - const currentKeys = new Set(current.map(r => r.key)); - const plan : Plan = { - clean: [], - run: [], - renames: [], - stale: stored.filter(s => !currentKeys.has(s.key)), - adopt: [], - }; - - // A stored rule's outputs are intact iff every recorded output exists on - // disk with its recorded stat. Rename sources must pass this (never copy - // tampered or unverified bytes). - const outputsIntact = (s : StoredRule) : boolean => - s.outputs.every(o => { - if (o.size === null || o.mtimeNs === null) { - return false; - } - const st = statOutput(o.path); - return st !== null && st.size === o.size && st.mtimeNs === o.mtimeNs; - }); - - const dirtyReason = (s : StoredRule) : DirtyReason | null => { - if (!s.ok) { - return 'failed-last-run'; - } - for (const inp of s.inputs) { - if (!hashes.get(inp.path)?.equals(inp.hash)) { - return 'input-changed'; - } - } - for (const o of s.outputs) { - const st = statOutput(o.path); - if (st === null) { - return 'output-missing'; - } - if (st.size !== o.size || st.mtimeNs !== o.mtimeNs) { - return 'output-tampered'; - } - } - return null; - }; - - for (const decl of current) { - const s = storedByKey.get(decl.key); - let reason : DirtyReason | null; - if (s !== undefined) { - reason = dirtyReason(s); - if (reason === null) { - plan.clean.push(decl); - continue; - } - } else { - reason = 'new'; - if (decl.inputs.length === 1 && decl.deps.length === 0) { - const sig = ruleInputSig(decl, hashes); - const candidates = renameIndex.get(renameKey(decl.template, sig)) ?? []; - const from = candidates.find(c => - c.outputs.length === decl.outputs.length && outputsIntact(c)); - if (from !== undefined) { - plan.renames.push({decl, from}); - continue; - } - } - } - // Adoption only covers rules we know nothing about; a known-dirty - // rule (input-changed, tampered, failed) must actually run. - if (adopt && reason === 'new' && decl.outputs.every(p => statOutput(p) !== null)) { - plan.adopt.push(decl); - } else { - plan.run.push({decl, reason}); - } - } - - return plan; -} diff --git a/tools/build/test/exec.test.ts b/tools/build/test/exec.test.ts index d3599c92..95475988 100644 --- a/tools/build/test/exec.test.ts +++ b/tools/build/test/exec.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import {test} from 'node:test'; -import {runShell, workerPool} from '../exec.ts'; +import {runShell} from '../exec.ts'; test('runShell runs && chains from stdin and reports exit status', async () => { const signal = new AbortController().signal; @@ -14,16 +14,3 @@ test('runShell runs && chains from stdin and reports exit status', async () => { assert.equal(bad.output, 'partial\n'); }); -test('workerPool waits for every worker before rethrowing', async () => { - let finished = 0; - await assert.rejects( - workerPool([1, 2, 3, 4], 2, async item => { - if (item === 1) { - throw new Error('boom'); - } - await new Promise(resolve => setTimeout(resolve, 20)); - finished++; - }), - /boom/); - assert.equal(finished, 3); -}); diff --git a/tools/build/test/graph.test.ts b/tools/build/test/graph.test.ts deleted file mode 100644 index 670971bb..00000000 --- a/tools/build/test/graph.test.ts +++ /dev/null @@ -1,29 +0,0 @@ - -import assert from 'node:assert/strict'; -import {test} from 'node:test'; - -import {getRules, resetRules, rule} from '../api.ts'; -import {BuildError, checkGraph} from '../graph.ts'; - -test('checkGraph rejects duplicate outputs', () => { - resetRules(); - rule('a.png', ['c1 %f %o'], 'out/x.png'); - rule('b.png', ['c2 %f %o'], 'out/x.png'); - assert.throws(() => checkGraph(getRules()), BuildError); -}); - -test('checkGraph rejects cycles', () => { - resetRules(); - rule('gen/y', ['c1 %f %o'], 'gen/x'); - rule('gen/x', ['c2 %f %o'], 'gen/y'); - assert.throws(() => checkGraph(getRules()), /cycle/i); -}); - -test('checkGraph orders producers before consumers', () => { - resetRules(); - rule('gen/mid', ['consume %f %o'], 'out/final'); - rule('src/a', ['produce %f %o'], 'gen/mid'); - const {order, generated} = checkGraph(getRules()); - assert.deepEqual(order.map(r => r.outputs[0]), ['gen/mid', 'out/final']); - assert.deepEqual([...generated].sort(), ['gen/mid', 'out/final']); -}); diff --git a/tools/build/test/plan.test.ts b/tools/build/test/plan.test.ts deleted file mode 100644 index 080f0b27..00000000 --- a/tools/build/test/plan.test.ts +++ /dev/null @@ -1,217 +0,0 @@ - -import assert from 'node:assert/strict'; -import {test} from 'node:test'; - -import {forEachRule, getRules, resetRules, rule, type RuleDecl} from '../api.ts'; -import type {StoredRule, StoredRuleOutput} from '../db.ts'; -import {computePlan, type OutputStat, ruleInputSig} from '../plan.ts'; - -function hashOf(content : string) : Buffer { - return Buffer.from(content.padEnd(32, '\0')); -} - -function makeForeach(input : string, outputDir : string) : RuleDecl { - resetRules(); - forEachRule(input, {cmds: ['convert %f %o']}, `${outputDir}/%b`); - const decl = getRules()[0]!; - assert(decl !== undefined); - return decl; -} - -function stored(decl : RuleDecl, hashes : Map, - opts : {ok? : boolean, outputs? : StoredRuleOutput[]} = {}) : StoredRule { - return { - id: 1n, - key: decl.key, - command: decl.command, - display: decl.display, - template: decl.template, - inputSig: ruleInputSig(decl, hashes), - ok: opts.ok ?? true, - inputs: [ - ...decl.inputs.map(path => ({path, isDep: false, hash: hashes.get(path)!})), - ...decl.deps.map(path => ({path, isDep: true, hash: hashes.get(path)!})), - ], - outputs: opts.outputs ?? decl.outputs.map(path => ({path, size: 10n, mtimeNs: 100n})), - }; -} - -function statFrom(entries : Record) { - return (path : string) : OutputStat | null => entries[path] ?? null; -} - -const GOOD : OutputStat = {size: 10n, mtimeNs: 100n}; - -test('unchanged rule is clean', () => { - const decl = makeForeach('src/a.png', 'build/out'); - const hashes = new Map([['src/a.png', hashOf('A')]]); - const plan = computePlan({ - current: [decl], - stored: [stored(decl, hashes)], - hashes, - statOutput: statFrom({'build/out/a.png': GOOD}), - adopt: false, - }); - assert.deepEqual(plan.clean, [decl]); - assert.equal(plan.run.length, 0); -}); - -test('dirty reasons', () => { - const decl = makeForeach('src/a.png', 'build/out'); - const hashes = new Map([['src/a.png', hashOf('A')]]); - const cases : [StoredRule, (p : string) => OutputStat | null, string][] = [ - [stored(decl, new Map([['src/a.png', hashOf('OLD')]])), - statFrom({'build/out/a.png': GOOD}), 'input-changed'], - [stored(decl, hashes, {ok: false}), - statFrom({'build/out/a.png': GOOD}), 'failed-last-run'], - [stored(decl, hashes), statFrom({}), 'output-missing'], - [stored(decl, hashes), - statFrom({'build/out/a.png': {size: 11n, mtimeNs: 100n}}), 'output-tampered'], - ]; - for (const [s, statOutput, reason] of cases) { - const plan = computePlan({current: [decl], stored: [s], hashes, statOutput, adopt: false}); - assert.deepEqual(plan.run.map(r => r.reason), [reason]); - } -}); - -test('unknown rule runs as new; removed rule is stale', () => { - const decl = makeForeach('src/a.png', 'build/out'); - const gone = makeForeach('src/z.png', 'build/out'); - const hashes = new Map([['src/a.png', hashOf('A')], ['src/z.png', hashOf('Z')]]); - const plan = computePlan({ - current: [decl], - stored: [stored(gone, hashes)], - hashes, - statOutput: statFrom({}), - adopt: false, - }); - assert.deepEqual(plan.run.map(r => r.reason), ['new']); - assert.deepEqual(plan.stale.map(s => s.key), [gone.key]); -}); - -test('renamed input with identical content matches instead of running', () => { - const oldDecl = makeForeach('src/a.png', 'build/out'); - const newDecl = makeForeach('src/b.png', 'build/out'); - const hashes = new Map([['src/a.png', hashOf('SAME')], ['src/b.png', hashOf('SAME')]]); - const plan = computePlan({ - current: [newDecl], - stored: [stored(oldDecl, hashes)], - hashes, - statOutput: statFrom({'build/out/a.png': GOOD}), - adopt: false, - }); - assert.equal(plan.run.length, 0); - assert.deepEqual(plan.renames.map(r => [r.from.outputs[0]!.path, r.decl.outputs[0]]), - [['build/out/a.png', 'build/out/b.png']]); - // the source rule is still deleted afterward - assert.deepEqual(plan.stale.map(s => s.key), [oldDecl.key]); -}); - -test('rename does not match differing content or tampered source outputs', () => { - const oldDecl = makeForeach('src/a.png', 'build/out'); - const newDecl = makeForeach('src/b.png', 'build/out'); - - const differing = new Map([['src/a.png', hashOf('X')], ['src/b.png', hashOf('Y')]]); - let plan = computePlan({ - current: [newDecl], - stored: [stored(oldDecl, differing)], - hashes: differing, - statOutput: statFrom({'build/out/a.png': GOOD}), - adopt: false, - }); - assert.deepEqual(plan.run.map(r => r.reason), ['new']); - - const same = new Map([['src/a.png', hashOf('SAME')], ['src/b.png', hashOf('SAME')]]); - plan = computePlan({ - current: [newDecl], - stored: [stored(oldDecl, same)], - hashes: same, - statOutput: statFrom({'build/out/a.png': {size: 99n, mtimeNs: 100n}}), - adopt: false, - }); - assert.deepEqual(plan.run.map(r => r.reason), ['new']); -}); - -test('rename is restricted to single-input, no-deps rules', () => { - // multi-input: same output basename, same content, but never rename-matched - resetRules(); - rule(['src/a.png', 'src/b.png'], {cmds: ['c %f %o']}, 'out1/x.png'); - rule(['src/a2.png', 'src/b2.png'], {cmds: ['c %f %o']}, 'out2/x.png'); - const [multiOld, multiNew] = getRules() as [RuleDecl, RuleDecl]; - const multiHashes = new Map([ - ['src/a.png', hashOf('A')], ['src/b.png', hashOf('B')], - ['src/a2.png', hashOf('A')], ['src/b2.png', hashOf('B')], - ]); - let plan = computePlan({ - current: [multiNew], - stored: [stored(multiOld, multiHashes)], - hashes: multiHashes, - statOutput: statFrom({'out1/x.png': GOOD}), - adopt: false, - }); - assert.deepEqual(plan.run.map(r => r.reason), ['new']); - - // dep-bearing: same template and sig, but never rename-matched - resetRules(); - forEachRule('src/a.png', {deps: 'src/d.json', cmds: ['c %f %o']}, 'out/%b'); - forEachRule('src/b.png', {deps: 'src/d.json', cmds: ['c %f %o']}, 'out/%b'); - const [depOld, depNew] = getRules() as [RuleDecl, RuleDecl]; - const depHashes = new Map([ - ['src/a.png', hashOf('SAME')], ['src/b.png', hashOf('SAME')], - ['src/d.json', hashOf('D')], - ]); - plan = computePlan({ - current: [depNew], - stored: [stored(depOld, depHashes)], - hashes: depHashes, - statOutput: statFrom({'out/a.png': GOOD}), - adopt: false, - }); - assert.deepEqual(plan.run.map(r => r.reason), ['new']); -}); - -test('rename does not match across input extension changes', () => { - const oldDecl = makeForeach('src/a.png', 'build/out'); - resetRules(); - forEachRule('src/a.gif', {cmds: ['convert %f %o']}, 'build/out/%b'); - const newDecl = getRules()[0]!; - const hashes = new Map([['src/a.png', hashOf('SAME')], ['src/a.gif', hashOf('SAME')]]); - const plan = computePlan({ - current: [newDecl], - stored: [stored(oldDecl, hashes)], - hashes, - statOutput: statFrom({'build/out/a.png': GOOD}), - adopt: false, - }); - assert.deepEqual(plan.run.map(r => r.reason), ['new']); -}); - -test('adopt does not bless known-dirty rules', () => { - const decl = makeForeach('src/a.png', 'build/out'); - const plan = computePlan({ - current: [decl], - stored: [stored(decl, new Map([['src/a.png', hashOf('OLD')]]))], - hashes: new Map([['src/a.png', hashOf('NEW')]]), - statOutput: statFrom({'build/out/a.png': GOOD}), - adopt: true, - }); - assert.equal(plan.adopt.length, 0); - assert.deepEqual(plan.run.map(r => r.reason), ['input-changed']); -}); - -test('adopt records existing outputs instead of running', () => { - resetRules(); - rule('src/a.png', {cmds: ['convert %f %o']}, 'build/present.png'); - rule('src/b.png', {cmds: ['convert %f %o']}, 'build/absent.png'); - const [present, absent] = getRules() as [RuleDecl, RuleDecl]; - const hashes = new Map([['src/a.png', hashOf('A')], ['src/b.png', hashOf('B')]]); - const plan = computePlan({ - current: [present, absent], - stored: [], - hashes, - statOutput: statFrom({'build/present.png': GOOD}), - adopt: true, - }); - assert.deepEqual(plan.adopt, [present]); - assert.deepEqual(plan.run.map(r => r.decl), [absent]); -}); diff --git a/tools/build/test/sprites.test.ts b/tools/build/test/sprites.test.ts index a7d45d37..eb64e4f9 100644 --- a/tools/build/test/sprites.test.ts +++ b/tools/build/test/sprites.test.ts @@ -5,7 +5,7 @@ import os from 'node:os'; import pathlib from 'node:path'; import {test} from 'node:test'; -import {base, spritedata, spriteglob} from '../api.ts'; +import {base, spritedata, spriteglob} from '../helpers.ts'; test('spritedata parses id and flags', () => { assert.deepEqual(spritedata('10080'), {id: '10080', data: {}}); diff --git a/tools/deploy/api.ts b/tools/deploy/api.ts new file mode 100644 index 00000000..a3f011e4 --- /dev/null +++ b/tools/deploy/api.ts @@ -0,0 +1,92 @@ + +import crypto from 'crypto'; +import fs from 'fs'; + +import b32encode from 'base32-encode'; + +import {Artifact} from '../build/artifact.ts'; +import {casPath} from '../build/cas.ts'; +import {type ActionQueue} from './queue.ts'; +import * as pathlib from './path.ts'; + +// A source-tree file from ctx.list(); shaped like a deploy Path plus the +// repo-relative path. +export interface SrcFile extends pathlib.Path { + path : string; +} + +export type CopySource = Artifact | SrcFile | string; // string = repo-relative path + +// The finish API: nice naming over built artifacts and raw sources. Ops are +// queued in call order (the tar entry order), so __key-style entries must be +// written first. +export interface DeployCtx { + copy(src : CopySource, dst : string) : void; + write(dst : string, data : string) : void; + read(src : CopySource) : string; + list(dir : string) : SrcFile[]; + // 8-char base32 content stamp. One source: the digest of its bytes, + // byte-compatible with artifact hashes. Several: a digest of the sorted + // per-source digests (order-insensitive). + hash(...srcs : CopySource[]) : string; +} + +export interface DeploySpec { + finish : (ctx : DeployCtx) => void | Promise; +} + +// Identity helper: a deploy module declares its rules at top level and +// `export default defineDeploy({finish})`. +export function defineDeploy(spec : DeploySpec) : DeploySpec { + return spec; +} + +function shortHash(digest : Buffer) : string { + return b32encode(digest, 'RFC4648').slice(0, 8); +} + +export function makeCtx(casDir : string, queue : ActionQueue) : DeployCtx { + // Every path here is repo-root-relative; the CLI chdirs to the root. + const srcPath = (src : CopySource) : string => { + if (src instanceof Artifact) { + return casPath(casDir, src.hash, src.ext); + } + return typeof src === 'string' ? src : src.path; + }; + const digestOf = (src : CopySource) : Buffer => { + if (src instanceof Artifact) { + return Buffer.from(src.hash, 'hex'); + } + return crypto.createHash('sha256').update(fs.readFileSync(srcPath(src))).digest(); + }; + return { + copy(src : CopySource, dst : string) : void { + queue.copy(srcPath(src), dst); + }, + write(dst : string, data : string) : void { + queue.write(data, dst); + }, + read(src : CopySource) : string { + return fs.readFileSync(srcPath(src), 'utf8'); + }, + list(dir : string) : SrcFile[] { + const result = []; + for (const filename of fs.readdirSync(dir).sort()) { + const p = pathlib.path(filename, {dir}); + result.push({...p, path: pathlib.format(p)}); + } + return result; + }, + hash(...srcs : CopySource[]) : string { + if (srcs.length === 1) { + return shortHash(digestOf(srcs[0]!)); + } + const digests = srcs.map(digestOf).sort(Buffer.compare); + const h = crypto.createHash('sha256'); + for (const d of digests) { + h.update(d); + } + return shortHash(h.digest()); + }, + }; +} diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index a25374e6..8e9fbdaf 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -1,78 +1,159 @@ -import program from 'commander'; -import * as script from './script.ts'; -import nodePath from 'path'; import {spawn, type ChildProcess} from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import nodePath from 'path'; +import {fileURLToPath, pathToFileURL} from 'url'; -function collect(value : string, previous : string[]) { - return previous.concat([value]); +import {program} from 'commander'; + +import {type RuleDecl, getDecls} from '../build/artifact.ts'; +import {casPath} from '../build/cas.ts'; +import {loadConfig} from '../build/config.ts'; +import {build} from '../build/driver.ts'; +import {BuildError} from '../build/errors.ts'; +import {killAllProcessGroups} from '../build/exec.ts'; +import {setConfig} from '../build/helpers.ts'; +import {Store, acquireLock, dbVersion} from '../build/store.ts'; +import {type DeploySpec, makeCtx} from './api.ts'; +import {ActionQueue} from './queue.ts'; + +const root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..'); +process.chdir(root); + +const DB_PATH = '.build/db.sqlite'; +const LOCK_PATH = '.build/lock.sqlite'; +const CAS_DIR = '.build/cas'; +const TMP_DIR = '.build/tmp'; + +interface CommonOpts { + jobs : string; + dryRun? : boolean; + failFast? : boolean; + config : string; + verbose? : boolean; } -async function runAq(aq : script.ActionQueue, mode: 'copy' | 'link' | 'tar', outputDir : undefined | string, verbose : undefined | true) { - const level = verbose ? 'all' : 'errors'; +function common(cmd : ReturnType) : ReturnType { + return cmd + .option('-j, --jobs ', 'number of parallel jobs', String(os.availableParallelism())) + .option('-n, --dry-run', 'print what would run without changing anything') + .option('--fail-fast', 'stop scheduling new rules after the first failure') + .option('--config ', 'config file', 'build.config') + .option('-v, --verbose', 'print more detail'); +} + +function discoverDeployFiles() : string[] { + const files = fs.readdirSync('.').filter(f => f.endsWith('.deploy.ts')).sort(); + if (files.length === 0) { + throw new BuildError('No *.deploy.ts files at the repo root'); + } + return files; +} + +// Importing a deploy module declares its rules; the default export carries +// the finish function. +async function importDeploys(files : string[]) : Promise> { + const specs = new Map(); + for (const file of files) { + const mod : {default? : unknown} = await import(pathToFileURL(nodePath.resolve(file)).href); + const spec = mod.default; + if (typeof spec === 'object' && spec !== null + && typeof (spec as DeploySpec).finish === 'function') { + specs.set(file, spec as DeploySpec); + } else { + specs.set(file, null); + } + } + return specs; +} + +// Build `decls` and, on success, run `then` while still holding the lock (a +// concurrent GC must not sweep CAS objects out from under a finish). Returns +// the process exit code. +async function buildThen(decls : readonly RuleDecl[], opts : CommonOpts, gc : boolean, + then? : () => Promise) : Promise { + const jobs = Number(opts.jobs); + if (!Number.isInteger(jobs) || jobs < 1) { + throw new BuildError(`Invalid --jobs value: ${opts.jobs}`); + } + const dryRun = Boolean(opts.dryRun); + const release = dryRun ? null : acquireLock(LOCK_PATH); + try { + // A dry run must not create state (opening a db migrates it); + // without a current-version db it reads from an empty in-memory one. + const dbPath = dryRun && dbVersion(DB_PATH) !== 2 ? ':memory:' : DB_PATH; + const store = new Store(dbPath); + if (!dryRun) { + fs.rmSync(TMP_DIR, {recursive: true, force: true}); + } + + const ac = new AbortController(); + let interrupted = false; + const onSignal = () => { + if (interrupted) { + killAllProcessGroups(); + process.exit(130); + } + interrupted = true; + console.error('\nInterrupted; waiting for running rules to stop...'); + ac.abort(); + }; + process.on('SIGINT', onSignal); + process.on('SIGTERM', onSignal); + let ok : boolean; + try { + const result = await build(decls, { + root, + store, + casDir: CAS_DIR, + tmpDir: TMP_DIR, + jobs, + dryRun, + failFast: Boolean(opts.failFast), + verbose: Boolean(opts.verbose), + gc, + signal: ac.signal, + }); + ok = result.ok; + } finally { + process.off('SIGINT', onSignal); + process.off('SIGTERM', onSignal); + store.close(); + } + if (interrupted) { + return 130; + } + if (dryRun) { + return 0; + } + if (!ok) { + return 1; + } + return then === undefined ? 0 : await then(); + } finally { + release?.(); + } +} + +function finishOf(specs : Map, file : string) : DeploySpec { + const spec = specs.get(file); + if (spec === null || spec === undefined) { + throw new BuildError(`${file} does not default-export a deploy (use defineDeploy)`); + } + return spec; +} + +async function runFinish(spec : DeploySpec, verbose : boolean) : Promise { + const aq = new ActionQueue(); + await spec.finish(makeCtx(CAS_DIR, aq)); if (!aq.valid) { - aq.print(level); - process.exit(1); - } - if (outputDir !== undefined) { - await aq.run(outputDir, mode); - } else { - if (level === 'errors') { - console.error(`Success, but nothing to do. Please rerun with -v or -o`); - } else { - aq.print('all'); - } + aq.print(verbose ? 'all' : 'errors'); + return null; } + return aq; } -program - .command('copy [files...]') - .option('-o, --output ', 'Output directory') - // TODO: default toID - .option('-e, --eval ', 'Expr') - .option('-m, --module ', 'Module') - .option('-v, --verbose', 'Verbose') - .option('--link', 'Link') - .option('--tar', 'Tar') - // TODO - // .option('-t, --tag ', 'Tag', collect, []) - // from rename(1) - .action(async (files : string[], {eval: expr, module: mod, output: outputDir, verbose, link, tar}) => { - let scr; - if (expr !== undefined) { - scr = new script.Script(expr, 'expr'); - } else if (mod !== undefined) { - scr = new script.Script(mod, 'file'); - } else { - throw new Error(`one of -e or -m must be provided`); - } - - const aq = new script.ActionQueue; - - for (const src of files) { - script.runOnFile(scr, src, aq); - } - - await runAq(aq, tar ? 'tar' : link ? 'link' : 'copy', outputDir, verbose); - }); - -program - .command('run [scripts...]') - .option('-o, --output ', 'Output directory') - .option('-v, --verbose', 'Verbose') - .option('--link', 'Link') - .option('--tar', 'Tar') - .action(async (scripts : string[], {output: outputDir, verbose, link, tar}) => { - const aq = new script.ActionQueue; - - for (const file of scripts) { - const scr = new script.Script(file, 'file'); - script.run(scr, nodePath.dirname(file), aq); - } - - await runAq(aq, tar ? 'tar' : link ? 'link' : 'copy', outputDir, verbose); - }); - function waitExit(child : ChildProcess) : Promise { return new Promise((resolve, reject) => { child.on('error', reject); @@ -80,49 +161,138 @@ function waitExit(child : ChildProcess) : Promise { }); } -program - .command('deploy [scripts...]') - .option('-v, --verbose', 'Verbose') - .action(async (scripts : string[], {verbose}) => { +common(program.command('build [files...]')) + .description('build the rules of the given deploys (default: all *.deploy.ts)') + .action(async (files : string[], opts : CommonOpts) => { + setConfig(loadConfig(opts.config)); + // GC needs the full rule universe: only an unfiltered union build + // can know which keys are no longer declared anywhere. + const gc = files.length === 0 && !Boolean(opts.dryRun); + await importDeploys(files.length > 0 ? files : discoverDeployFiles()); + process.exitCode = await buildThen(getDecls(), opts, gc); + }); + +common(program.command('deploy ')) + .description('build, finish, and pipe the tar to DEPLOY_COMMAND from .env') + .action(async (file : string, opts : CommonOpts) => { try { process.loadEnvFile('.env'); } catch { console.error(`missing .env; set DEPLOY_COMMAND="ssh smogon smogonctl assets upload"`); - process.exit(1); + process.exitCode = 1; + return; } const command = process.env.DEPLOY_COMMAND; if (command === undefined) { console.error(`DEPLOY_COMMAND not set in .env`); - process.exit(1); - } - - const build = spawn('pnpm', ['build'], {stdio: ['ignore', 'inherit', 'inherit']}); - if (await waitExit(build) !== 0) { - process.exit(1); - } - - const aq = new script.ActionQueue; - for (const file of scripts) { - const scr = new script.Script(file, 'file'); - script.run(scr, nodePath.dirname(file), aq); - } - if (!aq.valid) { - aq.print(verbose ? 'all' : 'errors'); - process.exit(1); - } - - const upload = spawn(command, {shell: true, stdio: ['pipe', 'inherit', 'inherit']}); - // If the command dies early we report its exit code; don't also crash - // on the resulting EPIPE. - upload.stdin!.on('error', () => {}); - aq.pack().pipe(upload.stdin!); - if (await waitExit(upload) !== 0) { - process.exit(1); + process.exitCode = 1; + return; } + setConfig(loadConfig(opts.config)); + const specs = await importDeploys([file]); + process.exitCode = await buildThen(getDecls(), opts, false, async () => { + const aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose)); + if (aq === null) { + return 1; + } + const upload = spawn(command, {shell: true, stdio: ['pipe', 'inherit', 'inherit']}); + // If the command dies early we report its exit code; don't also + // crash on the resulting EPIPE. + upload.stdin!.on('error', () => {}); + aq.pack().pipe(upload.stdin!); + return await waitExit(upload) !== 0 ? 1 : 0; + }); }); -program.parse(process.argv); +common(program.command('run ')) + .description('build, finish, and materialize to a directory (or tar file)') + .requiredOption('-o, --output ', 'output directory (a file with --tar)') + .option('--link', 'hardlink instead of copying') + .option('--tar', 'write a tar file') + .action(async (file : string, opts : CommonOpts & {output : string, link? : boolean, tar? : boolean}) => { + setConfig(loadConfig(opts.config)); + const specs = await importDeploys([file]); + process.exitCode = await buildThen(getDecls(), opts, false, async () => { + const aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose)); + if (aq === null) { + return 1; + } + await aq.run(opts.output, opts.tar ? 'tar' : opts.link ? 'link' : 'copy'); + return 0; + }); + }); -if (process.argv.slice(2).length === 0) { - program.outputHelp(); +function slugOf(decl : RuleDecl) : string { + const template = decl.displayTemplate ?? decl.cmds[0]!; + const slug = template.replace(/%[a-zA-Z0-9]+/g, ' ') + .toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); + return slug === '' ? 'rule' : slug; +} + +common(program.command('inspect ')) + .description('build every rule touching the given source paths and copy the outputs out') + .requiredOption('-o, --output ', 'output directory') + .action(async (paths : string[], opts : CommonOpts & {output : string}) => { + setConfig(loadConfig(opts.config)); + await importDeploys(discoverDeployFiles()); + const targets = paths.map(p => nodePath.normalize(p).replace(/\/+$/, '')); + + const closure = new Set(); + for (const decl of getDecls()) { + const hit = [...decl.inputs, ...decl.deps].some(i => typeof i === 'string' + && targets.some(t => i === t || i.startsWith(t + '/'))); + if (hit) { + closure.add(decl); + } + } + if (closure.size === 0) { + throw new BuildError(`No rules consume: ${targets.join(' ')}`); + } + // Transitive consumers: show everything these files end up in. The + // executor pulls in any producers the closure needs on its own. + for (let grew = true; grew;) { + grew = false; + for (const decl of getDecls()) { + if (closure.has(decl)) { + continue; + } + if ([...decl.inputs, ...decl.deps].some(i => typeof i !== 'string' && closure.has(i.decl))) { + closure.add(decl); + grew = true; + } + } + } + + const decls = getDecls().filter(d => closure.has(d)); + console.log(`inspect: ${decls.length} rules`); + process.exitCode = await buildThen(decls, opts, false, async () => { + for (const decl of decls) { + const dir = nodePath.join(opts.output, slugOf(decl)); + fs.mkdirSync(dir, {recursive: true}); + for (const artifact of decl.outputs) { + let dst = nodePath.join(dir, artifact.filename); + for (let n = 2; fs.existsSync(dst); n++) { + dst = nodePath.join(dir, `${artifact.name}-${n}.${artifact.ext}`); + } + fs.copyFileSync(casPath(CAS_DIR, artifact.hash, artifact.ext), dst); + fs.chmodSync(dst, 0o644); + console.log(`${nodePath.relative(opts.output, dst)}`); + } + } + return 0; + }); + }); + +try { + await program.parseAsync(process.argv); + if (process.argv.slice(2).length === 0) { + program.outputHelp(); + } +} catch (err) { + if (err instanceof BuildError) { + console.error(err.message); + process.exitCode = 1; + } else { + throw err; + } } diff --git a/tools/deploy/script.ts b/tools/deploy/queue.ts similarity index 59% rename from tools/deploy/script.ts rename to tools/deploy/queue.ts index 860cad8b..2a7ae153 100644 --- a/tools/deploy/script.ts +++ b/tools/deploy/queue.ts @@ -1,12 +1,7 @@ import fs from 'fs'; import nodePath from 'path'; -import vm from 'vm'; -import * as pathlib from './path.ts'; -import * as spritedata from '@smogon/sprite-data/index.ts'; import tar from 'tar-stream'; -import crypto from 'crypto'; -import b32encode from 'base32-encode'; type Op = { type : 'Write', @@ -184,127 +179,3 @@ export class ActionQueue { return t; } } - -export class Script extends vm.Script { - public readonly filename : string | null; - - constructor(x : string, type : 'file' | 'expr') { - let code : string; - let filename : string | null = null; - if (type === 'expr') { - // Force expression parsing - code = `(${x})`; - } else { - code = fs.readFileSync(x, 'utf8'); - filename = x; - } - super(code, filename !== null ? {filename} : undefined); - this.filename = filename; - } -} - -const SKIP = {}; - -const ENV0 = { - spritedata, - SKIP, - // Because throw SKIP isn't an expression - skip() { - throw SKIP; - } -}; - -function makeEnv1(queue: ActionQueue) { - return { - __proto__: ENV0, - debug(obj : unknown) { - queue.debug(obj); - }, - - gdebug(obj : unknown) { - queue.gdebug(obj, false); - } - } -}; - -function makeEnv2(srcDir : string, queue: ActionQueue) { - return { - __proto__: makeEnv1(queue), - - list(dir : string) : pathlib.Path[] { - const result = []; - for (const filename of fs.readdirSync(nodePath.join(srcDir, dir))) { - result.push(pathlib.path(filename, {dir})); - } - return result; - }, - - copy(srcp : pathlib.PathLike, dstp : string | pathlib.Delta /* todo deltalike */) { - const src = pathlib.format(pathlib.path(srcp)); - let dst : string; - if (typeof dstp === 'string') { - dst = dstp; - } else { - dst = pathlib.format(pathlib.path(srcp, dstp)); - } - queue.copy(nodePath.join(srcDir, src), dst); - }, - - read(srcp : pathlib.PathLike) : string { - const src = pathlib.format(pathlib.path(srcp)); - return fs.readFileSync(nodePath.join(srcDir, src), 'utf8'); - }, - - hash(...srcps : pathlib.PathLike[]) : string { - let hash = crypto.createHash("sha256"); - let srcs = srcps.map(srcp => pathlib.format(pathlib.path(srcp))).sort(); - for (let src of srcs) { - let data = fs.readFileSync(nodePath.join(srcDir, src)); - hash.update(data); - } - let buffer = hash.digest() - // Similar to esbuild? - return b32encode(buffer, 'RFC4648').slice(0, 8); - }, - - write(dstp : pathlib.PathLike, data : string) { - const dst = pathlib.format(pathlib.path(dstp)); - queue.write(data, dst); - } - } -} - -export function runOnFile(scr : Script, src : string, queue: ActionQueue) { - try { - const input = pathlib.path(src, {dir: ""}); - const result = scr.runInNewContext({ - __proto__: makeEnv1(queue), - path: input, - ...input - }); - if (result === undefined) { - throw new Error(`undefined output on ${src}`); - } - const output = pathlib.update(input, result); - const dst = pathlib.format(output); - queue.copy(src, dst); - } catch(e) { - if (e === SKIP) { - queue.skip(); - return; - } - queue.throw(e); - } -} - -export function run(scr : Script, srcDir : string, queue : ActionQueue) { - try { - scr.runInNewContext(makeEnv2(srcDir, queue)); - } catch(e) { - if (e === SKIP) { - queue.skip(); - return; - } - queue.throw(e); - } -} diff --git a/tools/deploy/test/api.test.ts b/tools/deploy/test/api.test.ts new file mode 100644 index 00000000..042ea620 --- /dev/null +++ b/tools/deploy/test/api.test.ts @@ -0,0 +1,141 @@ + +import assert from 'node:assert/strict'; +import {createHash} from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import pathlib from 'node:path'; +import {beforeEach, test} from 'node:test'; + +import b32encode from 'base32-encode'; +import tar from 'tar-stream'; + +import {resetDecls, rule} from '../../build/artifact.ts'; +import {casPath} from '../../build/cas.ts'; +import {makeCtx} from '../api.ts'; +import {ActionQueue} from '../queue.ts'; + +beforeEach(resetDecls); + +function tmpdir() : string { + return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-api-test-')); +} + +function shortHash(data : Buffer | string) : string { + return b32encode(createHash('sha256').update(data).digest(), 'RFC4648').slice(0, 8); +} + +// Stage `content` as a built artifact in a scratch CAS. +function makeArtifact(casDir : string, content : string, ext : string) { + const digest = createHash('sha256').update(content).digest('hex'); + const [artifact] = rule('in.png', ['t %f %o'], [`art.${ext}`]); + artifact!.resolve(digest); + const obj = casPath(casDir, digest, ext); + fs.mkdirSync(pathlib.dirname(obj), {recursive: true}); + fs.writeFileSync(obj, content); + return artifact!; +} + +test('ctx.hash matches the historical single-file stamp for artifacts and files', () => { + const dir = tmpdir(); + const file = pathlib.join(dir, 'f.png'); + fs.writeFileSync(file, 'stamp-me'); + const artifact = makeArtifact(pathlib.join(dir, 'cas'), 'stamp-me', 'png'); + const ctx = makeCtx(pathlib.join(dir, 'cas'), new ActionQueue()); + assert.equal(ctx.hash(file), shortHash('stamp-me')); + assert.equal(ctx.hash(artifact), shortHash('stamp-me')); +}); + +test('multi-source ctx.hash is order-insensitive and content-sensitive', () => { + const dir = tmpdir(); + const a = pathlib.join(dir, 'a.png'); + const b = pathlib.join(dir, 'b.png'); + fs.writeFileSync(a, 'aaa'); + fs.writeFileSync(b, 'bbb'); + const ctx = makeCtx(pathlib.join(dir, 'cas'), new ActionQueue()); + const before = ctx.hash(a, b); + assert.equal(before, ctx.hash(b, a)); + assert.notEqual(before, ctx.hash(a)); + fs.writeFileSync(b, 'changed'); + assert.notEqual(ctx.hash(a, b), before); +}); + +test('ctx queues artifact copies from the CAS, writes and reads', () => { + const dir = tmpdir(); + const casDir = pathlib.join(dir, 'cas'); + const artifact = makeArtifact(casDir, 'bytes', 'webp'); + const aq = new ActionQueue(); + const ctx = makeCtx(casDir, aq); + ctx.write('__key', 'sprites'); + ctx.copy(artifact, 'sprites/x.webp'); + assert.equal(ctx.read(artifact), 'bytes'); + const ops = aq.log.filter(e => e.type === 'Op'); + assert.deepEqual(ops.map(e => e.dst), ['__key', 'sprites/x.webp']); + assert.equal((ops[1] as {op : {src : string}}).op.src, casPath(casDir, artifact.hash, 'webp')); +}); + +test('ctx.list sorts and parses extensions', () => { + const dir = tmpdir(); + fs.writeFileSync(pathlib.join(dir, 'b.png'), ''); + fs.writeFileSync(pathlib.join(dir, 'a.gif'), ''); + fs.writeFileSync(pathlib.join(dir, 'noext'), ''); + const ctx = makeCtx('cas', new ActionQueue()); + assert.deepEqual(ctx.list(dir), [ + {dir, name: 'a', ext: 'gif', path: pathlib.join(dir, 'a.gif')}, + {dir, name: 'b', ext: 'png', path: pathlib.join(dir, 'b.png')}, + {dir, name: 'noext', ext: null, path: pathlib.join(dir, 'noext')}, + ]); +}); + +function packedEntries(aq : ActionQueue) : Promise<{name : string, data : string}[]> { + return new Promise((resolve, reject) => { + const extract = tar.extract(); + const entries : {name : string, data : string}[] = []; + extract.on('entry', (header, stream, next) => { + const chunks : Buffer[] = []; + stream.on('data', c => chunks.push(c)); + stream.on('end', () => { + entries.push({name: header.name, data: Buffer.concat(chunks).toString()}); + next(); + }); + }); + extract.on('finish', () => resolve(entries)); + extract.on('error', reject); + aq.pack().pipe(extract); + }); +} + +test('pack preserves op order with __key first', async () => { + const aq = new ActionQueue(); + aq.write('sprites', '__key'); + aq.write('zzz', 'z.txt'); + aq.write('aaa', 'a.txt'); + assert.deepEqual(await packedEntries(aq), [ + {name: '__key', data: 'sprites'}, + {name: 'z.txt', data: 'zzz'}, + {name: 'a.txt', data: 'aaa'}, + ]); +}); + +test('duplicate and absolute destinations invalidate the queue', () => { + const dup = new ActionQueue(); + dup.write('a', 'x.txt'); + dup.write('b', 'x.txt'); + assert.ok(!dup.valid); + assert.throws(() => dup.pack(), /Invalid ActionQueue/); + + const abs = new ActionQueue(); + abs.write('a', '/etc/passwd'); + assert.ok(!abs.valid); +}); + +test('copy-mode materialization restores 0644 on read-only sources', async () => { + const dir = tmpdir(); + const src = pathlib.join(dir, 'obj'); + fs.writeFileSync(src, 'x'); + fs.chmodSync(src, 0o444); + const aq = new ActionQueue(); + aq.copy(src, 'out/x.png'); + const out = pathlib.join(dir, 'deploy'); + await aq.run(out, 'copy'); + assert.equal(fs.statSync(pathlib.join(out, 'out/x.png')).mode & 0o777, 0o644); +}); diff --git a/tools/deploy/tsconfig.json b/tools/deploy/tsconfig.json index 7319ce4d..22048ddf 100644 --- a/tools/deploy/tsconfig.json +++ b/tools/deploy/tsconfig.json @@ -1,6 +1,5 @@ { "extends": "../../tsconfig-base", - "exclude": ["test"], "references": [ {"path": "../../data/lib"}, {"path": "../build"}, diff --git a/tsconfig.json b/tsconfig.json index 63561910..efd01059 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "./tsconfig-base", - "include": ["Buildfile.ts", "*.deploy.ts", "rules/*.ts"], + "include": ["*.deploy.ts", "rules/*.ts"], "references": [ {"path": "tools/build"}, {"path": "tools/deploy"}, diff --git a/updatexyicons.sh b/updatexyicons.sh index 5a05e2d6..2427bcff 100755 --- a/updatexyicons.sh +++ b/updatexyicons.sh @@ -4,9 +4,7 @@ set -ex cd ~/smogon/sprites -pnpm build - rm -rf deploy/smogon -node tools/deploy/index.ts run smogon.deploy.js -o deploy/smogon +node tools/deploy/index.ts run smogon.deploy.ts -o deploy/smogon rsync -a deploy/smogon/xyicons/ smogon:/smog2/sprites/xyicons