Leahnaya/sprites
1
0
Fork 0
mirror of https://github.com/smogon/sprites.git synced 2026-08-25 03:07:29 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Cut over to content-addressed deploys

Browse Source 
pnpm build/deploy now run the umbrella CLI (build/deploy/run/inspect); the
fixed-name build/ tree, the old planner (rename detection, adopt, tamper
checks), and the vm deploy sandbox are gone. Also lands tools/deploy/api.ts
and the CLI, which earlier commits referenced but an overbroad local
gitignore pattern had silently excluded.

Migration: the store migrates the v1 db in place; build/ is dead and can be
deleted. A full rebuild repopulated .build/cas, and the old and new deploy
trees were diffed: byte-identical except the two multi-file hash pointers
(algorithm change), fb/twitter PNG re-encodes (pixel-identical; old outputs
predated current tool versions), and a stale 2023 orphan sprite the old
system never cleaned up.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Christopher Monsanto
2026-08-17 00:46:05 -04:00
parent 499651b3f8
commit f6e0b189f3
23 changed files with 535 additions and 2053 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
Buildfile.ts
View File

@@ -1,200 +0,0 @@
import {base, compresspng, forEachRule, pad, rule, spriteglob, trimimg} from './tools/build/api.ts';
// Generate uniform size minisprites
forEachRule("src/minisprites/pokemon/gen6/*.png", {
display: "pad g6 minisprite %f",
cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})],
}, "build/gen6-minisprites-padded/%b");
forEachRule("src/minisprites/items/*.png", {
display: "pad item minisprite %f",
cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})],
}, "build/item-minisprites-padded/%b");
forEachRule("src/minisprites/pokemon/gen6/*.png", {
display: "trim g6 minisprite %f",
cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
}, "build/gen6-minisprites-trimmed/%b");
forEachRule("src/minisprites/items/*.png", {
display: "trim item minisprite %f",
cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
}, "build/item-minisprites-trimmed/%b");
// Gen 9
forEachRule("src/gen9species/*.png", {
display: "96x96 %f",
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
cmds: [
"magick convert %f -trim +repage -resize 90x90 %o",
"gifsicle -O3 -b %o",
],
}, "build/gen9-modelslike/%B.gif");
// Gen 10
forEachRule("src/champions/*.png", {
display: "96x96 %f",
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
cmds: [
"magick convert %f -trim +repage -resize 90x90 %o",
"gifsicle -O3 -b %o",
],
}, "build/gen10-modelslike/%B.gif");
// Gen 5 CAPs...
forEachRule("src/sprites/gen5/*.png", [
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
"magick convert %f %o",
"gifsicle -O3 -b %o",
], "build/gen5-gif/%B.gif");
// PS spritesheet
rule("ps-pokemon.sheet.mjs", {
display: "ps pokemon sheet",
// The sheet tool reads the minisprites (via readdir, which tup couldn't
// track) and the sprite data; declare them so changes rebuild the sheet.
deps: [
"src/minisprites/pokemon/gen6/*",
"data/species.json",
"data/items.json",
"data/lib/index.ts",
"lib/root/index.ts",
"tools/sheet/index.ts",
],
cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
}, "build/ps/pokemonicons-sheet.png");
// TODO: reenable when trainers are moved
// rule("ps-trainers.sheet.mjs", {
// display: "ps trainers sheet",
// cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
// }, "build/ps/trainers-sheet.png");
rule("ps-items.sheet.mjs", {
display: "ps items sheet",
deps: [
"src/minisprites/items/*",
"data/species.json",
"data/items.json",
"data/lib/index.ts",
"lib/root/index.ts",
"tools/sheet/index.ts",
],
cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
}, "build/ps/itemicons-sheet.png");
// PS pokeball icons
const balls = [
"src/_uncategorized/noncanonical/ui/battle/Ball-Normal.png",
"src/_uncategorized/noncanonical/ui/battle/Ball-Sick.png",
"src/_uncategorized/noncanonical/ui/battle/Ball-Null.png",
];
rule(balls, {
display: "pokemonicons-pokeball-sheet",
cmds: [
"magick convert -background transparent -gravity center -extent 40x30 %f +append %o",
compresspng({config: "SPRITESHEET"}),
],
}, "build/ps/pokemonicons-pokeball-sheet.png");
// Smogdex minisprites (webp)
forEachRule(spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false}), {
display: "webp minisprite %f",
cmds: ["cwebp -z 9 %f -o %o"],
}, "build/smogon/minisprites/%B.webp");
// Smogdex spritesheet
rule(spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false}), {
display: "smogdex sheet",
deps: [
"data/species.json",
"data/items.json",
"data/lib/index.ts",
"lib/root/index.ts",
"tools/smogdexspritesheet/index.ts",
],
// build/smogon/spritesheet.png is an undeclared temporary; it is removed
// before the rule finishes.
cmds: [
"node tools/smogdexspritesheet/index.ts --image build/smogon/spritesheet.png --stylesheet build/smogon/spritesheet.css -- %f",
"cwebp -z 9 build/smogon/spritesheet.png -o build/smogon/spritesheet.webp",
"rm build/smogon/spritesheet.png",
],
}, ["build/smogon/spritesheet.webp", "build/smogon/spritesheet.css"]);
// Smogdex social images
const social = spriteglob(["src/models/*"], {b: false, s: false});
const socialSeen = new Set(social.map(base));
for (const file of spriteglob(["src/gen9species/*"], {b: false, s: false})) {
if (!socialSeen.has(base(file))) {
social.push(file);
socialSeen.add(base(file));
}
}
forEachRule(social, {
display: "fbsprite %f",
cmds: [
'magick convert "%f[0]" -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o',
compresspng({config: "MODELS"}),
],
}, "build/smogon/fbsprites/xy/%B.png");
forEachRule(social, {
display: "twittersprite %f",
cmds: [
'magick convert "%f[0]" -trim -resize 115x115 -background white -gravity center -extent 120x120 %o',
compresspng({config: "MODELS"}),
],
}, "build/smogon/twittersprites/xy/%B.png");
// Trainers
// TODO: reenable when trainers are moved
// forEachRule("src/canonical/trainers/*", {
// display: "pad trainer %f",
// cmds: [pad({w: 80, h: 80}), compresspng({config: "TRAINERS"})],
// }, "build/padded-trainers/canonical/%b");
// Padded Dex
const dexOutput = forEachRule("src/dex/*", {
display: "pad dex %f",
cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})],
}, "build/padded-dex/%b");
// Build missing CAP dex
const dexSet = new Set(dexOutput.map(base));
const dexMissing = [];
for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) {
if (!dexSet.has(base(file))) {
dexMissing.push(file);
dexSet.add(base(file));
}
}
forEachRule(dexMissing, {
display: "missing dex %B",
cmds: [
'magick convert "%f[0]" -trim %o',
'magick mogrify -background transparent -gravity center -resize "120x120>" -extent 120x120 %o',
compresspng({config: "DEX"}),
],
}, "build/padded-dex/%B.png");

36
README.md
View File

@@ -46,21 +46,33 @@ Using [`brew`](https://brew.sh/) on a macOS:
$ brew install imagemagick gifsicle advancecomp optipng pngquant webp
```
## Building
## Building and deploying
Install dependencies once with `pnpm install`. Then, to build:
Install dependencies once with `pnpm install`.
Each deploy is a root `*.deploy.ts` module: it declares its build rules
(shared sets live in `rules/`) and a `finish` function that maps the built
artifacts to their published names. Build outputs are content-addressed:
rules declare nominal output filenames but the store names every object by
the hash of its bytes (under `.build/cas/`), so incrementality keys on
content, same-byte renames rebuild nothing, and hash-stamped publishing
reuses the build's digests. All state lives in `.build/`.
```
$ pnpm build
$ pnpm build # build every deploy's rules, GC stale state
$ pnpm deploy # assets.deploy.ts -> tar -> DEPLOY_COMMAND (.env)
$ node tools/deploy/index.ts build ps.deploy.ts # build one deploy's rules
$ node tools/deploy/index.ts run smogon.deploy.ts -o deploy/smogon
$ node tools/deploy/index.ts inspect src/minisprites/items/i1.png -o /tmp/out
```
The rules live in `Buildfile.ts`. Build state (content hashes, rule records)
is kept in `.build/`; outputs of removed rules are deleted automatically, and
renamed sources are detected and their outputs copied instead of rebuilt.
`run` materializes a deploy to a directory (`--link` hardlinks, `--tar`
writes a tar file) without uploading anything. `inspect` builds every rule
that consumes the given source paths (and their transitive consumers) and
copies the outputs out under readable names for eyeballing.
Useful flags: `-j <n>` parallelism, `-n` dry run, `-v` verbose,
`--adopt` record already-existing `build/` outputs as up to date instead of
rebuilding them (useful when `build/` was produced elsewhere).
`--fail-fast` stop after the first failure.
## Configuration
@@ -88,7 +100,13 @@ DEFAULT_ADVPNG=-z4 -i5000
- The build tool only tracks the inputs a rule declares. If a build tool reads
files that aren't on its command line (e.g. it does a `readdir()`), declare
them with the rule's `deps:` in `Buildfile.ts` so changes are detected.
them with the rule's `deps:` so changes are detected.
- Rule identity is content-only by default: renaming a source without
changing its bytes rebuilds nothing. If a tool bakes input *names* into
its output bytes (the spritesheet builders do), the rule must set
`nameSensitive: true` or renames will leave its output silently stale.
- Rules must be declared when a deploy module is imported (top level), not
inside `finish` — the build runs before finish does.
## License

149
assets.deploy.js
View File

@@ -1,149 +0,0 @@
// The upload contract wants __key first in the tar, naming the asset set;
// the tar packer emits ops in queue order, so it has to be the first op.
write("__key", "sprites");
function toSmogonAlias(name) {
return name.toLowerCase().
replace(/[ _]+/, "-").
replace(/[^a-z0-9-]+/g, '');
}
function toPSID(name) {
return name.toLowerCase().replace(/[^a-z0-9]+/g, '');
}
// Copy with a content-hash-stamped name and record the unhashed -> hashed
// mapping in `manifest`.
function stampcopy(f, {dir, ext, name}, manifest) {
const key = `${name}.${ext ?? f.ext}`;
// ActionQueue only dedups final dsts; hashed dsts differ even when
// unhashed names collide, so check the manifest key explicitly.
if (manifest[key] !== undefined) {
throw new Error(`duplicate sprite name ${key}`);
}
const h = hash(f);
manifest[key] = `${name}-${h}.${ext ?? f.ext}`;
copy(f, {dir, ext, name: `${name}-${h}`});
}
function writeManifest(dst, manifest) {
const sorted = {};
for (const k of Object.keys(manifest).sort()) {
sorted[k] = manifest[k];
}
write(dst, JSON.stringify(sorted, null, 4) + "\n");
}
function spritecopy(f, {dir, ext}, allowUnknown=false, manifest=null) {
const sn = spritedata.parseFilename(f.name);
let name;
// Skip asymmetrical for now
if (sn.extra.has("a") || sn.extra.has("b") || sn.extra.has("s")) {
return;
}
if (sn.extension) {
if (allowUnknown && sn.extension && sn.name === "Unknown") {
name = "unknown"
} else {
// Skip this, we don't use Unknown/Substitute
return;
}
} else {
const sd = spritedata.get(sn.id);
name = toSmogonAlias(sd.base);
if (sd.forme) {
name += `-${toSmogonAlias(sd.forme)}`;
}
}
if (sn.extra.has("f")) {
name += "-f";
}
if (sn.extra.has("g")) {
name += "-gmax";
}
if (manifest) {
stampcopy(f, {dir, ext, name}, manifest);
} else {
copy(f, {dir, ext, name});
}
}
// TODO: merge with above
function itemspritecopy(f, {dir, ext}, manifest=null) {
const sn = spritedata.parseFilename(f.name);
const sd = spritedata.get(sn.id);
for (const n of sd.names) {
const name = toSmogonAlias(n);
if (manifest) {
stampcopy(f, {dir, ext, name}, manifest);
} else {
copy(f, {dir, ext, name});
}
}
}
function newspritecopy(f, {dir, ext}) {
const sn = spritedata.parseFilename(f.name);
if (sn.extension) {
return
}
const sd = spritedata.get(sn.id);
for (const n of sd.type === 'item' ? sd.names : [sd.base + sd.forme]) {
let name = toPSID(n);
if (sn.extra.has("f")) {
name += "f";
}
if (sn.extra.has("g")) {
name += "gmax";
}
copy(f, {dir, ext, name});
}
}
// Dex spritesheet assets: hash-stamped css + webp. The css suffix pointer
// rides in __meta/ for the dex to read.
{
const wh = hash("build/smogon/spritesheet.webp");
copy("build/smogon/spritesheet.webp", `spritesheet-${wh}.webp`);
const src = read("build/smogon/spritesheet.css");
const css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`);
if (css === src) {
throw new Error("spritesheet.css: no webp urls rewritten");
}
// Suffix from source content: the rewritten css is a pure function of
// (css, webp), so this changes exactly when the served bytes change.
const ch = hash("build/smogon/spritesheet.css", "build/smogon/spritesheet.webp");
write(`spritesheet-${ch}.css`, css);
write("__meta/spritesheet_css_suffix.txt", `-${ch}\n`);
}
{
const h = hash(...list("build/smogon/minisprites"));
for (const f of list("build/smogon/minisprites")) {
newspritecopy(f, {dir: "minisprites/" + h});
}
write("__meta/minisprites-hash.txt", h);
}
{
const manifest = {};
for (const f of list("build/item-minisprites-padded")) {
itemspritecopy(f, {dir: "forumsprites"}, manifest);
}
for (const f of list("build/gen6-minisprites-padded")) {
spritecopy(f, {dir: "forumsprites"}, true, manifest);
}
writeManifest("__meta/forumsprites/manifest.json", manifest);
}
{
const manifest = {};
for (const f of list("src/pmd")) {
spritecopy(f, {dir: "pmd"}, false, manifest);
}
writeManifest("__meta/pmd/manifest.json", manifest);
}

4
package.json
View File

@@ -1,8 +1,8 @@
{
"type": "module",
"scripts": {
"build": "node tools/build/index.ts",
"deploy": "node tools/deploy/index.ts deploy assets.deploy.js",
"build": "node tools/deploy/index.ts build",
"deploy": "node tools/deploy/index.ts deploy assets.deploy.ts",
"check": "tsc --build tsconfig-workspace.json"
},
"dependencies": {

120
ps.deploy.js
View File

@@ -1,120 +0,0 @@
// function toID(name) {
// return name.toLowerCase().replace(/[^a-z0-9]+/g, '');
// }
// function spritecopy(f, {dir, ext}) {
// const sn = spritedata.parseFilename(f.name);
// let name;
// if (sn.extension) {
// name = toID(sn.name);
// } else {
// const sd = spritedata.get(sn.id);
// debug(sd);
// name = toID(sd.base);
// if (sd.forme) {
// name += `-${toID(sd.forme)}`;
// }
// }
// if (sn.extra.has("f")) {
// name += "-f";
// }
// if (sn.extra.has("g")) {
// name += "-gmax";
// }
// if (sn.extra.has("b")) {
// dir += "-back";
// }
// if (sn.extra.has("s")) {
// dir += "-shiny";
// }
// copy(f, {dir, ext, name});
// }
function toPSID(name) {
return name.toLowerCase().replace(/[^a-z0-9]+/g, '');
}
function spritecopy(f, {dir, ext}, allowUnknown=false) {
const sn = spritedata.parseFilename(f.name);
let name;
// Skip asymmetrical for now
if (sn.extra.has("a") || sn.extra.has("b") || sn.extra.has("s")) {
return;
}
if (sn.extension) {
if (allowUnknown && sn.extension && sn.name === "Unknown") {
name = "unknown"
} else {
// Skip this, we don't use Unknown/Substitute
return;
}
} else {
const sd = spritedata.get(sn.id);
name = toPSID(sd.base);
if (sd.forme) {
name += `-${toPSID(sd.forme)}`;
}
}
if (sn.extra.has("f")) {
name += "-f";
}
if (sn.extra.has("g")) {
name += "-gmax";
}
copy(f, {dir, ext, name});
}
let seenModels = new Set;
for (const f of list("src/models")) {
seenModels.add(f.name);
spritecopy(f, {dir: "ani"});
}
for (const f of list("build/gen10-modelslike")) {
if (seenModels.has(f.name)) continue;
seenModels.add(f.name);
spritecopy(f, {dir: "ani"});
}
// for (const f of list("src/sprites/gen5")) {
// spritecopy(f, {dir: "gen5ani"});
// }
// for (const f of list("src/afd")) {
// spritecopy(f, {dir: "afd"});
// }
// for (const f of list("build/padded-dex")) {
// spritecopy(f, {dir: "dex"});
// }
// function fixType(name) {
// return name.replace("Unknown", "???");
// }
// for (const f of list("src/canonical/ui/types/gen4").concat(list("src/noncanonical/ui/types/gen4"))) {
// copy(f, {dir: "types", name: fixType(f.name)});
// }
// for (const f of list("src/canonical/ui/categories/gen4")) {
// copy(f, {dir: "categories"});
// }
// copy("src/noncanonical/ui/categories/undefined.png", {dir: "categories"});
// copy("src/canonical/ui/battle/Alpha.png", {dir: "misc"});
// copy("src/canonical/ui/battle/Mega.png", {dir: "misc"});
// copy("src/canonical/ui/battle/Omega.png", {dir: "misc"});
// // TODO: reenable when trainers are moved
// // dest("trainers");
// // sel("build/padded-trainers/canonical");
// copy("build/ps/pokemonicons-pokeball-sheet.png", {dir: "."});
// copy("build/ps/pokemonicons-sheet.png", {dir: "."});
// //copy("build/ps/trainers-sheet.png", {dir: "."});
// copy("build/ps/itemicons-sheet.png", {dir: "."});

134
smogon.deploy.js
View File

@@ -1,134 +0,0 @@
function toSmogonAlias(name) {
return name.toLowerCase().
replace(/[ _]+/, "-").
replace(/[^a-z0-9-]+/g, '');
}
// Copy with a content-hash-stamped name and record the unhashed -> hashed
// mapping in `manifest`.
function stampcopy(f, {dir, ext, name}, manifest) {
const key = `${name}.${ext ?? f.ext}`;
// ActionQueue only dedups final dsts; hashed dsts differ even when
// unhashed names collide, so check the manifest key explicitly.
if (manifest[key] !== undefined) {
throw new Error(`duplicate sprite name ${key}`);
}
const h = hash(f);
manifest[key] = `${name}-${h}.${ext ?? f.ext}`;
copy(f, {dir, ext, name: `${name}-${h}`});
}
function writeManifest(dst, manifest) {
const sorted = {};
for (const k of Object.keys(manifest).sort()) {
sorted[k] = manifest[k];
}
write(dst, JSON.stringify(sorted, null, 4) + "\n");
}
function spritecopy(f, {dir, ext}, allowUnknown=false, manifest=null) {
const sn = spritedata.parseFilename(f.name);
let name;
// Skip asymmetrical for now
if (sn.extra.has("a") || sn.extra.has("b") || sn.extra.has("s")) {
return;
}
if (sn.extension) {
if (allowUnknown && sn.extension && sn.name === "Unknown") {
name = "unknown"
} else {
// Skip this, we don't use Unknown/Substitute
return;
}
} else {
const sd = spritedata.get(sn.id);
name = toSmogonAlias(sd.base);
if (sd.forme) {
name += `-${toSmogonAlias(sd.forme)}`;
}
}
if (sn.extra.has("f")) {
name += "-f";
}
if (sn.extra.has("g")) {
name += "-gmax";
}
if (manifest) {
stampcopy(f, {dir, ext, name}, manifest);
} else {
copy(f, {dir, ext, name});
}
}
// TODO: merge with above
function itemspritecopy(f, {dir, ext}, manifest=null) {
const sn = spritedata.parseFilename(f.name);
const sd = spritedata.get(sn.id);
for (const n of sd.names) {
const name = toSmogonAlias(n);
if (manifest) {
stampcopy(f, {dir, ext, name}, manifest);
} else {
copy(f, {dir, ext, name});
}
}
}
let seenModels = new Set;
const xyManifest = {};
for (const f of list("src/models")) {
seenModels.add(f.name);
spritecopy(f, {dir: "xy"}, false, xyManifest);
}
for (const f of list("build/gen9-modelslike")) {
if (seenModels.has(f.name)) continue;
seenModels.add(f.name);
spritecopy(f, {dir: "xy"}, false, xyManifest);
}
for (const f of list("build/gen10-modelslike")) {
if (seenModels.has(f.name)) continue;
seenModels.add(f.name);
spritecopy(f, {dir: "xy"}, false, xyManifest);
}
// Non-model CAPs
for (const f of list("src/sprites/gen5")) {
if (f.ext !== 'gif' || seenModels.has(f.name)) continue;
seenModels.add(f.name);
spritecopy(f, {dir: "xy"}, false, xyManifest);
}
for (const f of list("build/gen5-gif")) {
if (seenModels.has(f.name)) continue;
seenModels.add(f.name);
spritecopy(f, {dir: "xy"}, false, xyManifest);
}
writeManifest("xy/manifest.json", xyManifest);
{
const manifest = {};
for (const f of list("build/gen6-minisprites-trimmed")) {
spritecopy(f, {dir: "xyicons"}, false, manifest);
}
writeManifest("xyicons/manifest.json", manifest);
}
for (const f of list("build/item-minisprites-trimmed")) {
itemspritecopy(f, {dir: "xyitems"});
}
for (const f of list("build/smogon/fbsprites/xy")) {
spritecopy(f, {dir: "fbsprites/xy"});
}
for (const f of list("build/smogon/twittersprites/xy")) {
spritecopy(f, {dir: "twittersprites/xy"});
}

110
tools/build/api.ts
View File

@@ -1,110 +0,0 @@
import pathlib from 'path';
import {createHash} from 'crypto';
import {astable, glob} from './helpers.ts';
import {type Cmd, flattenCmds, substitute} from './subst.ts';
export type {Cmd};
export {base, compresspng, getconfig, glob, pad, setConfig,
spritedata, spriteglob, trimimg, type SpriteData} from './helpers.ts';
export interface CmdSpec {
display? : string;
// Tracked-but-not-substituted inputs: hashed and part of rule identity,
// but never expanded into %f. Use for files a tool reads on its own
// (e.g. tools/sheet readdirs the minisprite directories).
deps? : string | string[];
cmds : Cmd[];
}
export interface RuleDecl {
inputs : string[]; // %f source, ordered
deps : string[]; // hashed, never substituted
outputs : string[]; // post-substitution paths
command : string; // final ' && '-joined shell command
display : string | null; // post-substitution; cosmetic, not part of identity
template : string; // pre-substitution cmds + output basename template(s)
key : string; // identity for incremental skip
}
let rules : RuleDecl[] = [];
export function getRules() : RuleDecl[] {
return rules;
}
export function resetRules() : void {
rules = [];
}
function normalizeSpec(spec : CmdSpec | Cmd[]) : CmdSpec {
return Array.isArray(spec) ? {cmds: spec} : spec;
}
// The rename-detection template deliberately excludes output directories
// (basename only) so that content-preserving moves across directories with
// identical processing still match. Input extensions are included because
// tools like magick pick their output format from file extensions, so an
// extension-only rename must not match.
function templateOf(cmds : string[], outputTemplates : string[], inputs : string[]) : string {
return [
cmds.join('\n'),
outputTemplates.map(t => pathlib.basename(t)).join('\0'),
inputs.map(p => pathlib.extname(p)).join('\0'),
].join('\x01');
}
function keyOf(command : string, inputs : string[], deps : string[], outputs : string[]) : string {
const h = createHash('sha256');
h.update([command, inputs.join('\0'), deps.join('\0'), outputs.join('\0')].join('\x01'));
return h.digest('hex');
}
function makeRule(inputs : string[], deps : string[], spec : CmdSpec,
outputs : string[], outputTemplates : string[]) : RuleDecl {
const cmds = flattenCmds(spec.cmds).map(c => substitute(c, inputs, outputs));
if (cmds.length === 0) {
throw new Error(`Rule with no commands (outputs: ${outputs.join(' ')})`);
}
const command = cmds.join(' && ');
const decl : RuleDecl = {
inputs,
deps,
outputs,
command,
display: spec.display !== undefined ? substitute(spec.display, inputs, outputs) : null,
template: templateOf(flattenCmds(spec.cmds), outputTemplates, inputs),
key: keyOf(command, inputs, deps, outputs),
};
rules.push(decl);
return decl;
}
export function rule(input : string | string[], spec : CmdSpec | Cmd[],
output : string | string[]) : string[] {
const s = normalizeSpec(spec);
const outputs = astable(output);
for (const out of outputs) {
if (out.includes('%')) {
throw new Error(`rule() outputs are literal paths, no substitutions: ${out}`);
}
}
const decl = makeRule(glob(input), glob(astable(s.deps)), s, outputs, outputs);
return decl.outputs;
}
export function forEachRule(input : string | string[], spec : CmdSpec | Cmd[],
output : string) : string[] {
const s = normalizeSpec(spec);
if (/%[fo]/.test(output)) {
throw new Error(`forEachRule output template may only use %b/%B: ${output}`);
}
const deps = glob(astable(s.deps));
const outputs = [];
for (const file of glob(input)) {
const decl = makeRule([file], deps, s, [substitute(output, [file], [])], [output]);
outputs.push(...decl.outputs);
}
return outputs;
}

256
tools/build/db.ts
View File

@@ -1,256 +0,0 @@
import fs from 'fs';
import pathlib from 'path';
import Database from 'better-sqlite3';
import type {RuleDecl} from './api.ts';
import type {FileStat} from './hash.ts';
import {BuildError} from './graph.ts';
export interface StoredRuleInput {
path : string;
isDep : boolean;
hash : Buffer;
}
export interface StoredRuleOutput {
path : string;
size : bigint | null; // null when the rule last failed
mtimeNs : bigint | null;
}
export interface StoredRule {
id : bigint;
key : string;
command : string;
display : string | null;
template : string;
inputSig : Buffer;
ok : boolean;
inputs : StoredRuleInput[]; // ordered: inputs (in %f order), then deps
outputs : StoredRuleOutput[]; // ordered: position-mapped for renames
}
export interface RecordedOutput {
path : string;
size : bigint;
mtimeNs : bigint;
}
const DDL = `
CREATE TABLE IF NOT EXISTS file_cache (
path TEXT PRIMARY KEY,
size INTEGER NOT NULL,
mtime_ns INTEGER NOT NULL,
hash BLOB NOT NULL
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS rules (
id INTEGER PRIMARY KEY,
key TEXT NOT NULL UNIQUE,
command TEXT NOT NULL,
display TEXT,
template TEXT NOT NULL,
input_sig BLOB NOT NULL,
ok INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS rules_rename ON rules(template, input_sig);
CREATE TABLE IF NOT EXISTS rule_inputs (
rule_id INTEGER NOT NULL REFERENCES rules(id) ON DELETE CASCADE,
ord INTEGER NOT NULL,
path TEXT NOT NULL,
is_dep INTEGER NOT NULL DEFAULT 0,
hash BLOB NOT NULL,
PRIMARY KEY (rule_id, ord)
);
CREATE INDEX IF NOT EXISTS rule_inputs_path ON rule_inputs(path);
CREATE TABLE IF NOT EXISTS rule_outputs (
rule_id INTEGER NOT NULL REFERENCES rules(id) ON DELETE CASCADE,
ord INTEGER NOT NULL,
path TEXT NOT NULL,
size INTEGER,
mtime_ns INTEGER,
PRIMARY KEY (rule_id, ord)
);
CREATE INDEX IF NOT EXISTS rule_outputs_path ON rule_outputs(path);
`;
export class BuildDb {
private db : Database.Database;
constructor(dbPath : string) {
fs.mkdirSync(pathlib.dirname(dbPath), {recursive: true});
this.db = new Database(dbPath);
this.db.defaultSafeIntegers(true);
this.db.pragma('journal_mode = WAL');
this.db.pragma('foreign_keys = ON');
this.db.pragma('synchronous = NORMAL');
this.migrate();
}
private migrate() : void {
const version = Number(this.db.pragma('user_version', {simple: true}));
if (version === 0) {
this.db.exec('BEGIN;' + DDL + 'PRAGMA user_version = 1; COMMIT;');
} else if (version !== 1) {
throw new BuildError(
`Unknown build db schema version ${version}; delete .build/ and re-run with --adopt`);
}
}
loadFileCache() : Map<string, FileStat> {
const result = new Map<string, FileStat>();
const rows = this.db.prepare<[], {path : string, size : bigint, mtime_ns : bigint, hash : Buffer}>(
'SELECT path, size, mtime_ns, hash FROM file_cache').all();
for (const row of rows) {
result.set(row.path, {size: row.size, mtimeNs: row.mtime_ns, hash: row.hash});
}
return result;
}
saveFileCache(entries : Map<string, FileStat>) : void {
const upsert = this.db.prepare(`
INSERT INTO file_cache (path, size, mtime_ns, hash) VALUES (?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET
size = excluded.size, mtime_ns = excluded.mtime_ns, hash = excluded.hash`);
this.db.transaction(() => {
for (const [path, stat] of entries) {
upsert.run(path, stat.size, stat.mtimeNs, stat.hash);
}
})();
}
pruneFileCache(live : Set<string>) : void {
const paths = this.db.prepare<[], {path : string}>('SELECT path FROM file_cache').all();
const del = this.db.prepare('DELETE FROM file_cache WHERE path = ?');
this.db.transaction(() => {
for (const {path} of paths) {
if (!live.has(path)) {
del.run(path);
}
}
})();
}
loadStoredRules() : StoredRule[] {
// One transaction: the three queries must see a single snapshot, or a
// concurrent writer (e.g. a build racing a dry run) tears the view.
return this.db.transaction(() => this.loadStoredRulesInner())();
}
private loadStoredRulesInner() : StoredRule[] {
const byId = new Map<bigint, StoredRule>();
const ruleRows = this.db.prepare<[], {
id : bigint, key : string, command : string, display : string | null,
template : string, input_sig : Buffer, ok : bigint,
}>('SELECT id, key, command, display, template, input_sig, ok FROM rules').all();
for (const row of ruleRows) {
byId.set(row.id, {
id: row.id,
key: row.key,
command: row.command,
display: row.display,
template: row.template,
inputSig: row.input_sig,
ok: row.ok !== 0n,
inputs: [],
outputs: [],
});
}
const inputRows = this.db.prepare<[], {rule_id : bigint, path : string, is_dep : bigint, hash : Buffer}>(
'SELECT rule_id, path, is_dep, hash FROM rule_inputs ORDER BY rule_id, ord').all();
for (const row of inputRows) {
byId.get(row.rule_id)!.inputs.push({path: row.path, isDep: row.is_dep !== 0n, hash: row.hash});
}
const outputRows = this.db.prepare<[], {rule_id : bigint, path : string, size : bigint | null, mtime_ns : bigint | null}>(
'SELECT rule_id, path, size, mtime_ns FROM rule_outputs ORDER BY rule_id, ord').all();
for (const row of outputRows) {
byId.get(row.rule_id)!.outputs.push({path: row.path, size: row.size, mtimeNs: row.mtime_ns});
}
return [...byId.values()];
}
// One transaction per completed rule: an interrupted build only ever
// contains fully-recorded rules. outputs === null records a failure (ok=0,
// output paths kept for GC, stats nulled so the rule stays dirty).
recordRuleResult(decl : RuleDecl, inputHashes : Map<string, Buffer>, sig : Buffer,
outputs : RecordedOutput[] | null) : void {
const ok = outputs !== null;
const upsert = this.db.prepare<[string, string, string | null, string, Buffer, bigint], {id : bigint}>(`
INSERT INTO rules (key, command, display, template, input_sig, ok)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(key) DO UPDATE SET
display = excluded.display, input_sig = excluded.input_sig, ok = excluded.ok
RETURNING id`);
const delInputs = this.db.prepare('DELETE FROM rule_inputs WHERE rule_id = ?');
const delOutputs = this.db.prepare('DELETE FROM rule_outputs WHERE rule_id = ?');
const insInput = this.db.prepare(
'INSERT INTO rule_inputs (rule_id, ord, path, is_dep, hash) VALUES (?, ?, ?, ?, ?)');
const insOutput = this.db.prepare(
'INSERT INTO rule_outputs (rule_id, ord, path, size, mtime_ns) VALUES (?, ?, ?, ?, ?)');
this.db.transaction(() => {
const {id} = upsert.get(decl.key, decl.command, decl.display, decl.template,
sig, ok ? 1n : 0n)!;
delInputs.run(id);
delOutputs.run(id);
let ord = 0;
for (const path of decl.inputs) {
insInput.run(id, ord++, path, 0, inputHashes.get(path)!);
}
for (const path of decl.deps) {
insInput.run(id, ord++, path, 1, inputHashes.get(path)!);
}
const outputRows = outputs ?? decl.outputs.map(path => ({path, size: null, mtimeNs: null}));
outputRows.forEach((o, i) => insOutput.run(id, i, o.path, o.size, o.mtimeNs));
})();
}
deleteRule(id : bigint) : void {
this.db.prepare('DELETE FROM rules WHERE id = ?').run(id);
}
// template/display are pure functions of the rule declaration but are not
// part of the rule key; refresh stored values that have drifted (e.g. the
// template format changed in a newer version of this tool), otherwise
// rename detection quietly stops matching older records.
refreshRuleMeta(entries : {id : bigint, template : string, display : string | null}[]) : void {
if (entries.length === 0) {
return;
}
const update = this.db.prepare('UPDATE rules SET template = ?, display = ? WHERE id = ?');
this.db.transaction(() => {
for (const e of entries) {
update.run(e.template, e.display, e.id);
}
})();
}
close() : void {
this.db.pragma('wal_checkpoint(TRUNCATE)');
this.db.close();
}
}
// Guards against two concurrent builds; the exclusive transaction is released
// by the OS on any crash, so there are no stale lock files.
export function acquireLock(lockPath : string) : () => void {
fs.mkdirSync(pathlib.dirname(lockPath), {recursive: true});
const lock = new Database(lockPath, {timeout: 0});
try {
lock.exec('BEGIN EXCLUSIVE');
} catch (err) {
lock.close();
if ((err as {code? : string}).code === 'SQLITE_BUSY') {
throw new BuildError('Another build is already running.');
}
throw err;
}
return () => {
try {
lock.exec('COMMIT');
} catch {}
lock.close();
};
}

22
tools/build/exec.ts
View File

@@ -79,25 +79,3 @@ export function runShell(command : string, opts : {cwd : string, signal : AbortS
});
}
// A worker that throws stops its own loop, but the pool always waits for
// every other worker to finish before rethrowing: failing fast here would
// return control (and e.g. close the database) while rules are still running.
export async function workerPool<T>(items : readonly T[], jobs : number,
fn : (item : T, index : number) => Promise<void>) : Promise<void> {
let next = 0;
const workers = [];
for (let i = 0; i < Math.max(1, Math.min(jobs, items.length)); i++) {
workers.push((async () => {
while (next < items.length) {
const index = next++;
await fn(items[index]!, index);
}
})());
}
const results = await Promise.allSettled(workers);
for (const result of results) {
if (result.status === 'rejected') {
throw result.reason;
}
}
}

63
tools/build/graph.ts
View File

@@ -1,63 +0,0 @@
import type {RuleDecl} from './api.ts';
import {BuildError} from './errors.ts';
export {BuildError};
export interface GraphResult {
order : RuleDecl[]; // topological, stable w.r.t. declaration order
generated : Set<string>; // every path produced by some rule
}
export function checkGraph(rules : RuleDecl[]) : GraphResult {
const owner = new Map<string, RuleDecl>();
for (const rule of rules) {
for (const output of rule.outputs) {
const other = owner.get(output);
if (other !== undefined) {
throw new BuildError(
`Output ${output} produced by multiple rules:\n ${other.command}\n ${rule.command}`);
}
owner.set(output, rule);
}
}
const consumers = new Map<RuleDecl, RuleDecl[]>();
const indegree = new Map<RuleDecl, number>();
for (const rule of rules) {
indegree.set(rule, 0);
}
for (const rule of rules) {
for (const input of [...rule.inputs, ...rule.deps]) {
const producer = owner.get(input);
if (producer !== undefined && producer !== rule) {
let list = consumers.get(producer);
if (list === undefined) {
consumers.set(producer, list = []);
}
list.push(rule);
indegree.set(rule, indegree.get(rule)! + 1);
}
}
}
const queue = rules.filter(r => indegree.get(r) === 0);
const order = [];
for (let i = 0; i < queue.length; i++) {
const rule = queue[i]!;
order.push(rule);
for (const consumer of consumers.get(rule) ?? []) {
const deg = indegree.get(consumer)! - 1;
indegree.set(consumer, deg);
if (deg === 0) {
queue.push(consumer);
}
}
}
if (order.length !== rules.length) {
const stuck = rules.filter(r => indegree.get(r)! > 0);
throw new BuildError(`Dependency cycle involving:\n ${stuck.map(r => r.command).join('\n ')}`);
}
return {order, generated: new Set(owner.keys())};
}

355
tools/build/index.ts
View File

@@ -1,355 +0,0 @@
import fs from 'fs';
import os from 'os';
import pathlib from 'path';
import {fileURLToPath, pathToFileURL} from 'url';
import {program} from 'commander';
import debugfn from 'debug';
import {getRules, type RuleDecl, setConfig} from './api.ts';
import {loadConfig} from './config.ts';
import {acquireLock, BuildDb, type RecordedOutput} from './db.ts';
import {killAllProcessGroups, runShell, workerPool} from './exec.ts';
import {BuildError, checkGraph} from './graph.ts';
import {reconcileHashes} from './hash.ts';
import {computePlan, type OutputStat, ruleInputSig} from './plan.ts';
const debug = debugfn('build');
program
.option('-j, --jobs <n>', 'number of parallel jobs', String(os.availableParallelism()))
.option('-n, --dry-run', 'print the plan without changing anything')
.option('--adopt', 'record existing outputs as up to date instead of running (migration)')
.option('--fail-fast', 'stop scheduling new rules after the first failure')
.option('--config <file>', 'config file', 'build.config')
.option('-v, --verbose', 'print more detail');
program.parse(process.argv);
const opts = program.opts();
const root = pathlib.resolve(fileURLToPath(import.meta.url), '../../..');
process.chdir(root);
function statPath(path : string) : OutputStat | null {
try {
const st = fs.statSync(path, {bigint: true});
return st.isFile() ? {size: st.size, mtimeNs: st.mtimeNs} : null;
} catch {
return null;
}
}
function label(decl : RuleDecl) : string {
return decl.display ?? decl.command.split(' && ')[0]!;
}
function indent(text : string) : string {
return text.replace(/\n$/, '').split('\n').map(l => ' ' + l).join('\n');
}
async function main() : Promise<number> {
const jobs = Number(opts.jobs);
if (!Number.isInteger(jobs) || jobs < 1) {
throw new BuildError(`Invalid --jobs value: ${opts.jobs}`);
}
const dryRun = Boolean(opts.dryRun);
const releaseLock = dryRun ? null : acquireLock('.build/lock.sqlite');
// A dry run must not create state; without an existing db it reads from
// an empty in-memory one.
const dbPath = dryRun && !fs.existsSync('.build/db.sqlite') ? ':memory:' : '.build/db.sqlite';
const db = new BuildDb(dbPath);
try {
// Phase 1: evaluate the rule set
setConfig(loadConfig(opts.config));
await import(pathToFileURL(pathlib.join(root, 'Buildfile.ts')).href);
const rules = getRules();
const {order, generated} = checkGraph(rules);
for (const rule of rules) {
for (const path of [...rule.inputs, ...rule.deps]) {
if (generated.has(path)) {
// The planner assumes all inputs are hashable before
// execution; support this when a rule needs it.
throw new BuildError(`Rules consuming generated files are not yet supported: ${path}`);
}
}
}
debug('%d rules', rules.length);
// Phase 2: hash source files (stat-cached)
const sources = new Set<string>();
for (const rule of rules) {
for (const path of [...rule.inputs, ...rule.deps]) {
sources.add(path);
}
}
const {hashes, updated, missing} = reconcileHashes(sources, db.loadFileCache());
if (missing.length > 0) {
throw new BuildError(`Missing input files:\n ${missing.slice(0, 20).join('\n ')}`
+ (missing.length > 20 ? `\n ... and ${missing.length - 20} more` : ''));
}
if (!dryRun && updated.size > 0) {
db.saveFileCache(updated);
}
debug('hashed %d files (%d cached)', sources.size, sources.size - updated.size);
// Phase 3: plan
const statMemo = new Map<string, OutputStat | null>();
const statOutput = (path : string) => {
let st = statMemo.get(path);
if (st === undefined) {
statMemo.set(path, st = statPath(path));
}
return st;
};
const stored = db.loadStoredRules();
const plan = computePlan({
current: rules,
stored,
hashes,
statOutput,
adopt: Boolean(opts.adopt),
});
// Keep stored template/display in sync for key-matched rules; they are
// outside the key, so e.g. a template format change in this tool would
// otherwise silently disable rename detection for old records.
if (!dryRun) {
const storedByKey = new Map(stored.map(s => [s.key, s]));
const staleMeta = [];
for (const decl of rules) {
const s = storedByKey.get(decl.key);
if (s !== undefined && (s.template !== decl.template || s.display !== decl.display)) {
staleMeta.push({id: s.id, template: decl.template, display: decl.display});
}
}
db.refreshRuleMeta(staleMeta);
}
const staleOutputs = [];
const currentOutputs = new Set(rules.flatMap(r => r.outputs));
for (const s of plan.stale) {
for (const o of s.outputs) {
if (!currentOutputs.has(o.path)) {
staleOutputs.push(o.path);
}
}
}
if (plan.run.length + plan.renames.length + plan.adopt.length + plan.stale.length === 0) {
console.log(`${plan.clean.length} rules up to date.`);
return 0;
}
if (opts.verbose || dryRun) {
for (const {decl, reason} of plan.run) {
console.log(`run (${reason}): ${label(decl)}`);
}
for (const {decl, from} of plan.renames) {
console.log(`rename: ${from.outputs.map(o => o.path).join(' ')} -> ${decl.outputs.join(' ')}`);
}
for (const decl of plan.adopt) {
console.log(`adopt: ${label(decl)}`);
}
for (const path of staleOutputs) {
console.log(`delete: ${path}`);
}
}
if (dryRun) {
console.log(`would run ${plan.run.length}, rename ${plan.renames.length}, `
+ `adopt ${plan.adopt.length}, delete ${staleOutputs.length} outputs `
+ `(${plan.clean.length} up to date)`);
return 0;
}
// Phase 4: renames (before stale deletion: sources must still exist)
let renamed = 0;
for (const {decl, from} of plan.renames) {
// An earlier copy in this loop may have overwritten this rename's
// source (rename destinations can collide with rename sources);
// re-verify every source against its recorded stat before copying
// so only verified bytes ever propagate.
const intact = from.outputs.every(o => {
const st = statPath(o.path);
return o.size !== null && st !== null
&& st.size === o.size && st.mtimeNs === o.mtimeNs;
});
if (!intact) {
plan.run.push({decl, reason: 'new'});
continue;
}
const recorded : RecordedOutput[] = [];
for (let i = 0; i < decl.outputs.length; i++) {
const src = from.outputs[i]!.path;
const dst = decl.outputs[i]!;
if (src !== dst) {
fs.mkdirSync(pathlib.dirname(dst), {recursive: true});
fs.copyFileSync(src, dst);
}
const st = statPath(dst);
if (st === null) {
throw new BuildError(`Rename copy failed: ${src} -> ${dst}`);
}
recorded.push({path: dst, size: st.size, mtimeNs: st.mtimeNs});
}
db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded);
renamed++;
}
// Phase 5: delete outputs of removed rules, prune empty dirs
const staleDirs = new Set<string>();
for (const s of plan.stale) {
for (const o of s.outputs) {
if (!currentOutputs.has(o.path)) {
try {
fs.unlinkSync(o.path);
} catch (err) {
if ((err as {code? : string}).code !== 'ENOENT') {
throw err;
}
}
staleDirs.add(pathlib.dirname(o.path));
}
}
db.deleteRule(s.id);
}
for (let dir of staleDirs) {
while (dir.startsWith('build/')) {
try {
fs.rmdirSync(dir);
} catch {
break;
}
dir = pathlib.dirname(dir);
}
}
db.pruneFileCache(sources);
// Phase 6: adoption (migration): trust existing outputs
for (const decl of plan.adopt) {
const recorded = decl.outputs.map(path => {
const st = statPath(path)!;
return {path, size: st.size, mtimeNs: st.mtimeNs};
});
db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded);
}
// Phase 7: execute
// The worker pool does not serialize producers before consumers; that
// is safe because rules consuming generated files are rejected above.
// `order` is used for a stable, declaration-ordered schedule.
const orderIndex = new Map(order.map((r, i) => [r, i]));
const runList = [...plan.run].sort((a, b) => orderIndex.get(a.decl)! - orderIndex.get(b.decl)!);
const ac = new AbortController();
let interrupted = false;
const onSignal = () => {
if (interrupted) {
killAllProcessGroups();
process.exit(130);
}
interrupted = true;
console.error('\nInterrupted; waiting for running rules to stop...');
ac.abort();
};
process.on('SIGINT', onSignal);
process.on('SIGTERM', onSignal);
const failures : RuleDecl[] = [];
let done = 0;
await workerPool(runList, jobs, async ({decl}) => {
if (ac.signal.aborted) {
return;
}
try {
for (const out of decl.outputs) {
fs.mkdirSync(pathlib.dirname(out), {recursive: true});
}
const result = await runShell(decl.command, {cwd: root, signal: ac.signal});
if (ac.signal.aborted && result.code !== 0) {
return; // killed by the abort, not a real failure; stays dirty
}
let recorded : RecordedOutput[] | null = null;
const missingOutputs = [];
if (result.code === 0) {
recorded = [];
for (const path of decl.outputs) {
const st = statPath(path);
if (st === null) {
missingOutputs.push(path);
recorded = null;
break;
}
recorded.push({path, size: st.size, mtimeNs: st.mtimeNs});
}
}
db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded);
done++;
if (recorded !== null) {
console.log(`[${done}/${runList.length}] ${label(decl)}`);
if (result.output !== '') {
console.log(indent(result.output));
}
} else {
failures.push(decl);
console.error(`[${done}/${runList.length}] FAILED: ${label(decl)}`);
console.error(` command: ${decl.command}`);
if (result.output !== '') {
console.error(indent(result.output));
}
if (missingOutputs.length > 0) {
console.error(` command succeeded but did not produce: ${missingOutputs.join(' ')}`);
}
if (opts.failFast) {
ac.abort();
}
}
} catch (err) {
// Unexpected (infrastructure) error: count the rule failed and
// stop scheduling; something systemic is wrong.
failures.push(decl);
console.error(`FAILED (internal error): ${label(decl)}`);
console.error(indent(err instanceof Error ? err.stack ?? err.message : String(err)));
ac.abort();
}
});
process.off('SIGINT', onSignal);
process.off('SIGTERM', onSignal);
// Phase 8: summary
const parts = [`${plan.clean.length} up to date`];
if (runList.length > 0) {
parts.push(`${done - failures.length} ran`);
}
if (renamed > 0) {
parts.push(`${renamed} renamed`);
}
if (plan.adopt.length > 0) {
parts.push(`${plan.adopt.length} adopted`);
}
if (plan.stale.length > 0) {
parts.push(`${plan.stale.length} removed`);
}
if (failures.length > 0) {
parts.push(`${failures.length} FAILED`);
}
console.log(parts.join(', ') + '.');
if (failures.length > 0) {
console.error('Failed rules:');
for (const decl of failures) {
console.error(` ${label(decl)}`);
}
}
return interrupted ? 130 : failures.length > 0 ? 1 : 0;
} finally {
db.close();
releaseLock?.();
}
}
try {
process.exitCode = await main();
} catch (err) {
if (err instanceof BuildError) {
console.error(err.message);
process.exitCode = 1;
} else {
throw err;
}
}

139
tools/build/plan.ts
View File

@@ -1,139 +0,0 @@
import {createHash} from 'crypto';
import type {RuleDecl} from './api.ts';
import type {StoredRule} from './db.ts';
export type DirtyReason = 'new' | 'failed-last-run' | 'input-changed'
| 'output-missing' | 'output-tampered';
export interface OutputStat {
size : bigint;
mtimeNs : bigint;
}
export interface Plan {
clean : RuleDecl[];
run : {decl : RuleDecl, reason : DirtyReason}[];
renames : {decl : RuleDecl, from : StoredRule}[];
stale : StoredRule[];
adopt : RuleDecl[];
}
export function inputSig(hashes : Buffer[]) : Buffer {
const h = createHash('sha256');
for (const hash of hashes) {
h.update(hash);
}
return h.digest();
}
export function ruleInputSig(decl : RuleDecl, hashes : Map<string, Buffer>) : Buffer {
return inputSig([...decl.inputs, ...decl.deps].map(p => hashes.get(p)!));
}
function renameKey(template : string, sig : Buffer) : string {
return template + '\0' + sig.toString('hex');
}
export function computePlan(opts : {
current : RuleDecl[],
stored : StoredRule[],
hashes : Map<string, Buffer>,
statOutput : (path : string) => OutputStat | null,
adopt : boolean,
}) : Plan {
const {current, stored, hashes, statOutput, adopt} = opts;
const storedByKey = new Map<string, StoredRule>();
const renameIndex = new Map<string, StoredRule[]>();
for (const s of stored) {
storedByKey.set(s.key, s);
// Rename detection is restricted to single-input, no-deps rules:
// sheet-style rules embed input *names* in their output bytes, so
// identical input content does not imply identical outputs there.
if (s.ok && s.inputs.length === 1 && !s.inputs[0]!.isDep) {
const key = renameKey(s.template, s.inputSig);
let list = renameIndex.get(key);
if (list === undefined) {
renameIndex.set(key, list = []);
}
list.push(s);
}
}
const currentKeys = new Set(current.map(r => r.key));
const plan : Plan = {
clean: [],
run: [],
renames: [],
stale: stored.filter(s => !currentKeys.has(s.key)),
adopt: [],
};
// A stored rule's outputs are intact iff every recorded output exists on
// disk with its recorded stat. Rename sources must pass this (never copy
// tampered or unverified bytes).
const outputsIntact = (s : StoredRule) : boolean =>
s.outputs.every(o => {
if (o.size === null || o.mtimeNs === null) {
return false;
}
const st = statOutput(o.path);
return st !== null && st.size === o.size && st.mtimeNs === o.mtimeNs;
});
const dirtyReason = (s : StoredRule) : DirtyReason | null => {
if (!s.ok) {
return 'failed-last-run';
}
for (const inp of s.inputs) {
if (!hashes.get(inp.path)?.equals(inp.hash)) {
return 'input-changed';
}
}
for (const o of s.outputs) {
const st = statOutput(o.path);
if (st === null) {
return 'output-missing';
}
if (st.size !== o.size || st.mtimeNs !== o.mtimeNs) {
return 'output-tampered';
}
}
return null;
};
for (const decl of current) {
const s = storedByKey.get(decl.key);
let reason : DirtyReason | null;
if (s !== undefined) {
reason = dirtyReason(s);
if (reason === null) {
plan.clean.push(decl);
continue;
}
} else {
reason = 'new';
if (decl.inputs.length === 1 && decl.deps.length === 0) {
const sig = ruleInputSig(decl, hashes);
const candidates = renameIndex.get(renameKey(decl.template, sig)) ?? [];
const from = candidates.find(c =>
c.outputs.length === decl.outputs.length && outputsIntact(c));
if (from !== undefined) {
plan.renames.push({decl, from});
continue;
}
}
}
// Adoption only covers rules we know nothing about; a known-dirty
// rule (input-changed, tampered, failed) must actually run.
if (adopt && reason === 'new' && decl.outputs.every(p => statOutput(p) !== null)) {
plan.adopt.push(decl);
} else {
plan.run.push({decl, reason});
}
}
return plan;
}

15
tools/build/test/exec.test.ts
View File

@@ -2,7 +2,7 @@
import assert from 'node:assert/strict';
import {test} from 'node:test';
import {runShell, workerPool} from '../exec.ts';
import {runShell} from '../exec.ts';
test('runShell runs && chains from stdin and reports exit status', async () => {
const signal = new AbortController().signal;
@@ -14,16 +14,3 @@ test('runShell runs && chains from stdin and reports exit status', async () => {
assert.equal(bad.output, 'partial\n');
});
test('workerPool waits for every worker before rethrowing', async () => {
let finished = 0;
await assert.rejects(
workerPool([1, 2, 3, 4], 2, async item => {
if (item === 1) {
throw new Error('boom');
}
await new Promise(resolve => setTimeout(resolve, 20));
finished++;
}),
/boom/);
assert.equal(finished, 3);
});

29
tools/build/test/graph.test.ts
View File

@@ -1,29 +0,0 @@
import assert from 'node:assert/strict';
import {test} from 'node:test';
import {getRules, resetRules, rule} from '../api.ts';
import {BuildError, checkGraph} from '../graph.ts';
test('checkGraph rejects duplicate outputs', () => {
resetRules();
rule('a.png', ['c1 %f %o'], 'out/x.png');
rule('b.png', ['c2 %f %o'], 'out/x.png');
assert.throws(() => checkGraph(getRules()), BuildError);
});
test('checkGraph rejects cycles', () => {
resetRules();
rule('gen/y', ['c1 %f %o'], 'gen/x');
rule('gen/x', ['c2 %f %o'], 'gen/y');
assert.throws(() => checkGraph(getRules()), /cycle/i);
});
test('checkGraph orders producers before consumers', () => {
resetRules();
rule('gen/mid', ['consume %f %o'], 'out/final');
rule('src/a', ['produce %f %o'], 'gen/mid');
const {order, generated} = checkGraph(getRules());
assert.deepEqual(order.map(r => r.outputs[0]), ['gen/mid', 'out/final']);
assert.deepEqual([...generated].sort(), ['gen/mid', 'out/final']);
});

217
tools/build/test/plan.test.ts
View File

@@ -1,217 +0,0 @@
import assert from 'node:assert/strict';
import {test} from 'node:test';
import {forEachRule, getRules, resetRules, rule, type RuleDecl} from '../api.ts';
import type {StoredRule, StoredRuleOutput} from '../db.ts';
import {computePlan, type OutputStat, ruleInputSig} from '../plan.ts';
function hashOf(content : string) : Buffer {
return Buffer.from(content.padEnd(32, '\0'));
}
function makeForeach(input : string, outputDir : string) : RuleDecl {
resetRules();
forEachRule(input, {cmds: ['convert %f %o']}, `${outputDir}/%b`);
const decl = getRules()[0]!;
assert(decl !== undefined);
return decl;
}
function stored(decl : RuleDecl, hashes : Map<string, Buffer>,
opts : {ok? : boolean, outputs? : StoredRuleOutput[]} = {}) : StoredRule {
return {
id: 1n,
key: decl.key,
command: decl.command,
display: decl.display,
template: decl.template,
inputSig: ruleInputSig(decl, hashes),
ok: opts.ok ?? true,
inputs: [
...decl.inputs.map(path => ({path, isDep: false, hash: hashes.get(path)!})),
...decl.deps.map(path => ({path, isDep: true, hash: hashes.get(path)!})),
],
outputs: opts.outputs ?? decl.outputs.map(path => ({path, size: 10n, mtimeNs: 100n})),
};
}
function statFrom(entries : Record<string, OutputStat | null>) {
return (path : string) : OutputStat | null => entries[path] ?? null;
}
const GOOD : OutputStat = {size: 10n, mtimeNs: 100n};
test('unchanged rule is clean', () => {
const decl = makeForeach('src/a.png', 'build/out');
const hashes = new Map([['src/a.png', hashOf('A')]]);
const plan = computePlan({
current: [decl],
stored: [stored(decl, hashes)],
hashes,
statOutput: statFrom({'build/out/a.png': GOOD}),
adopt: false,
});
assert.deepEqual(plan.clean, [decl]);
assert.equal(plan.run.length, 0);
});
test('dirty reasons', () => {
const decl = makeForeach('src/a.png', 'build/out');
const hashes = new Map([['src/a.png', hashOf('A')]]);
const cases : [StoredRule, (p : string) => OutputStat | null, string][] = [
[stored(decl, new Map([['src/a.png', hashOf('OLD')]])),
statFrom({'build/out/a.png': GOOD}), 'input-changed'],
[stored(decl, hashes, {ok: false}),
statFrom({'build/out/a.png': GOOD}), 'failed-last-run'],
[stored(decl, hashes), statFrom({}), 'output-missing'],
[stored(decl, hashes),
statFrom({'build/out/a.png': {size: 11n, mtimeNs: 100n}}), 'output-tampered'],
];
for (const [s, statOutput, reason] of cases) {
const plan = computePlan({current: [decl], stored: [s], hashes, statOutput, adopt: false});
assert.deepEqual(plan.run.map(r => r.reason), [reason]);
}
});
test('unknown rule runs as new; removed rule is stale', () => {
const decl = makeForeach('src/a.png', 'build/out');
const gone = makeForeach('src/z.png', 'build/out');
const hashes = new Map([['src/a.png', hashOf('A')], ['src/z.png', hashOf('Z')]]);
const plan = computePlan({
current: [decl],
stored: [stored(gone, hashes)],
hashes,
statOutput: statFrom({}),
adopt: false,
});
assert.deepEqual(plan.run.map(r => r.reason), ['new']);
assert.deepEqual(plan.stale.map(s => s.key), [gone.key]);
});
test('renamed input with identical content matches instead of running', () => {
const oldDecl = makeForeach('src/a.png', 'build/out');
const newDecl = makeForeach('src/b.png', 'build/out');
const hashes = new Map([['src/a.png', hashOf('SAME')], ['src/b.png', hashOf('SAME')]]);
const plan = computePlan({
current: [newDecl],
stored: [stored(oldDecl, hashes)],
hashes,
statOutput: statFrom({'build/out/a.png': GOOD}),
adopt: false,
});
assert.equal(plan.run.length, 0);
assert.deepEqual(plan.renames.map(r => [r.from.outputs[0]!.path, r.decl.outputs[0]]),
[['build/out/a.png', 'build/out/b.png']]);
// the source rule is still deleted afterward
assert.deepEqual(plan.stale.map(s => s.key), [oldDecl.key]);
});
test('rename does not match differing content or tampered source outputs', () => {
const oldDecl = makeForeach('src/a.png', 'build/out');
const newDecl = makeForeach('src/b.png', 'build/out');
const differing = new Map([['src/a.png', hashOf('X')], ['src/b.png', hashOf('Y')]]);
let plan = computePlan({
current: [newDecl],
stored: [stored(oldDecl, differing)],
hashes: differing,
statOutput: statFrom({'build/out/a.png': GOOD}),
adopt: false,
});
assert.deepEqual(plan.run.map(r => r.reason), ['new']);
const same = new Map([['src/a.png', hashOf('SAME')], ['src/b.png', hashOf('SAME')]]);
plan = computePlan({
current: [newDecl],
stored: [stored(oldDecl, same)],
hashes: same,
statOutput: statFrom({'build/out/a.png': {size: 99n, mtimeNs: 100n}}),
adopt: false,
});
assert.deepEqual(plan.run.map(r => r.reason), ['new']);
});
test('rename is restricted to single-input, no-deps rules', () => {
// multi-input: same output basename, same content, but never rename-matched
resetRules();
rule(['src/a.png', 'src/b.png'], {cmds: ['c %f %o']}, 'out1/x.png');
rule(['src/a2.png', 'src/b2.png'], {cmds: ['c %f %o']}, 'out2/x.png');
const [multiOld, multiNew] = getRules() as [RuleDecl, RuleDecl];
const multiHashes = new Map([
['src/a.png', hashOf('A')], ['src/b.png', hashOf('B')],
['src/a2.png', hashOf('A')], ['src/b2.png', hashOf('B')],
]);
let plan = computePlan({
current: [multiNew],
stored: [stored(multiOld, multiHashes)],
hashes: multiHashes,
statOutput: statFrom({'out1/x.png': GOOD}),
adopt: false,
});
assert.deepEqual(plan.run.map(r => r.reason), ['new']);
// dep-bearing: same template and sig, but never rename-matched
resetRules();
forEachRule('src/a.png', {deps: 'src/d.json', cmds: ['c %f %o']}, 'out/%b');
forEachRule('src/b.png', {deps: 'src/d.json', cmds: ['c %f %o']}, 'out/%b');
const [depOld, depNew] = getRules() as [RuleDecl, RuleDecl];
const depHashes = new Map([
['src/a.png', hashOf('SAME')], ['src/b.png', hashOf('SAME')],
['src/d.json', hashOf('D')],
]);
plan = computePlan({
current: [depNew],
stored: [stored(depOld, depHashes)],
hashes: depHashes,
statOutput: statFrom({'out/a.png': GOOD}),
adopt: false,
});
assert.deepEqual(plan.run.map(r => r.reason), ['new']);
});
test('rename does not match across input extension changes', () => {
const oldDecl = makeForeach('src/a.png', 'build/out');
resetRules();
forEachRule('src/a.gif', {cmds: ['convert %f %o']}, 'build/out/%b');
const newDecl = getRules()[0]!;
const hashes = new Map([['src/a.png', hashOf('SAME')], ['src/a.gif', hashOf('SAME')]]);
const plan = computePlan({
current: [newDecl],
stored: [stored(oldDecl, hashes)],
hashes,
statOutput: statFrom({'build/out/a.png': GOOD}),
adopt: false,
});
assert.deepEqual(plan.run.map(r => r.reason), ['new']);
});
test('adopt does not bless known-dirty rules', () => {
const decl = makeForeach('src/a.png', 'build/out');
const plan = computePlan({
current: [decl],
stored: [stored(decl, new Map([['src/a.png', hashOf('OLD')]]))],
hashes: new Map([['src/a.png', hashOf('NEW')]]),
statOutput: statFrom({'build/out/a.png': GOOD}),
adopt: true,
});
assert.equal(plan.adopt.length, 0);
assert.deepEqual(plan.run.map(r => r.reason), ['input-changed']);
});
test('adopt records existing outputs instead of running', () => {
resetRules();
rule('src/a.png', {cmds: ['convert %f %o']}, 'build/present.png');
rule('src/b.png', {cmds: ['convert %f %o']}, 'build/absent.png');
const [present, absent] = getRules() as [RuleDecl, RuleDecl];
const hashes = new Map([['src/a.png', hashOf('A')], ['src/b.png', hashOf('B')]]);
const plan = computePlan({
current: [present, absent],
stored: [],
hashes,
statOutput: statFrom({'build/present.png': GOOD}),
adopt: true,
});
assert.deepEqual(plan.adopt, [present]);
assert.deepEqual(plan.run.map(r => r.decl), [absent]);
});

2
tools/build/test/sprites.test.ts
View File

@@ -5,7 +5,7 @@ import os from 'node:os';
import pathlib from 'node:path';
import {test} from 'node:test';
import {base, spritedata, spriteglob} from '../api.ts';
import {base, spritedata, spriteglob} from '../helpers.ts';
test('spritedata parses id and flags', () => {
assert.deepEqual(spritedata('10080'), {id: '10080', data: {}});

92
tools/deploy/api.ts Normal file
View File

@@ -0,0 +1,92 @@
import crypto from 'crypto';
import fs from 'fs';
import b32encode from 'base32-encode';
import {Artifact} from '../build/artifact.ts';
import {casPath} from '../build/cas.ts';
import {type ActionQueue} from './queue.ts';
import * as pathlib from './path.ts';
// A source-tree file from ctx.list(); shaped like a deploy Path plus the
// repo-relative path.
export interface SrcFile extends pathlib.Path {
path : string;
}
export type CopySource = Artifact | SrcFile | string; // string = repo-relative path
// The finish API: nice naming over built artifacts and raw sources. Ops are
// queued in call order (the tar entry order), so __key-style entries must be
// written first.
export interface DeployCtx {
copy(src : CopySource, dst : string) : void;
write(dst : string, data : string) : void;
read(src : CopySource) : string;
list(dir : string) : SrcFile[];
// 8-char base32 content stamp. One source: the digest of its bytes,
// byte-compatible with artifact hashes. Several: a digest of the sorted
// per-source digests (order-insensitive).
hash(...srcs : CopySource[]) : string;
}
export interface DeploySpec {
finish : (ctx : DeployCtx) => void | Promise<void>;
}
// Identity helper: a deploy module declares its rules at top level and
// `export default defineDeploy({finish})`.
export function defineDeploy(spec : DeploySpec) : DeploySpec {
return spec;
}
function shortHash(digest : Buffer) : string {
return b32encode(digest, 'RFC4648').slice(0, 8);
}
export function makeCtx(casDir : string, queue : ActionQueue) : DeployCtx {
// Every path here is repo-root-relative; the CLI chdirs to the root.
const srcPath = (src : CopySource) : string => {
if (src instanceof Artifact) {
return casPath(casDir, src.hash, src.ext);
}
return typeof src === 'string' ? src : src.path;
};
const digestOf = (src : CopySource) : Buffer => {
if (src instanceof Artifact) {
return Buffer.from(src.hash, 'hex');
}
return crypto.createHash('sha256').update(fs.readFileSync(srcPath(src))).digest();
};
return {
copy(src : CopySource, dst : string) : void {
queue.copy(srcPath(src), dst);
},
write(dst : string, data : string) : void {
queue.write(data, dst);
},
read(src : CopySource) : string {
return fs.readFileSync(srcPath(src), 'utf8');
},
list(dir : string) : SrcFile[] {
const result = [];
for (const filename of fs.readdirSync(dir).sort()) {
const p = pathlib.path(filename, {dir});
result.push({...p, path: pathlib.format(p)});
}
return result;
},
hash(...srcs : CopySource[]) : string {
if (srcs.length === 1) {
return shortHash(digestOf(srcs[0]!));
}
const digests = srcs.map(digestOf).sort(Buffer.compare);
const h = crypto.createHash('sha256');
for (const d of digests) {
h.update(d);
}
return shortHash(h.digest());
},
};
}

368
tools/deploy/index.ts
View File

@@ -1,78 +1,159 @@
import program from 'commander';
import * as script from './script.ts';
import nodePath from 'path';
import {spawn, type ChildProcess} from 'child_process';
import fs from 'fs';
import os from 'os';
import nodePath from 'path';
import {fileURLToPath, pathToFileURL} from 'url';
function collect(value : string, previous : string[]) {
return previous.concat([value]);
import {program} from 'commander';
import {type RuleDecl, getDecls} from '../build/artifact.ts';
import {casPath} from '../build/cas.ts';
import {loadConfig} from '../build/config.ts';
import {build} from '../build/driver.ts';
import {BuildError} from '../build/errors.ts';
import {killAllProcessGroups} from '../build/exec.ts';
import {setConfig} from '../build/helpers.ts';
import {Store, acquireLock, dbVersion} from '../build/store.ts';
import {type DeploySpec, makeCtx} from './api.ts';
import {ActionQueue} from './queue.ts';
const root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..');
process.chdir(root);
const DB_PATH = '.build/db.sqlite';
const LOCK_PATH = '.build/lock.sqlite';
const CAS_DIR = '.build/cas';
const TMP_DIR = '.build/tmp';
interface CommonOpts {
jobs : string;
dryRun? : boolean;
failFast? : boolean;
config : string;
verbose? : boolean;
}
async function runAq(aq : script.ActionQueue, mode: 'copy' | 'link' | 'tar', outputDir : undefined | string, verbose : undefined | true) {
const level = verbose ? 'all' : 'errors';
function common(cmd : ReturnType<typeof program.command>) : ReturnType<typeof program.command> {
return cmd
.option('-j, --jobs <n>', 'number of parallel jobs', String(os.availableParallelism()))
.option('-n, --dry-run', 'print what would run without changing anything')
.option('--fail-fast', 'stop scheduling new rules after the first failure')
.option('--config <file>', 'config file', 'build.config')
.option('-v, --verbose', 'print more detail');
}
function discoverDeployFiles() : string[] {
const files = fs.readdirSync('.').filter(f => f.endsWith('.deploy.ts')).sort();
if (files.length === 0) {
throw new BuildError('No *.deploy.ts files at the repo root');
}
return files;
}
// Importing a deploy module declares its rules; the default export carries
// the finish function.
async function importDeploys(files : string[]) : Promise<Map<string, DeploySpec | null>> {
const specs = new Map<string, DeploySpec | null>();
for (const file of files) {
const mod : {default? : unknown} = await import(pathToFileURL(nodePath.resolve(file)).href);
const spec = mod.default;
if (typeof spec === 'object' && spec !== null
&& typeof (spec as DeploySpec).finish === 'function') {
specs.set(file, spec as DeploySpec);
} else {
specs.set(file, null);
}
}
return specs;
}
// Build `decls` and, on success, run `then` while still holding the lock (a
// concurrent GC must not sweep CAS objects out from under a finish). Returns
// the process exit code.
async function buildThen(decls : readonly RuleDecl[], opts : CommonOpts, gc : boolean,
then? : () => Promise<number>) : Promise<number> {
const jobs = Number(opts.jobs);
if (!Number.isInteger(jobs) || jobs < 1) {
throw new BuildError(`Invalid --jobs value: ${opts.jobs}`);
}
const dryRun = Boolean(opts.dryRun);
const release = dryRun ? null : acquireLock(LOCK_PATH);
try {
// A dry run must not create state (opening a db migrates it);
// without a current-version db it reads from an empty in-memory one.
const dbPath = dryRun && dbVersion(DB_PATH) !== 2 ? ':memory:' : DB_PATH;
const store = new Store(dbPath);
if (!dryRun) {
fs.rmSync(TMP_DIR, {recursive: true, force: true});
}
const ac = new AbortController();
let interrupted = false;
const onSignal = () => {
if (interrupted) {
killAllProcessGroups();
process.exit(130);
}
interrupted = true;
console.error('\nInterrupted; waiting for running rules to stop...');
ac.abort();
};
process.on('SIGINT', onSignal);
process.on('SIGTERM', onSignal);
let ok : boolean;
try {
const result = await build(decls, {
root,
store,
casDir: CAS_DIR,
tmpDir: TMP_DIR,
jobs,
dryRun,
failFast: Boolean(opts.failFast),
verbose: Boolean(opts.verbose),
gc,
signal: ac.signal,
});
ok = result.ok;
} finally {
process.off('SIGINT', onSignal);
process.off('SIGTERM', onSignal);
store.close();
}
if (interrupted) {
return 130;
}
if (dryRun) {
return 0;
}
if (!ok) {
return 1;
}
return then === undefined ? 0 : await then();
} finally {
release?.();
}
}
function finishOf(specs : Map<string, DeploySpec | null>, file : string) : DeploySpec {
const spec = specs.get(file);
if (spec === null || spec === undefined) {
throw new BuildError(`${file} does not default-export a deploy (use defineDeploy)`);
}
return spec;
}
async function runFinish(spec : DeploySpec, verbose : boolean) : Promise<ActionQueue | null> {
const aq = new ActionQueue();
await spec.finish(makeCtx(CAS_DIR, aq));
if (!aq.valid) {
aq.print(level);
process.exit(1);
}
if (outputDir !== undefined) {
await aq.run(outputDir, mode);
} else {
if (level === 'errors') {
console.error(`Success, but nothing to do. Please rerun with -v or -o`);
} else {
aq.print('all');
}
aq.print(verbose ? 'all' : 'errors');
return null;
}
return aq;
}
program
.command('copy [files...]')
.option('-o, --output <dir>', 'Output directory')
// TODO: default toID
.option('-e, --eval <expr>', 'Expr')
.option('-m, --module <mod>', 'Module')
.option('-v, --verbose', 'Verbose')
.option('--link', 'Link')
.option('--tar', 'Tar')
// TODO
// .option('-t, --tag <tag>', 'Tag', collect, [])
// from rename(1)
.action(async (files : string[], {eval: expr, module: mod, output: outputDir, verbose, link, tar}) => {
let scr;
if (expr !== undefined) {
scr = new script.Script(expr, 'expr');
} else if (mod !== undefined) {
scr = new script.Script(mod, 'file');
} else {
throw new Error(`one of -e or -m must be provided`);
}
const aq = new script.ActionQueue;
for (const src of files) {
script.runOnFile(scr, src, aq);
}
await runAq(aq, tar ? 'tar' : link ? 'link' : 'copy', outputDir, verbose);
});
program
.command('run [scripts...]')
.option('-o, --output <dir>', 'Output directory')
.option('-v, --verbose', 'Verbose')
.option('--link', 'Link')
.option('--tar', 'Tar')
.action(async (scripts : string[], {output: outputDir, verbose, link, tar}) => {
const aq = new script.ActionQueue;
for (const file of scripts) {
const scr = new script.Script(file, 'file');
script.run(scr, nodePath.dirname(file), aq);
}
await runAq(aq, tar ? 'tar' : link ? 'link' : 'copy', outputDir, verbose);
});
function waitExit(child : ChildProcess) : Promise<number | null> {
return new Promise((resolve, reject) => {
child.on('error', reject);
@@ -80,49 +161,138 @@ function waitExit(child : ChildProcess) : Promise<number | null> {
});
}
program
.command('deploy [scripts...]')
.option('-v, --verbose', 'Verbose')
.action(async (scripts : string[], {verbose}) => {
common(program.command('build [files...]'))
.description('build the rules of the given deploys (default: all *.deploy.ts)')
.action(async (files : string[], opts : CommonOpts) => {
setConfig(loadConfig(opts.config));
// GC needs the full rule universe: only an unfiltered union build
// can know which keys are no longer declared anywhere.
const gc = files.length === 0 && !Boolean(opts.dryRun);
await importDeploys(files.length > 0 ? files : discoverDeployFiles());
process.exitCode = await buildThen(getDecls(), opts, gc);
});
common(program.command('deploy <file>'))
.description('build, finish, and pipe the tar to DEPLOY_COMMAND from .env')
.action(async (file : string, opts : CommonOpts) => {
try {
process.loadEnvFile('.env');
} catch {
console.error(`missing .env; set DEPLOY_COMMAND="ssh smogon smogonctl assets upload"`);
process.exit(1);
process.exitCode = 1;
return;
}
const command = process.env.DEPLOY_COMMAND;
if (command === undefined) {
console.error(`DEPLOY_COMMAND not set in .env`);
process.exit(1);
}
const build = spawn('pnpm', ['build'], {stdio: ['ignore', 'inherit', 'inherit']});
if (await waitExit(build) !== 0) {
process.exit(1);
}
const aq = new script.ActionQueue;
for (const file of scripts) {
const scr = new script.Script(file, 'file');
script.run(scr, nodePath.dirname(file), aq);
}
if (!aq.valid) {
aq.print(verbose ? 'all' : 'errors');
process.exit(1);
}
const upload = spawn(command, {shell: true, stdio: ['pipe', 'inherit', 'inherit']});
// If the command dies early we report its exit code; don't also crash
// on the resulting EPIPE.
upload.stdin!.on('error', () => {});
aq.pack().pipe(upload.stdin!);
if (await waitExit(upload) !== 0) {
process.exit(1);
process.exitCode = 1;
return;
}
setConfig(loadConfig(opts.config));
const specs = await importDeploys([file]);
process.exitCode = await buildThen(getDecls(), opts, false, async () => {
const aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose));
if (aq === null) {
return 1;
}
const upload = spawn(command, {shell: true, stdio: ['pipe', 'inherit', 'inherit']});
// If the command dies early we report its exit code; don't also
// crash on the resulting EPIPE.
upload.stdin!.on('error', () => {});
aq.pack().pipe(upload.stdin!);
return await waitExit(upload) !== 0 ? 1 : 0;
});
});
program.parse(process.argv);
common(program.command('run <file>'))
.description('build, finish, and materialize to a directory (or tar file)')
.requiredOption('-o, --output <dir>', 'output directory (a file with --tar)')
.option('--link', 'hardlink instead of copying')
.option('--tar', 'write a tar file')
.action(async (file : string, opts : CommonOpts & {output : string, link? : boolean, tar? : boolean}) => {
setConfig(loadConfig(opts.config));
const specs = await importDeploys([file]);
process.exitCode = await buildThen(getDecls(), opts, false, async () => {
const aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose));
if (aq === null) {
return 1;
}
await aq.run(opts.output, opts.tar ? 'tar' : opts.link ? 'link' : 'copy');
return 0;
});
});
if (process.argv.slice(2).length === 0) {
program.outputHelp();
function slugOf(decl : RuleDecl) : string {
const template = decl.displayTemplate ?? decl.cmds[0]!;
const slug = template.replace(/%[a-zA-Z0-9]+/g, ' ')
.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
return slug === '' ? 'rule' : slug;
}
common(program.command('inspect <paths...>'))
.description('build every rule touching the given source paths and copy the outputs out')
.requiredOption('-o, --output <dir>', 'output directory')
.action(async (paths : string[], opts : CommonOpts & {output : string}) => {
setConfig(loadConfig(opts.config));
await importDeploys(discoverDeployFiles());
const targets = paths.map(p => nodePath.normalize(p).replace(/\/+$/, ''));
const closure = new Set<RuleDecl>();
for (const decl of getDecls()) {
const hit = [...decl.inputs, ...decl.deps].some(i => typeof i === 'string'
&& targets.some(t => i === t || i.startsWith(t + '/')));
if (hit) {
closure.add(decl);
}
}
if (closure.size === 0) {
throw new BuildError(`No rules consume: ${targets.join(' ')}`);
}
// Transitive consumers: show everything these files end up in. The
// executor pulls in any producers the closure needs on its own.
for (let grew = true; grew;) {
grew = false;
for (const decl of getDecls()) {
if (closure.has(decl)) {
continue;
}
if ([...decl.inputs, ...decl.deps].some(i => typeof i !== 'string' && closure.has(i.decl))) {
closure.add(decl);
grew = true;
}
}
}
const decls = getDecls().filter(d => closure.has(d));
console.log(`inspect: ${decls.length} rules`);
process.exitCode = await buildThen(decls, opts, false, async () => {
for (const decl of decls) {
const dir = nodePath.join(opts.output, slugOf(decl));
fs.mkdirSync(dir, {recursive: true});
for (const artifact of decl.outputs) {
let dst = nodePath.join(dir, artifact.filename);
for (let n = 2; fs.existsSync(dst); n++) {
dst = nodePath.join(dir, `${artifact.name}-${n}.${artifact.ext}`);
}
fs.copyFileSync(casPath(CAS_DIR, artifact.hash, artifact.ext), dst);
fs.chmodSync(dst, 0o644);
console.log(`${nodePath.relative(opts.output, dst)}`);
}
}
return 0;
});
});
try {
await program.parseAsync(process.argv);
if (process.argv.slice(2).length === 0) {
program.outputHelp();
}
} catch (err) {
if (err instanceof BuildError) {
console.error(err.message);
process.exitCode = 1;
} else {
throw err;
}
}

129
tools/deploy/script.ts → tools/deploy/queue.ts
View File

@@ -1,12 +1,7 @@
import fs from 'fs';
import nodePath from 'path';
import vm from 'vm';
import * as pathlib from './path.ts';
import * as spritedata from '@smogon/sprite-data/index.ts';
import tar from 'tar-stream';
import crypto from 'crypto';
import b32encode from 'base32-encode';
type Op = {
type : 'Write',
@@ -184,127 +179,3 @@ export class ActionQueue {
return t;
}
}
export class Script extends vm.Script {
public readonly filename : string | null;
constructor(x : string, type : 'file' | 'expr') {
let code : string;
let filename : string | null = null;
if (type === 'expr') {
// Force expression parsing
code = `(${x})`;
} else {
code = fs.readFileSync(x, 'utf8');
filename = x;
}
super(code, filename !== null ? {filename} : undefined);
this.filename = filename;
}
}
const SKIP = {};
const ENV0 = {
spritedata,
SKIP,
// Because throw SKIP isn't an expression
skip() {
throw SKIP;
}
};
function makeEnv1(queue: ActionQueue) {
return {
__proto__: ENV0,
debug(obj : unknown) {
queue.debug(obj);
},
gdebug(obj : unknown) {
queue.gdebug(obj, false);
}
}
};
function makeEnv2(srcDir : string, queue: ActionQueue) {
return {
__proto__: makeEnv1(queue),
list(dir : string) : pathlib.Path[] {
const result = [];
for (const filename of fs.readdirSync(nodePath.join(srcDir, dir))) {
result.push(pathlib.path(filename, {dir}));
}
return result;
},
copy(srcp : pathlib.PathLike, dstp : string | pathlib.Delta /* todo deltalike */) {
const src = pathlib.format(pathlib.path(srcp));
let dst : string;
if (typeof dstp === 'string') {
dst = dstp;
} else {
dst = pathlib.format(pathlib.path(srcp, dstp));
}
queue.copy(nodePath.join(srcDir, src), dst);
},
read(srcp : pathlib.PathLike) : string {
const src = pathlib.format(pathlib.path(srcp));
return fs.readFileSync(nodePath.join(srcDir, src), 'utf8');
},
hash(...srcps : pathlib.PathLike[]) : string {
let hash = crypto.createHash("sha256");
let srcs = srcps.map(srcp => pathlib.format(pathlib.path(srcp))).sort();
for (let src of srcs) {
let data = fs.readFileSync(nodePath.join(srcDir, src));
hash.update(data);
}
let buffer = hash.digest()
// Similar to esbuild?
return b32encode(buffer, 'RFC4648').slice(0, 8);
},
write(dstp : pathlib.PathLike, data : string) {
const dst = pathlib.format(pathlib.path(dstp));
queue.write(data, dst);
}
}
}
export function runOnFile(scr : Script, src : string, queue: ActionQueue) {
try {
const input = pathlib.path(src, {dir: ""});
const result = scr.runInNewContext({
__proto__: makeEnv1(queue),
path: input,
...input
});
if (result === undefined) {
throw new Error(`undefined output on ${src}`);
}
const output = pathlib.update(input, result);
const dst = pathlib.format(output);
queue.copy(src, dst);
} catch(e) {
if (e === SKIP) {
queue.skip();
return;
}
queue.throw(e);
}
}
export function run(scr : Script, srcDir : string, queue : ActionQueue) {
try {
scr.runInNewContext(makeEnv2(srcDir, queue));
} catch(e) {
if (e === SKIP) {
queue.skip();
return;
}
queue.throw(e);
}
}

141
tools/deploy/test/api.test.ts Normal file
View File

@@ -0,0 +1,141 @@
import assert from 'node:assert/strict';
import {createHash} from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import pathlib from 'node:path';
import {beforeEach, test} from 'node:test';
import b32encode from 'base32-encode';
import tar from 'tar-stream';
import {resetDecls, rule} from '../../build/artifact.ts';
import {casPath} from '../../build/cas.ts';
import {makeCtx} from '../api.ts';
import {ActionQueue} from '../queue.ts';
beforeEach(resetDecls);
function tmpdir() : string {
return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-api-test-'));
}
function shortHash(data : Buffer | string) : string {
return b32encode(createHash('sha256').update(data).digest(), 'RFC4648').slice(0, 8);
}
// Stage `content` as a built artifact in a scratch CAS.
function makeArtifact(casDir : string, content : string, ext : string) {
const digest = createHash('sha256').update(content).digest('hex');
const [artifact] = rule('in.png', ['t %f %o'], [`art.${ext}`]);
artifact!.resolve(digest);
const obj = casPath(casDir, digest, ext);
fs.mkdirSync(pathlib.dirname(obj), {recursive: true});
fs.writeFileSync(obj, content);
return artifact!;
}
test('ctx.hash matches the historical single-file stamp for artifacts and files', () => {
const dir = tmpdir();
const file = pathlib.join(dir, 'f.png');
fs.writeFileSync(file, 'stamp-me');
const artifact = makeArtifact(pathlib.join(dir, 'cas'), 'stamp-me', 'png');
const ctx = makeCtx(pathlib.join(dir, 'cas'), new ActionQueue());
assert.equal(ctx.hash(file), shortHash('stamp-me'));
assert.equal(ctx.hash(artifact), shortHash('stamp-me'));
});
test('multi-source ctx.hash is order-insensitive and content-sensitive', () => {
const dir = tmpdir();
const a = pathlib.join(dir, 'a.png');
const b = pathlib.join(dir, 'b.png');
fs.writeFileSync(a, 'aaa');
fs.writeFileSync(b, 'bbb');
const ctx = makeCtx(pathlib.join(dir, 'cas'), new ActionQueue());
const before = ctx.hash(a, b);
assert.equal(before, ctx.hash(b, a));
assert.notEqual(before, ctx.hash(a));
fs.writeFileSync(b, 'changed');
assert.notEqual(ctx.hash(a, b), before);
});
test('ctx queues artifact copies from the CAS, writes and reads', () => {
const dir = tmpdir();
const casDir = pathlib.join(dir, 'cas');
const artifact = makeArtifact(casDir, 'bytes', 'webp');
const aq = new ActionQueue();
const ctx = makeCtx(casDir, aq);
ctx.write('__key', 'sprites');
ctx.copy(artifact, 'sprites/x.webp');
assert.equal(ctx.read(artifact), 'bytes');
const ops = aq.log.filter(e => e.type === 'Op');
assert.deepEqual(ops.map(e => e.dst), ['__key', 'sprites/x.webp']);
assert.equal((ops[1] as {op : {src : string}}).op.src, casPath(casDir, artifact.hash, 'webp'));
});
test('ctx.list sorts and parses extensions', () => {
const dir = tmpdir();
fs.writeFileSync(pathlib.join(dir, 'b.png'), '');
fs.writeFileSync(pathlib.join(dir, 'a.gif'), '');
fs.writeFileSync(pathlib.join(dir, 'noext'), '');
const ctx = makeCtx('cas', new ActionQueue());
assert.deepEqual(ctx.list(dir), [
{dir, name: 'a', ext: 'gif', path: pathlib.join(dir, 'a.gif')},
{dir, name: 'b', ext: 'png', path: pathlib.join(dir, 'b.png')},
{dir, name: 'noext', ext: null, path: pathlib.join(dir, 'noext')},
]);
});
function packedEntries(aq : ActionQueue) : Promise<{name : string, data : string}[]> {
return new Promise((resolve, reject) => {
const extract = tar.extract();
const entries : {name : string, data : string}[] = [];
extract.on('entry', (header, stream, next) => {
const chunks : Buffer[] = [];
stream.on('data', c => chunks.push(c));
stream.on('end', () => {
entries.push({name: header.name, data: Buffer.concat(chunks).toString()});
next();
});
});
extract.on('finish', () => resolve(entries));
extract.on('error', reject);
aq.pack().pipe(extract);
});
}
test('pack preserves op order with __key first', async () => {
const aq = new ActionQueue();
aq.write('sprites', '__key');
aq.write('zzz', 'z.txt');
aq.write('aaa', 'a.txt');
assert.deepEqual(await packedEntries(aq), [
{name: '__key', data: 'sprites'},
{name: 'z.txt', data: 'zzz'},
{name: 'a.txt', data: 'aaa'},
]);
});
test('duplicate and absolute destinations invalidate the queue', () => {
const dup = new ActionQueue();
dup.write('a', 'x.txt');
dup.write('b', 'x.txt');
assert.ok(!dup.valid);
assert.throws(() => dup.pack(), /Invalid ActionQueue/);
const abs = new ActionQueue();
abs.write('a', '/etc/passwd');
assert.ok(!abs.valid);
});
test('copy-mode materialization restores 0644 on read-only sources', async () => {
const dir = tmpdir();
const src = pathlib.join(dir, 'obj');
fs.writeFileSync(src, 'x');
fs.chmodSync(src, 0o444);
const aq = new ActionQueue();
aq.copy(src, 'out/x.png');
const out = pathlib.join(dir, 'deploy');
await aq.run(out, 'copy');
assert.equal(fs.statSync(pathlib.join(out, 'out/x.png')).mode & 0o777, 0o644);
});

1
tools/deploy/tsconfig.json
View File

@@ -1,6 +1,5 @@
{
"extends": "../../tsconfig-base",
"exclude": ["test"],
"references": [
{"path": "../../data/lib"},
{"path": "../build"},

2
tsconfig.json
View File

@@ -1,6 +1,6 @@
{
"extends": "./tsconfig-base",
"include": ["Buildfile.ts", "*.deploy.ts", "rules/*.ts"],
"include": ["*.deploy.ts", "rules/*.ts"],
"references": [
{"path": "tools/build"},
{"path": "tools/deploy"},

4
updatexyicons.sh
View File

@@ -4,9 +4,7 @@ set -ex
cd ~/smogon/sprites
pnpm build
rm -rf deploy/smogon
node tools/deploy/index.ts run smogon.deploy.js -o deploy/smogon
node tools/deploy/index.ts run smogon.deploy.ts -o deploy/smogon
rsync -a deploy/smogon/xyicons/ smogon:/smog2/sprites/xyicons