Leahnaya/sprites
1
0
Fork 0
mirror of https://github.com/smogon/sprites.git synced 2026-10-01 13:47:35 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix review findings in the content-addressed core

Browse Source 
- casInsert verifies an existing object's bytes before trusting it, fsyncs
  before the rename, and the store records object sizes so a crash-truncated
  object reads as dirty instead of clean forever.
- The driver hashes sources over the producer closure, so partial builds
  (inspect) that pull in producers outside the demanded set work.
- Unknown executor errors are reported and abort the build instead of
  vanishing into allSettled.
- %b/%B expand to nominal names at declaration (never CAS basenames), which
  also puts name-dependence in the identity key; cmds are key-joined with
  NUL so a multi-line command cannot collide with split commands. This
  churns every rule key once.
- Rules must declare at least one output; output names reject shell-hostile
  characters.
- run --link copies read-only (CAS) sources instead of hardlinking 0444
  modes into deploy trees; ctx.list filters dotfiles and directories like
  the build-side glob; inspect resolves targets against the invoking cwd.
- The registry dedupes identical declarations (same cmds/inputs/deps/
  outputs/display), returning the existing artifacts, so shared rule sets
  are plain functions and memo() is gone; ps.deploy.ts no longer declares
  the gen5 gif set it never ships.
- rule() returns a tuple typed by its literal output list (single string
  output returns the Artifact directly), so destructuring needs no
  non-null assertions.
- Manifest is a class carrying the deploy ctx: set() rejects duplicate
  keys, write(dst) emits the sorted JSON, and spritecopy/itemspritecopy/
  stampcopy take just the manifest. The unstamped smogon sets (xyitems,
  fbsprites, twittersprites) are deprecated and commented out.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Christopher Monsanto
2026-08-17 01:15:45 -04:00
parent f6e0b189f3
commit bc39c94851
23 changed files with 498 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@@ -51,8 +51,10 @@ $ brew install imagemagick gifsicle advancecomp optipng pngquant webp
Install dependencies once with `pnpm install`.
Each deploy is a root `*.deploy.ts` module: it declares its build rules
(shared sets live in `rules/`) and a `finish` function that maps the built
artifacts to their published names. Build outputs are content-addressed:
(shared sets are plain functions in `rules/`; declaring an identical rule
twice is a no-op returning the existing artifacts, so any number of deploys
can call the same set) and a `finish` function that maps the built artifacts
to their published names. Build outputs are content-addressed:
rules declare nominal output filenames but the store names every object by
the hash of its bytes (under `.build/cas/`), so incrementality keys on
content, same-byte renames rebuild nothing, and hash-stamped publishing

26
assets.deploy.ts
View File

@@ -1,6 +1,6 @@
import {gen6Padded, itemPadded} from './rules/minisprites.ts';
import {itemspritecopy, type Manifest, newspritecopy, spritecopy, writeManifest} from './rules/publish.ts';
import {Manifest, itemspritecopy, newspritecopy, spritecopy} from './rules/publish.ts';
import {forEachRule, rule} from './tools/build/artifact.ts';
import {spriteglob} from './tools/build/helpers.ts';
import {defineDeploy} from './tools/deploy/api.ts';
@@ -30,7 +30,7 @@ const [sheetPng, sheetCss] = rule(minispriteInputs, {
cmds: ["node tools/smogdexspritesheet/index.ts --image %o1 --stylesheet %o2 -- %f"],
}, ["spritesheet.png", "spritesheet.css"]);
const [sheetWebp] = rule(sheetPng!, ["cwebp -z 9 %f -o %o"], ["spritesheet.webp"]);
const sheetWebp = rule(sheetPng, ["cwebp -z 9 %f -o %o"], "spritesheet.webp");
// Forumsprites sources. Rules must be declared at import time (the build
// runs before finish), so these calls cannot live inside finish().
@@ -47,9 +47,9 @@ export default defineDeploy({
// Dex spritesheet assets: hash-stamped css + webp. The css suffix
// pointer rides in __meta/ for the dex to read.
{
const wh = ctx.hash(sheetWebp!);
ctx.copy(sheetWebp!, `spritesheet-${wh}.webp`);
const src = ctx.read(sheetCss!);
const wh = ctx.hash(sheetWebp);
ctx.copy(sheetWebp, `spritesheet-${wh}.webp`);
const src = ctx.read(sheetCss);
const css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`);
if (css === src) {
throw new Error("spritesheet.css: no webp urls rewritten");
@@ -57,7 +57,7 @@ export default defineDeploy({
// Suffix from source content: the rewritten css is a pure function
// of (css, webp), so this changes exactly when the served bytes
// change.
const ch = ctx.hash(sheetCss!, sheetWebp!);
const ch = ctx.hash(sheetCss, sheetWebp);
ctx.write(`spritesheet-${ch}.css`, css);
ctx.write("__meta/spritesheet_css_suffix.txt", `-${ch}\n`);
}
@@ -71,22 +71,22 @@ export default defineDeploy({
}
{
const manifest : Manifest = {};
const manifest = new Manifest(ctx);
for (const f of forumItems) {
itemspritecopy(ctx, f, {dir: "forumsprites"}, manifest);
itemspritecopy(manifest, f, {dir: "forumsprites"});
}
for (const f of forumG6) {
spritecopy(ctx, f, {dir: "forumsprites"}, true, manifest);
spritecopy(manifest, f, {dir: "forumsprites"}, true);
}
writeManifest(ctx, "__meta/forumsprites/manifest.json", manifest);
manifest.write("__meta/forumsprites/manifest.json");
}
{
const manifest : Manifest = {};
const manifest = new Manifest(ctx);
for (const f of ctx.list("src/pmd")) {
spritecopy(ctx, f, {dir: "pmd"}, false, manifest);
spritecopy(manifest, f, {dir: "pmd"});
}
writeManifest(ctx, "__meta/pmd/manifest.json", manifest);
manifest.write("__meta/pmd/manifest.json");
}
},
});

54
ps.deploy.ts
View File

@@ -1,9 +1,9 @@
import * as spritedata from '@smogon/sprite-data/index.ts';
import {gen10Modelslike, gen5Gifs} from './rules/modelslike.ts';
import {gen10Modelslike} from './rules/modelslike.ts';
import {type Sprite, toPSID} from './rules/publish.ts';
import {forEachRule, memo, rule} from './tools/build/artifact.ts';
import {forEachRule, rule} from './tools/build/artifact.ts';
import {PNG_DETERMINISTIC, base, compresspng, pad, spriteglob} from './tools/build/helpers.ts';
import {type DeployCtx, defineDeploy} from './tools/deploy/api.ts';
@@ -24,21 +24,21 @@ rule("ps-pokemon.sheet.mjs", {
nameSensitive: true,
deps: ["src/minisprites/pokemon/gen6/*", ...sheetDeps],
cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
}, ["pokemonicons-sheet.png"]);
}, "pokemonicons-sheet.png");
// TODO: reenable when trainers are moved
// rule("ps-trainers.sheet.mjs", {
// display: "ps trainers sheet",
// nameSensitive: true,
// cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
// }, ["trainers-sheet.png"]);
// }, "trainers-sheet.png");
rule("ps-items.sheet.mjs", {
display: "ps items sheet",
nameSensitive: true,
deps: ["src/minisprites/items/*", ...sheetDeps],
cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
}, ["itemicons-sheet.png"]);
}, "itemicons-sheet.png");
// PS pokeball icons; input order is the sheet order.
@@ -52,38 +52,34 @@ rule([
`magick convert ${PNG_DETERMINISTIC} -background transparent -gravity center -extent 40x30 %f +append %o`,
compresspng({config: "SPRITESHEET"}),
],
}, ["pokemonicons-pokeball-sheet.png"]);
}, "pokemonicons-pokeball-sheet.png");
// Padded Dex, plus missing CAPs backfilled from the gen5/model gifs.
const paddedDex = memo(() => {
const dex = forEachRule("src/dex/*", {
display: "pad dex %f",
cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})],
}, "%b");
const dex = forEachRule("src/dex/*", {
display: "pad dex %f",
cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})],
}, "%b");
const dexSet = new Set(dex.map(base));
const dexMissing = [];
for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) {
if (!dexSet.has(base(file))) {
dexMissing.push(file);
dexSet.add(base(file));
}
const dexSet = new Set(dex.map(base));
const dexMissing = [];
for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) {
if (!dexSet.has(base(file))) {
dexMissing.push(file);
dexSet.add(base(file));
}
}
return dex.concat(forEachRule(dexMissing, {
display: "missing dex %B",
cmds: [
`magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim %o`,
`magick mogrify ${PNG_DETERMINISTIC} -background transparent -gravity center -resize "120x120>" -extent 120x120 %o`,
compresspng({config: "DEX"}),
],
}, "%B.png"));
});
paddedDex();
forEachRule(dexMissing, {
display: "missing dex %B",
cmds: [
`magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim %o`,
`magick mogrify ${PNG_DETERMINISTIC} -background transparent -gravity center -resize "120x120>" -extent 120x120 %o`,
compresspng({config: "DEX"}),
],
}, "%B.png");
const aniChampions = gen10Modelslike();
gen5Gifs();
// PS ids keep the forme dash, unlike the smogon aliases.
function psSpritecopy(ctx : DeployCtx, f : Sprite, dir : string) : void {

42
rules/minisprites.ts
View File

@@ -1,25 +1,33 @@
import {forEachRule, memo} from '../tools/build/artifact.ts';
import {type Artifact, forEachRule} from '../tools/build/artifact.ts';
import {compresspng, pad, trimimg} from '../tools/build/helpers.ts';
// Uniform size minisprites
export const gen6Padded = memo(() => forEachRule("src/minisprites/pokemon/gen6/*.png", {
display: "pad g6 minisprite %f",
cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})],
}, "%b"));
export function gen6Padded() : Artifact[] {
return forEachRule("src/minisprites/pokemon/gen6/*.png", {
display: "pad g6 minisprite %f",
cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})],
}, "%b");
}
export const itemPadded = memo(() => forEachRule("src/minisprites/items/*.png", {
display: "pad item minisprite %f",
cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})],
}, "%b"));
export function itemPadded() : Artifact[] {
return forEachRule("src/minisprites/items/*.png", {
display: "pad item minisprite %f",
cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})],
}, "%b");
}
export const gen6Trimmed = memo(() => forEachRule("src/minisprites/pokemon/gen6/*.png", {
display: "trim g6 minisprite %f",
cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
}, "%b"));
export function gen6Trimmed() : Artifact[] {
return forEachRule("src/minisprites/pokemon/gen6/*.png", {
display: "trim g6 minisprite %f",
cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
}, "%b");
}
export const itemTrimmed = memo(() => forEachRule("src/minisprites/items/*.png", {
display: "trim item minisprite %f",
cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
}, "%b"));
export function itemTrimmed() : Artifact[] {
return forEachRule("src/minisprites/items/*.png", {
display: "trim item minisprite %f",
cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
}, "%b");
}

56
rules/modelslike.ts
View File

@@ -1,35 +1,41 @@
import {forEachRule, memo} from '../tools/build/artifact.ts';
import {type Artifact, forEachRule} from '../tools/build/artifact.ts';
// Gen 9
export const gen9Modelslike = memo(() => forEachRule("src/gen9species/*.png", {
display: "96x96 %f",
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
cmds: [
"magick convert %f -trim +repage -resize 90x90 %o",
"gifsicle -O3 -b %o",
],
}, "%B.gif"));
export function gen9Modelslike() : Artifact[] {
return forEachRule("src/gen9species/*.png", {
display: "96x96 %f",
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
cmds: [
"magick convert %f -trim +repage -resize 90x90 %o",
"gifsicle -O3 -b %o",
],
}, "%B.gif");
}
// Gen 10
export const gen10Modelslike = memo(() => forEachRule("src/champions/*.png", {
display: "96x96 %f",
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
cmds: [
"magick convert %f -trim +repage -resize 90x90 %o",
"gifsicle -O3 -b %o",
],
}, "%B.gif"));
export function gen10Modelslike() : Artifact[] {
return forEachRule("src/champions/*.png", {
display: "96x96 %f",
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
cmds: [
"magick convert %f -trim +repage -resize 90x90 %o",
"gifsicle -O3 -b %o",
],
}, "%B.gif");
}
// Gen 5 CAPs...
export const gen5Gifs = memo(() => forEachRule("src/sprites/gen5/*.png", [
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
"magick convert %f %o",
"gifsicle -O3 -b %o",
], "%B.gif"));
export function gen5Gifs() : Artifact[] {
return forEachRule("src/sprites/gen5/*.png", [
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
"magick convert %f %o",
"gifsicle -O3 -b %o",
], "%B.gif");
}

70
rules/publish.ts
View File

@@ -8,7 +8,32 @@ import type {DeployCtx, SrcFile} from '../tools/deploy/api.ts';
// or a raw source file.
export type Sprite = Artifact | SrcFile;
export type Manifest = Record<string, string>;
// The unhashed -> hashed name mapping published beside a stamped set.
export class Manifest {
readonly ctx : DeployCtx;
private entries = new Map<string, string>();
constructor(ctx : DeployCtx) {
this.ctx = ctx;
}
set(key : string, value : string) : void {
// ActionQueue only dedups final dsts; hashed dsts differ even when
// unhashed names collide, so check the key explicitly.
if (this.entries.has(key)) {
throw new Error(`duplicate sprite name ${key}`);
}
this.entries.set(key, value);
}
write(dst : string) : void {
const sorted : Record<string, string> = {};
for (const k of [...this.entries.keys()].sort()) {
sorted[k] = this.entries.get(k)!;
}
this.ctx.write(dst, JSON.stringify(sorted, null, 4) + "\n");
}
}
export interface Dest {
dir : string;
@@ -35,38 +60,14 @@ export function toPSID(name : string) : string {
// Copy with a content-hash-stamped name and record the unhashed -> hashed
// mapping in `manifest`.
export function stampcopy(ctx : DeployCtx, f : Sprite, {dir, ext}: Dest, name : string,
manifest : Manifest) : void {
const key = `${name}.${extOf(f, ext)}`;
// ActionQueue only dedups final dsts; hashed dsts differ even when
// unhashed names collide, so check the manifest key explicitly.
if (manifest[key] !== undefined) {
throw new Error(`duplicate sprite name ${key}`);
}
const h = ctx.hash(f);
manifest[key] = `${name}-${h}.${extOf(f, ext)}`;
ctx.copy(f, `${dir}/${name}-${h}.${extOf(f, ext)}`);
export function stampcopy(manifest : Manifest, f : Sprite, {dir, ext}: Dest, name : string) : void {
const h = manifest.ctx.hash(f);
manifest.set(`${name}.${extOf(f, ext)}`, `${name}-${h}.${extOf(f, ext)}`);
manifest.ctx.copy(f, `${dir}/${name}-${h}.${extOf(f, ext)}`);
}
export function writeManifest(ctx : DeployCtx, dst : string, manifest : Manifest) : void {
const sorted : Manifest = {};
for (const k of Object.keys(manifest).sort()) {
sorted[k] = manifest[k]!;
}
ctx.write(dst, JSON.stringify(sorted, null, 4) + "\n");
}
function emit(ctx : DeployCtx, f : Sprite, dest : Dest, name : string,
manifest : Manifest | null) : void {
if (manifest) {
stampcopy(ctx, f, dest, name, manifest);
} else {
ctx.copy(f, `${dest.dir}/${name}.${extOf(f, dest.ext)}`);
}
}
export function spritecopy(ctx : DeployCtx, f : Sprite, dest : Dest,
allowUnknown = false, manifest : Manifest | null = null) : void {
export function spritecopy(manifest : Manifest, f : Sprite, dest : Dest,
allowUnknown = false) : void {
const sn = spritedata.parseFilename(f.name);
let name : string;
@@ -99,12 +100,11 @@ export function spritecopy(ctx : DeployCtx, f : Sprite, dest : Dest,
name += "-gmax";
}
emit(ctx, f, dest, name, manifest);
stampcopy(manifest, f, dest, name);
}
// TODO: merge with above
export function itemspritecopy(ctx : DeployCtx, f : Sprite, dest : Dest,
manifest : Manifest | null = null) : void {
export function itemspritecopy(manifest : Manifest, f : Sprite, dest : Dest) : void {
const sn = spritedata.parseFilename(f.name);
if (sn.extension) {
throw new Error(`Not an item sprite: ${f.name}`);
@@ -114,7 +114,7 @@ export function itemspritecopy(ctx : DeployCtx, f : Sprite, dest : Dest,
throw new Error(`Not an item sprite: ${f.name}`);
}
for (const n of sd.names) {
emit(ctx, f, dest, toSmogonAlias(n), manifest);
stampcopy(manifest, f, dest, toSmogonAlias(n));
}
}

38
rules/social.ts
View File

@@ -1,11 +1,11 @@
import {forEachRule, memo} from '../tools/build/artifact.ts';
import {type Artifact, forEachRule} from '../tools/build/artifact.ts';
import {PNG_DETERMINISTIC, base, compresspng, spriteglob} from '../tools/build/helpers.ts';
// Smogdex social images: models, backfilled with gen9 species not yet in
// models (first source wins).
const socialInputs = memo(() => {
function socialInputs() : string[] {
const social = spriteglob(["src/models/*"], {b: false, s: false});
const socialSeen = new Set(social.map(base));
for (const file of spriteglob(["src/gen9species/*"], {b: false, s: false})) {
@@ -15,20 +15,24 @@ const socialInputs = memo(() => {
}
}
return social;
});
}
export const fbSprites = memo(() => forEachRule(socialInputs(), {
display: "fbsprite %f",
cmds: [
`magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o`,
compresspng({config: "MODELS"}),
],
}, "%B.png"));
export function fbSprites() : Artifact[] {
return forEachRule(socialInputs(), {
display: "fbsprite %f",
cmds: [
`magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o`,
compresspng({config: "MODELS"}),
],
}, "%B.png");
}
export const twitterSprites = memo(() => forEachRule(socialInputs(), {
display: "twittersprite %f",
cmds: [
`magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim -resize 115x115 -background white -gravity center -extent 120x120 %o`,
compresspng({config: "MODELS"}),
],
}, "%B.png"));
export function twitterSprites() : Artifact[] {
return forEachRule(socialInputs(), {
display: "twittersprite %f",
cmds: [
`magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim -resize 115x115 -background white -gravity center -extent 120x120 %o`,
compresspng({config: "MODELS"}),
],
}, "%B.png");
}

46
smogon.deploy.ts
View File

@@ -1,29 +1,30 @@
import {gen6Trimmed, itemTrimmed} from './rules/minisprites.ts';
import {gen6Trimmed} from './rules/minisprites.ts';
import {gen10Modelslike, gen5Gifs, gen9Modelslike} from './rules/modelslike.ts';
import {itemspritecopy, type Manifest, type Sprite, spritecopy, writeManifest} from './rules/publish.ts';
import {fbSprites, twitterSprites} from './rules/social.ts';
import {Manifest, type Sprite, spritecopy} from './rules/publish.ts';
import {defineDeploy} from './tools/deploy/api.ts';
const xyModels = gen9Modelslike();
const xyChampions = gen10Modelslike();
const xyGen5 = gen5Gifs();
const xyIcons = gen6Trimmed();
const xyItems = itemTrimmed();
const fb = fbSprites();
const twitter = twitterSprites();
// Deprecated, unstamped sets:
// const xyItems = itemTrimmed(); (rules/minisprites.ts)
// const fb = fbSprites(); (rules/social.ts)
// const twitter = twitterSprites(); (rules/social.ts)
export default defineDeploy({
finish(ctx) {
// xy/ animations: first source wins per sprite name.
const seenModels = new Set<string>();
const xyManifest : Manifest = {};
const xyManifest = new Manifest(ctx);
const xycopy = (f : Sprite) => {
if (seenModels.has(f.name)) {
return;
}
seenModels.add(f.name);
spritecopy(ctx, f, {dir: "xy"}, false, xyManifest);
spritecopy(xyManifest, f, {dir: "xy"});
};
for (const f of ctx.list("src/models")) {
@@ -44,26 +45,25 @@ export default defineDeploy({
for (const f of xyGen5) {
xycopy(f);
}
writeManifest(ctx, "xy/manifest.json", xyManifest);
xyManifest.write("xy/manifest.json");
{
const manifest : Manifest = {};
const manifest = new Manifest(ctx);
for (const f of xyIcons) {
spritecopy(ctx, f, {dir: "xyicons"}, false, manifest);
spritecopy(manifest, f, {dir: "xyicons"});
}
writeManifest(ctx, "xyicons/manifest.json", manifest);
manifest.write("xyicons/manifest.json");
}
for (const f of xyItems) {
itemspritecopy(ctx, f, {dir: "xyitems"});
}
for (const f of fb) {
spritecopy(ctx, f, {dir: "fbsprites/xy"});
}
for (const f of twitter) {
spritecopy(ctx, f, {dir: "twittersprites/xy"});
}
// Deprecated, unstamped:
// for (const f of xyItems) {
// itemspritecopy(?, f, {dir: "xyitems"});
// }
// for (const f of fb) {
// spritecopy(?, f, {dir: "fbsprites/xy"});
// }
// for (const f of twitter) {
// spritecopy(?, f, {dir: "twittersprites/xy"});
// }
},
});

71
tools/build/artifact.ts
View File

@@ -3,7 +3,7 @@ import pathlib from 'path';
import {createHash} from 'crypto';
import {astable, glob} from './helpers.ts';
import {type Cmd, basenameNoExt, flattenCmds, substitute} from './subst.ts';
import {type Cmd, basenameNoExt, flattenCmds, substitute, substituteNames} from './subst.ts';
// A rule's output: content-addressed bytes with a nominal name. The nominal
// name exists for provenance, inspection, and deploy-time naming; it is NOT
@@ -65,6 +65,7 @@ export interface CmdSpec {
}
export interface RuleDecl {
id : number; // registration order; identity for artifact inputs
inputs : Input[]; // ordered (%f order; some rules are order-sensitive)
deps : Input[];
outputs : Artifact[];
@@ -75,6 +76,9 @@ export interface RuleDecl {
}
let decls : RuleDecl[] = [];
// Declaring an identical rule twice returns the existing artifacts, so
// shared rule sets are plain functions that any number of deploys may call.
let declIndex = new Map<string, RuleDecl>();
export function getDecls() : RuleDecl[] {
return decls;
@@ -82,20 +86,7 @@ export function getDecls() : RuleDecl[] {
export function resetDecls() : void {
decls = [];
}
// Share a rule set between deploys: importing declares nothing, the first
// call declares once, later calls return the same artifacts.
export function memo<T>(fn : () => T) : () => T {
let called = false;
let value : T;
return () => {
if (!called) {
value = fn();
called = true;
}
return value!;
};
declIndex = new Map();
}
// Nominal path of an input, for %b/%B and displays.
@@ -123,7 +114,9 @@ export function computeKey(decl : RuleDecl, digestOf : (i : Input) => string) :
const h = createHash('sha256');
h.update([
'v2',
decl.cmds.join('\n'),
// \0, not \n: a single multi-line command executes differently from
// the same lines as separate ' && '-joined commands.
decl.cmds.join('\0'),
decl.inputs.map(digestOf).join('\0'),
decl.inputs.map(extOf).join('\0'),
decl.deps.map(digestOf).join('\0'),
@@ -146,11 +139,36 @@ function resolveInputs(input : Input | Input[] | undefined) : Input[] {
}
function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : string[]) : RuleDecl {
const cmds = flattenCmds(spec.cmds);
const nominalInputs = inputs.map(nominal);
// %b/%B expand to nominal input names, never CAS paths, so they are
// resolved at declaration. This also lands them in the identity key: a
// command embedding input names is name-dependent by construction.
const cmds = flattenCmds(spec.cmds).map(c => substituteNames(c, nominalInputs));
if (cmds.length === 0) {
throw new Error(`Rule with no commands (outputs: ${outputs.join(' ')})`);
}
if (outputs.length === 0) {
throw new Error(`Rule with no outputs (cmds: ${cmds[0]!})`);
}
// An identical declaration returns the already-registered rule. Artifact
// inputs are keyed by their producing rule's id, so chains dedupe too.
const token = (i : Input) => typeof i === 'string' ? `s:${i}` : `a:${i.decl.id}:${i.index}`;
const identity = [
cmds.join('\0'),
inputs.map(token).join('\0'),
deps.map(token).join('\0'),
outputs.join('\0'),
spec.display ?? '',
spec.nameSensitive ? '1' : '0',
].join('\x01');
const existing = declIndex.get(identity);
if (existing !== undefined) {
return existing;
}
const decl : RuleDecl = {
id: decls.length,
inputs,
deps,
outputs: [],
@@ -168,6 +186,10 @@ function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : st
if (out.includes('/') || out.includes('%')) {
throw new Error(`Rule outputs are nominal filenames, no paths or substitutions: ${out}`);
}
if (/[\s'"\\]/.test(out)) {
// Output names are substituted into shell commands unquoted.
throw new Error(`Rule output name has shell-hostile characters: ${out}`);
}
if (outputs.indexOf(out) !== index) {
throw new Error(`Duplicate rule output: ${out}`);
}
@@ -179,7 +201,6 @@ function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : st
});
// Validate substitutions now (unknown escapes, out-of-range %oN) rather
// than at execution; the results are discarded.
const nominalInputs = inputs.map(nominal);
const nominalOutputs = decl.outputs.map(o => o.filename);
for (const cmd of cmds) {
substitute(cmd, nominalInputs, nominalOutputs);
@@ -188,13 +209,23 @@ function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : st
decl.display = substitute(spec.display, nominalInputs, nominalOutputs);
}
decls.push(decl);
declIndex.set(identity, decl);
return decl;
}
// One Artifact per declared output, as a tuple when the output list is a
// literal, so `const [png, css] = rule(...)` needs no undefined checks. A
// single string output returns its Artifact directly.
export function rule(input : Input | Input[], spec : CmdSpec | Cmd[],
output : string | string[]) : Artifact[] {
output : string) : Artifact;
export function rule<const T extends readonly string[]>(
input : Input | Input[], spec : CmdSpec | Cmd[],
output : T) : {[K in keyof T] : Artifact};
export function rule(input : Input | Input[], spec : CmdSpec | Cmd[],
output : string | readonly string[]) : Artifact | Artifact[] {
const s = normalizeSpec(spec);
return makeDecl(resolveInputs(input), resolveInputs(s.deps), s, astable(output)).outputs;
const decl = makeDecl(resolveInputs(input), resolveInputs(s.deps), s, astable(output));
return typeof output === 'string' ? decl.outputs[0]! : decl.outputs;
}
export function forEachRule(input : Input | Input[], spec : CmdSpec | Cmd[],

39
tools/build/cas.ts
View File

@@ -16,26 +16,49 @@ export function casPath(casDir : string, digest : string, ext : string) : string
}
export function casExists(casDir : string, digest : string, ext : string) : boolean {
return casStat(casDir, digest, ext) !== null;
}
// Size of an object, or null if absent. Callers verify it against the
// recorded size: a crash between rename and data flush can leave a
// truncated object, which must read as dirty, not clean.
export function casStat(casDir : string, digest : string, ext : string) : bigint | null {
try {
return fs.statSync(casPath(casDir, digest, ext)).isFile();
const st = fs.statSync(casPath(casDir, digest, ext), {bigint: true});
return st.isFile() ? st.size : null;
} catch {
return false;
return null;
}
}
// Move tmpPath into the store, returning the content digest (hex). If the
// object already exists the temp file is simply discarded.
export function casInsert(casDir : string, tmpPath : string, ext : string) : string {
export interface CasObject {
digest : string; // sha256 hex of the bytes
size : bigint;
}
// Move tmpPath into the store, returning the content digest and size. An
// existing object is trusted only if its bytes actually hash to the digest;
// otherwise (crash-truncated object) the fresh bytes replace it.
export function casInsert(casDir : string, tmpPath : string, ext : string) : CasObject {
const digest = hashFileSync(tmpPath).toString('hex');
const size = fs.statSync(tmpPath, {bigint: true}).size;
const target = casPath(casDir, digest, ext);
if (fs.existsSync(target)) {
if (fs.existsSync(target) && hashFileSync(target).toString('hex') === digest) {
fs.unlinkSync(tmpPath);
return digest;
return {digest, size};
}
fs.mkdirSync(pathlib.dirname(target), {recursive: true});
fs.chmodSync(tmpPath, 0o444);
// Flush the bytes before the rename becomes visible, so a power loss
// cannot journal the rename while dropping the data pages.
const fd = fs.openSync(tmpPath, 'r');
try {
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
fs.renameSync(tmpPath, target);
return digest;
return {digest, size};
}
// Remove every object not in `live` (keys are "<digest>.<ext>", the object

14
tools/build/driver.ts
View File

@@ -34,14 +34,24 @@ export async function build(decls : readonly RuleDecl[], opts : BuildOpts) : Pro
const logError = opts.logError ?? console.error;
const {store} = opts;
// Hash sources over the producer closure: demanding a rule demands the
// producers of its artifact inputs, even ones outside `decls`.
const closure = new Set<RuleDecl>();
const sources = new Set<string>();
for (const decl of decls) {
const add = (decl : RuleDecl) => {
if (closure.has(decl)) {
return;
}
closure.add(decl);
for (const input of [...decl.inputs, ...decl.deps]) {
if (typeof input === 'string') {
sources.add(input);
} else {
add(input.decl);
}
}
}
};
decls.forEach(add);
const {hashes, updated, missing} = reconcileHashes(sources, store.loadFileCache());
if (missing.length > 0) {
throw new BuildError(`Missing input files:\n ${missing.slice(0, 20).join('\n ')}`

37
tools/build/executor.ts
View File

@@ -3,7 +3,7 @@ import fs from 'fs';
import pathlib from 'path';
import {type Input, type RuleDecl, computeKey} from './artifact.ts';
import {casExists, casInsert, casPath} from './cas.ts';
import {casInsert, casPath, casStat} from './cas.ts';
import {BuildError} from './errors.ts';
import {runShell} from './exec.ts';
import {type Store} from './store.ts';
@@ -118,7 +118,19 @@ export class Executor {
private demand(decl : RuleDecl) : Promise<string[]> {
let p = this.memo.get(decl);
if (p === undefined) {
p = this.demandInner(decl);
// Any non-sentinel escape (a store error, a resolve conflict, a
// bug) must surface as a reported failure, not vanish into the
// allSettled in build().
p = this.demandInner(decl).catch(err => {
if (err instanceof RuleFailed || err instanceof DryDirty || err instanceof Aborted) {
throw err;
}
this.outcomes.set(decl, {status: 'failed', message: 'internal error'});
this.logError(`FAILED (internal error): ${label(decl)}`);
this.logError(indent(err instanceof Error ? err.stack ?? err.message : String(err)));
this.failAc.abort();
throw new RuleFailed();
});
this.memo.set(decl, p);
}
return p;
@@ -185,7 +197,7 @@ export class Executor {
if (stored !== null
&& stored.length === decl.outputs.length
&& stored.every((o, n) => o.ext === decl.outputs[n]!.ext)
&& stored.every(o => casExists(casDir, o.digest, o.ext))) {
&& stored.every(o => casStat(casDir, o.digest, o.ext) === o.size)) {
this.outcomes.set(decl, {status: 'clean'});
return stored.map(o => o.digest);
}
@@ -203,17 +215,6 @@ export class Executor {
throw new Aborted();
}
return await this.execute(decl, key, reason);
} catch (err) {
if (err instanceof Aborted || err instanceof RuleFailed) {
throw err;
}
// Unexpected (infrastructure) error: count the rule failed and
// stop scheduling; something systemic is wrong.
this.outcomes.set(decl, {status: 'failed', message: 'internal error'});
this.logError(`FAILED (internal error): ${label(decl)}`);
this.logError(indent(err instanceof Error ? err.stack ?? err.message : String(err)));
this.failAc.abort();
throw new RuleFailed();
} finally {
this.semaphore.release();
}
@@ -236,10 +237,10 @@ export class Executor {
if (result.code === 0) {
const missing = tempOutputs.filter(p => !fs.existsSync(p));
if (missing.length === 0) {
const outputs = decl.outputs.map((o, n) => ({
digest: casInsert(casDir, tempOutputs[n]!, o.ext),
ext: o.ext,
}));
const outputs = decl.outputs.map((o, n) => {
const object = casInsert(casDir, tempOutputs[n]!, o.ext);
return {digest: object.digest, ext: o.ext, size: object.size};
});
store.recordRule(key, decl.cmds, outputs);
this.outcomes.set(decl, {status: 'ran', reason});
this.log(`[${++this.counter}] ${label(decl)}`

4
tools/build/helpers.ts
View File

@@ -16,11 +16,11 @@ export function getconfig(name : string) : string | undefined {
return value === '' ? undefined : value;
}
export function astable(x : string | string[] | undefined) : string[] {
export function astable(x : string | readonly string[] | undefined) : string[] {
if (x === undefined) {
return [];
}
return typeof x === 'string' ? [x] : x;
return typeof x === 'string' ? [x] : [...x];
}
// Single-directory, single-'*' glob (all Tupfile patterns were of this form).

10
tools/build/store.ts
View File

@@ -9,6 +9,7 @@ import type {FileStat} from './hash.ts';
export interface StoredOutput {
digest : string; // sha256 hex of the bytes; the CAS object is <digest>.<ext>
ext : string;
size : bigint; // verified against the object on every clean check
}
const DDL = `
@@ -33,6 +34,7 @@ CREATE TABLE IF NOT EXISTS rule_outputs (
ord INTEGER NOT NULL,
digest TEXT NOT NULL,
ext TEXT NOT NULL,
size INTEGER NOT NULL,
PRIMARY KEY (rule_id, ord)
);
`;
@@ -111,8 +113,8 @@ export class Store {
if (rule === undefined) {
return null;
}
return this.db.prepare<[bigint], {digest : string, ext : string}>(
'SELECT digest, ext FROM rule_outputs WHERE rule_id = ? ORDER BY ord').all(rule.id);
return this.db.prepare<[bigint], StoredOutput>(
'SELECT digest, ext, size FROM rule_outputs WHERE rule_id = ? ORDER BY ord').all(rule.id);
})();
}
@@ -125,11 +127,11 @@ export class Store {
RETURNING id`);
const delOutputs = this.db.prepare('DELETE FROM rule_outputs WHERE rule_id = ?');
const insOutput = this.db.prepare(
'INSERT INTO rule_outputs (rule_id, ord, digest, ext) VALUES (?, ?, ?, ?)');
'INSERT INTO rule_outputs (rule_id, ord, digest, ext, size) VALUES (?, ?, ?, ?, ?)');
this.db.transaction(() => {
const {id} = upsert.get(key, cmds.join('\n'))!;
delOutputs.run(id);
outputs.forEach((o, i) => insOutput.run(id, i, o.digest, o.ext));
outputs.forEach((o, i) => insOutput.run(id, i, o.digest, o.ext, o.size));
})();
}

9
tools/build/subst.ts
View File

@@ -19,6 +19,15 @@ export function basenameNoExt(path : string) : string {
return dot > 0 ? base.slice(0, dot) : base;
}
// Expand only the name substitutions (%b/%B). They are static per rule, so
// declarations expand them eagerly; %f/%o/%oN wait for execution, when
// concrete paths exist.
export function substituteNames(s : string, inputs : string[]) : string {
return s.replace(/%([bB])/g, (_, c : string) =>
c === 'b' ? inputs.map(p => pathlib.basename(p)).join(' ')
: inputs.map(basenameNoExt).join(' '));
}
// Tup-style substitutions:
// %f inputs, space-joined %b input basenames
// %o outputs, space-joined %B input basenames without extension

139
tools/build/test/artifact.test.ts
View File

@@ -2,7 +2,7 @@
import assert from 'node:assert/strict';
import {beforeEach, test} from 'node:test';
import {type Input, computeKey, forEachRule, memo, resetDecls, rule} from '../artifact.ts';
import {type Input, computeKey, forEachRule, getDecls, resetDecls, rule} from '../artifact.ts';
beforeEach(resetDecls);
@@ -19,27 +19,56 @@ function digests(map : Record<string, string>) : (i : Input) => string {
test('rule declares artifacts with nominal names', () => {
const [png, css] = rule('src/a.png', ['tool %f %o1 %o2'], ['sheet.png', 'sheet.css']);
assert.equal(png!.name, 'sheet');
assert.equal(png!.ext, 'png');
assert.equal(css!.filename, 'sheet.css');
assert.deepEqual(png!.sources, ['src/a.png']);
assert.equal(png!.decl, css!.decl);
assert.throws(() => png!.hash, /not been built/);
png!.resolve('d1');
assert.equal(png!.hash, 'd1');
assert.throws(() => png!.resolve('d2'), /resolved twice/);
assert.equal(png.name, 'sheet');
assert.equal(png.ext, 'png');
assert.equal(css.filename, 'sheet.css');
assert.deepEqual(png.sources, ['src/a.png']);
assert.equal(png.decl, css.decl);
assert.throws(() => png.hash, /not been built/);
png.resolve('d1');
assert.equal(png.hash, 'd1');
assert.throws(() => png.resolve('d2'), /resolved twice/);
});
test('a single string output returns its artifact directly', () => {
const out = rule('src/a.png', ['tool %f %o'], 'only.png');
assert.equal(out.filename, 'only.png');
});
test('rule rejects paths, substitutions, missing extensions in outputs', () => {
assert.throws(() => rule('a.png', ['c'], ['dir/x.png']), /nominal filenames/);
assert.throws(() => rule('a.png', ['c'], ['%B.png']), /nominal filenames/);
assert.throws(() => rule('a.png', ['c'], ['noext']), /needs an extension/);
assert.throws(() => rule('a.png', ['c'], []), /no outputs/);
assert.throws(() => rule('a.png', ['c'], ['has space.png']), /shell-hostile/);
assert.throws(() => rule('a.png', ['c'], ['quote"d.png']), /shell-hostile/);
});
test('%b/%B in commands expand to nominal names at declaration', () => {
const art = rule('src/dir/a.png', ['tool %f %o'], 'mid.png');
const out = rule(art, ['emit --name %B %f %o'], 'x.css');
// Expanded eagerly (never a CAS basename), and thus part of the key.
assert.deepEqual(out.decl.cmds, ['emit --name mid %f %o']);
art.resolve('d1');
const renamed = rule('src/dir/b.png', ['tool %f %o'], 'other.png');
const out2 = rule(renamed, ['emit --name %B %f %o'], 'x.css');
renamed.resolve('d1');
assert.notEqual(
computeKey(out.decl, digests({})),
computeKey(out2.decl, digests({})));
});
test('a multi-line command and split commands get different keys', () => {
const a = rule('a.png', ['one\ntwo'], 'x.png');
const b = rule('a.png', ['one', 'two'], 'x.png');
const map = {'a.png': 'd1'};
assert.notEqual(computeKey(a.decl, digests(map)), computeKey(b.decl, digests(map)));
});
test('rule validates command substitutions at declaration', () => {
assert.throws(() => rule('a.png', ['tool %q'], ['x.png']), /Unknown substitution/);
assert.throws(() => rule('a.png', ['tool %o2'], ['x.png']), /out of range/);
assert.throws(() => rule('a.png', [''], ['x.png']), /no commands/);
assert.throws(() => rule('a.png', ['tool %q'], 'x.png'), /Unknown substitution/);
assert.throws(() => rule('a.png', ['tool %o2'], 'x.png'), /out of range/);
assert.throws(() => rule('a.png', [''], 'x.png'), /no commands/);
});
test('forEachRule declares one rule per input, %b/%B templates', () => {
@@ -50,54 +79,78 @@ test('forEachRule declares one rule per input, %b/%B templates', () => {
});
test('chained rules accept artifacts as inputs', () => {
const [png] = rule('src/a.png', ['tool %f %o'], ['x.png']);
const [webp] = rule(png!, ['cwebp %f -o %o'], ['x.webp']);
assert.equal(webp!.decl.inputs[0], png);
assert.deepEqual(webp!.sources, []);
png!.resolve('dp');
const key = computeKey(webp!.decl, digests({}));
const png = rule('src/a.png', ['tool %f %o'], 'x.png');
const webp = rule(png, ['cwebp %f -o %o'], 'x.webp');
assert.equal(webp.decl.inputs[0], png);
assert.deepEqual(webp.sources, []);
png.resolve('dp');
const key = computeKey(webp.decl, digests({}));
assert.equal(typeof key, 'string');
});
test('key ignores input paths but not bytes, order, exts, or commands', () => {
const [a] = rule('src/a.png', ['convert %f %o'], ['out.png']);
const [b] = rule('src/elsewhere/z.png', ['convert %f %o'], ['other.png']);
const kA = computeKey(a!.decl, digests({'src/a.png': 'd1'}));
const a = rule('src/a.png', ['convert %f %o'], 'out.png');
const b = rule('src/elsewhere/z.png', ['convert %f %o'], 'other.png');
const kA = computeKey(a.decl, digests({'src/a.png': 'd1'}));
// Renamed source, same bytes, different nominal output: same key.
assert.equal(kA, computeKey(b!.decl, digests({'src/elsewhere/z.png': 'd1'})));
assert.equal(kA, computeKey(b.decl, digests({'src/elsewhere/z.png': 'd1'})));
// Different bytes: different key.
assert.notEqual(kA, computeKey(b!.decl, digests({'src/elsewhere/z.png': 'd2'})));
assert.notEqual(kA, computeKey(b.decl, digests({'src/elsewhere/z.png': 'd2'})));
const [two] = rule(['x.png', 'y.png'], ['join %f %o'], ['out.png']);
const [reversed] = rule(['y.png', 'x.png'], ['join %f %o'], ['out.png']);
const two = rule(['x.png', 'y.png'], ['join %f %o'], 'out.png');
const reversed = rule(['y.png', 'x.png'], ['join %f %o'], 'out.png');
const map = {'x.png': 'dx', 'y.png': 'dy'};
assert.notEqual(computeKey(two!.decl, digests(map)), computeKey(reversed!.decl, digests(map)));
assert.notEqual(computeKey(two.decl, digests(map)), computeKey(reversed.decl, digests(map)));
const [gif] = rule('src/a.png', ['convert %f %o'], ['out.gif']);
assert.notEqual(kA, computeKey(gif!.decl, digests({'src/a.png': 'd1'})));
const gif = rule('src/a.png', ['convert %f %o'], 'out.gif');
assert.notEqual(kA, computeKey(gif.decl, digests({'src/a.png': 'd1'})));
});
test('nameSensitive keys on paths; rejects artifact inputs', () => {
const spec = {nameSensitive: true, cmds: ['tool %f %o']};
const [a] = rule('src/a.png', spec, ['out.png']);
const [b] = rule('src/b.png', spec, ['out.png']);
const a = rule('src/a.png', spec, 'out.png');
const b = rule('src/b.png', spec, 'out.png');
assert.notEqual(
computeKey(a!.decl, digests({'src/a.png': 'd1'})),
computeKey(b!.decl, digests({'src/b.png': 'd1'})));
const [art] = rule('src/a.png', ['t %f %o'], ['x.png']);
assert.throws(() => rule(art!, spec, ['y.png']), /not yet supported/);
computeKey(a.decl, digests({'src/a.png': 'd1'})),
computeKey(b.decl, digests({'src/b.png': 'd1'})));
const art = rule('src/a.png', ['t %f %o'], 'x.png');
assert.throws(() => rule(art, spec, 'y.png'), /not yet supported/);
});
test('deps are part of identity', () => {
const [a] = rule('src/a.png', {deps: 'data/d.json', cmds: ['tool %f %o']}, ['out.png']);
const kOld = computeKey(a!.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j1'}));
const kNew = computeKey(a!.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j2'}));
const a = rule('src/a.png', {deps: 'data/d.json', cmds: ['tool %f %o']}, 'out.png');
const kOld = computeKey(a.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j1'}));
const kNew = computeKey(a.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j2'}));
assert.notEqual(kOld, kNew);
});
test('memo runs once', () => {
let calls = 0;
const f = memo(() => { calls++; return rule('a.png', ['c %f %o'], ['x.png']); });
assert.equal(f(), f());
assert.equal(calls, 1);
test('identical declarations return the existing artifacts', () => {
const first = rule('a.png', ['c %f %o'], 'x.png');
const second = rule('a.png', ['c %f %o'], 'x.png');
assert.equal(first, second);
assert.equal(getDecls().length, 1);
});
test('forEachRule dedupes per input, sharing overlap', () => {
const spec = {display: 'd %f', cmds: ['c %f %o']};
const first = forEachRule(['src/a.png', 'src/b.png'], spec, '%B.gif');
const second = forEachRule(['src/b.png', 'src/c.png'], spec, '%B.gif');
assert.equal(first[1], second[0]);
assert.equal(getDecls().length, 3);
});
test('declarations differing in display, cmds, or outputs stay distinct', () => {
rule('a.png', {display: 'one', cmds: ['c %f %o']}, 'x.png');
rule('a.png', {display: 'two', cmds: ['c %f %o']}, 'x.png');
rule('a.png', {display: 'one', cmds: ['other %f %o']}, 'x.png');
rule('a.png', {display: 'one', cmds: ['c %f %o']}, 'y.png');
assert.equal(getDecls().length, 4);
});
test('chained declarations dedupe through artifact inputs', () => {
const mid = rule('a.png', ['c %f %o'], 'mid.png');
const first = rule(mid, ['convert %f %o'], 'out.webp');
const second = rule(mid, ['convert %f %o'], 'out.webp');
assert.equal(first, second);
assert.equal(getDecls().length, 2);
});

33
tools/build/test/cas.test.ts
View File

@@ -6,7 +6,7 @@ import os from 'node:os';
import pathlib from 'node:path';
import {test} from 'node:test';
import {casExists, casInsert, casPath, casSweep} from '../cas.ts';
import {casExists, casInsert, casPath, casStat, casSweep} from '../cas.ts';
function makeTmpRoot() : string {
return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'cas-test-'));
@@ -21,37 +21,54 @@ function stage(root : string, data : string) : string {
test('casInsert stores by content digest, read-only', () => {
const root = makeTmpRoot();
const cas = pathlib.join(root, 'cas');
const digest = casInsert(cas, stage(root, 'hello'), 'png');
const {digest, size} = casInsert(cas, stage(root, 'hello'), 'png');
assert.equal(digest, createHash('sha256').update('hello').digest('hex'));
assert.equal(size, 5n);
const obj = casPath(cas, digest, 'png');
assert.equal(fs.readFileSync(obj, 'utf8'), 'hello');
assert.equal(fs.statSync(obj).mode & 0o777, 0o444);
assert.ok(casExists(cas, digest, 'png'));
assert.ok(!casExists(cas, digest, 'gif'));
assert.equal(casStat(cas, digest, 'png'), 5n);
assert.equal(casStat(cas, digest, 'gif'), null);
fs.rmSync(root, {recursive: true, force: true});
});
test('casInsert dedupes an existing object and discards the temp', () => {
const root = makeTmpRoot();
const cas = pathlib.join(root, 'cas');
const d1 = casInsert(cas, stage(root, 'same'), 'png');
const d1 = casInsert(cas, stage(root, 'same'), 'png').digest;
const tmp2 = stage(root, 'same');
const d2 = casInsert(cas, tmp2, 'png');
const d2 = casInsert(cas, tmp2, 'png').digest;
assert.equal(d1, d2);
assert.ok(!fs.existsSync(tmp2));
// Same bytes under a different extension is a distinct object.
const d3 = casInsert(cas, stage(root, 'same'), 'gif');
const d3 = casInsert(cas, stage(root, 'same'), 'gif').digest;
assert.equal(d1, d3);
assert.ok(casExists(cas, d1, 'png'));
assert.ok(casExists(cas, d1, 'gif'));
fs.rmSync(root, {recursive: true, force: true});
});
test('casInsert replaces a corrupt object instead of trusting it', () => {
const root = makeTmpRoot();
const cas = pathlib.join(root, 'cas');
const {digest} = casInsert(cas, stage(root, 'good bytes'), 'png');
const obj = casPath(cas, digest, 'png');
// Simulate a crash-truncated object under the same digest name.
fs.chmodSync(obj, 0o644);
fs.truncateSync(obj);
const again = casInsert(cas, stage(root, 'good bytes'), 'png');
assert.equal(again.digest, digest);
assert.equal(fs.readFileSync(obj, 'utf8'), 'good bytes');
assert.equal(fs.statSync(obj).mode & 0o777, 0o444);
fs.rmSync(root, {recursive: true, force: true});
});
test('casSweep removes non-live objects and prunes empty fanout dirs', () => {
const root = makeTmpRoot();
const cas = pathlib.join(root, 'cas');
const keep = casInsert(cas, stage(root, 'keep'), 'png');
const drop = casInsert(cas, stage(root, 'drop'), 'png');
const keep = casInsert(cas, stage(root, 'keep'), 'png').digest;
const drop = casInsert(cas, stage(root, 'drop'), 'png').digest;
const removed = casSweep(cas, new Set([`${keep}.png`]));
assert.equal(removed, 1);
assert.ok(casExists(cas, keep, 'png'));

43
tools/build/test/executor.test.ts
View File

@@ -42,7 +42,7 @@ function src(env : Env, name : string, content : string) : string {
// A copy rule that also counts its executions in env.execLog.
function copyRule(env : Env, input : string | Artifact, out : string,
extra : Partial<CmdSpec> = {}) : Artifact {
return rule(input, {cmds: [`cat %f > %o && echo x >> ${env.execLog}`], ...extra}, [out])[0]!;
return rule(input, {cmds: [`cat %f > %o && echo x >> ${env.execLog}`], ...extra}, out);
}
function execCount(env : Env) : number {
@@ -53,10 +53,11 @@ function execCount(env : Env) : number {
}
}
async function runBuild(env : Env, opts : {dryRun? : boolean, gc? : boolean} = {}) : Promise<DriveResult> {
async function runBuild(env : Env, opts : {dryRun? : boolean, gc? : boolean} = {},
decls = getDecls()) : Promise<DriveResult> {
const store = new Store(env.dbPath);
try {
return await build(getDecls(), {
return await build(decls, {
root: env.root,
store,
casDir: env.casDir,
@@ -147,7 +148,7 @@ test('byte-identical inputs share one execution across two declarations', async
// The consumer must be a *different* computation: an identical command over
// identical bytes would share the producer's key (by design).
function upcaseRule(env : Env, input : Artifact, out : string) : Artifact {
return rule(input, [`tr a-z A-Z < %f > %o && echo x >> ${env.execLog}`], [out])[0]!;
return rule(input, [`tr a-z A-Z < %f > %o && echo x >> ${env.execLog}`], out);
}
test('chained rules: consumer follows producer, cas-missing reruns alone', async () => {
@@ -177,7 +178,7 @@ test('chained rules: consumer follows producer, cas-missing reruns alone', async
test('failed producer blocks the consumer; nothing is recorded', async () => {
const env = setup();
const bad = rule(src(env, 'a.txt', 'x'), ['false'], ['mid.txt'])[0]!;
const bad = rule(src(env, 'a.txt', 'x'), ['false'], 'mid.txt');
copyRule(env, bad, 'final.txt');
const result = await runBuild(env);
assert.ok(!result.ok);
@@ -194,11 +195,11 @@ test('multi-output rules route %oN and skip all-or-nothing', async () => {
['x.txt', 'y.css']);
const [x1, y1] = declare();
await runBuild(env);
assert.equal(fs.readFileSync(casPath(env.casDir, x1!.hash, 'txt'), 'utf8'), 'one');
assert.equal(fs.readFileSync(casPath(env.casDir, y1!.hash, 'css'), 'utf8'), 'two');
assert.equal(fs.readFileSync(casPath(env.casDir, x1.hash, 'txt'), 'utf8'), 'one');
assert.equal(fs.readFileSync(casPath(env.casDir, y1.hash, 'css'), 'utf8'), 'two');
resetDecls();
fs.rmSync(casPath(env.casDir, y1!.hash, 'css'));
fs.rmSync(casPath(env.casDir, y1.hash, 'css'));
declare();
const rerun = await runBuild(env);
assert.deepEqual(statuses(rerun), ['ran']);
@@ -233,6 +234,32 @@ test('dry run reports without writing state', async () => {
store.close();
});
test('a truncated CAS object reads as dirty and is repaired', async () => {
const env = setup();
const out = copyRule(env, src(env, 'a.txt', 'truncate-me'), 'out.txt');
await runBuild(env);
const obj = casPath(env.casDir, out.hash, 'txt');
fs.chmodSync(obj, 0o644);
fs.truncateSync(obj);
resetDecls();
copyRule(env, pathlib.join(env.root, 'src/a.txt'), 'out.txt');
const rerun = await runBuild(env);
assert.deepEqual(statuses(rerun), ['ran']);
assert.equal(fs.readFileSync(obj, 'utf8'), 'truncate-me');
assert.equal(fs.statSync(obj).mode & 0o777, 0o444);
});
test('building a consumer alone pulls in and hashes its producer', async () => {
const env = setup();
const mid = copyRule(env, src(env, 'a.txt', 'solo'), 'mid.txt');
const final = rule(mid, [`tr a-z A-Z < %f > %o && echo x >> ${env.execLog}`], 'final.txt');
const result = await runBuild(env, {}, [final.decl]);
assert.ok(result.ok);
assert.equal(execCount(env), 2);
assert.equal(fs.readFileSync(casPath(env.casDir, final.hash, 'txt'), 'utf8'), 'SOLO');
});
test('missing sources fail upfront', async () => {
const env = setup();
copyRule(env, pathlib.join(env.root, 'src/nope.txt'), 'out.txt');

16
tools/build/test/store.test.ts
View File

@@ -16,17 +16,17 @@ function makeDbPath() : string {
test('recordRule/lookupRule roundtrip, upsert replaces outputs', () => {
const store = new Store(makeDbPath());
assert.equal(store.lookupRule('k1'), null);
store.recordRule('k1', ['cmd a'], [{digest: 'd1', ext: 'png'}, {digest: 'd2', ext: 'css'}]);
assert.deepEqual(store.lookupRule('k1'), [{digest: 'd1', ext: 'png'}, {digest: 'd2', ext: 'css'}]);
store.recordRule('k1', ['cmd a'], [{digest: 'd3', ext: 'png'}, {digest: 'd4', ext: 'css'}]);
assert.deepEqual(store.lookupRule('k1'), [{digest: 'd3', ext: 'png'}, {digest: 'd4', ext: 'css'}]);
store.recordRule('k1', ['cmd a'], [{digest: 'd1', ext: 'png', size: 1n}, {digest: 'd2', ext: 'css', size: 2n}]);
assert.deepEqual(store.lookupRule('k1'), [{digest: 'd1', ext: 'png', size: 1n}, {digest: 'd2', ext: 'css', size: 2n}]);
store.recordRule('k1', ['cmd a'], [{digest: 'd3', ext: 'png', size: 3n}, {digest: 'd4', ext: 'css', size: 4n}]);
assert.deepEqual(store.lookupRule('k1'), [{digest: 'd3', ext: 'png', size: 3n}, {digest: 'd4', ext: 'css', size: 4n}]);
store.close();
});
test('deleteKeysNotIn cascades outputs; liveObjects reflects survivors', () => {
const store = new Store(makeDbPath());
store.recordRule('keep', ['c'], [{digest: 'da', ext: 'png'}]);
store.recordRule('drop', ['c'], [{digest: 'db', ext: 'gif'}]);
store.recordRule('keep', ['c'], [{digest: 'da', ext: 'png', size: 1n}]);
store.recordRule('drop', ['c'], [{digest: 'db', ext: 'gif', size: 1n}]);
assert.deepEqual(store.liveObjects(), new Set(['da.png', 'db.gif']));
assert.equal(store.deleteKeysNotIn(new Set(['keep'])), 1);
assert.equal(store.lookupRule('drop'), null);
@@ -77,8 +77,8 @@ test('migrates a v1 db: drops rule tables, keeps file_cache', () => {
const store = new Store(dbPath);
assert.deepEqual([...store.loadFileCache().keys()], ['src/a.png']);
assert.equal(store.lookupRule('old'), null);
store.recordRule('new', ['c'], [{digest: 'd', ext: 'png'}]);
assert.deepEqual(store.lookupRule('new'), [{digest: 'd', ext: 'png'}]);
store.recordRule('new', ['c'], [{digest: 'd', ext: 'png', size: 1n}]);
assert.deepEqual(store.lookupRule('new'), [{digest: 'd', ext: 'png', size: 1n}]);
store.close();
const check = new Database(dbPath);

9
tools/deploy/api.ts
View File

@@ -71,8 +71,13 @@ export function makeCtx(casDir : string, queue : ActionQueue) : DeployCtx {
},
list(dir : string) : SrcFile[] {
const result = [];
for (const filename of fs.readdirSync(dir).sort()) {
const p = pathlib.path(filename, {dir});
// Files only, no dotfiles: the same filtering the build-side
// glob applies to rule inputs.
for (const ent of fs.readdirSync(dir, {withFileTypes: true}).sort((a, b) => a.name < b.name ? -1 : 1)) {
if (ent.name.startsWith('.') || (!ent.isFile() && !ent.isSymbolicLink())) {
continue;
}
const p = pathlib.path(ent.name, {dir});
result.push({...p, path: pathlib.format(p)});
}
return result;

11
tools/deploy/index.ts
View File

@@ -19,6 +19,7 @@ import {type DeploySpec, makeCtx} from './api.ts';
import {ActionQueue} from './queue.ts';
const root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..');
const invocationCwd = process.cwd();
process.chdir(root);
const DB_PATH = '.build/db.sqlite';
@@ -235,7 +236,15 @@ common(program.command('inspect <paths...>'))
.action(async (paths : string[], opts : CommonOpts & {output : string}) => {
setConfig(loadConfig(opts.config));
await importDeploys(discoverDeployFiles());
const targets = paths.map(p => nodePath.normalize(p).replace(/\/+$/, ''));
// Accept absolute paths and paths relative to where the user ran the
// command; rules declare repo-root-relative paths.
const targets = paths.map(p => {
const target = nodePath.relative(root, nodePath.resolve(invocationCwd, p));
if (target.startsWith('..')) {
throw new BuildError(`Not under the repo root: ${p}`);
}
return target;
});
const closure = new Set<RuleDecl>();
for (const decl of getDecls()) {

7
tools/deploy/queue.ts
View File

@@ -136,12 +136,13 @@ export class ActionQueue {
const dst = nodePath.join(dir, entry.dst);
fs.mkdirSync(nodePath.dirname(dst), {recursive: true});
if (op.type === 'Copy'){
if (mode === 'link') {
// Read-only sources are CAS objects; their mode must not
// leak into deploy trees (rsync -a would ship it), and a
// hardlink cannot carry its own mode, so copy those.
if (mode === 'link' && (fs.statSync(op.src).mode & 0o200) !== 0) {
fs.linkSync(op.src, dst);
} else {
fs.copyFileSync(op.src, dst);
// The copy inherits the source mode; CAS objects are
// read-only, which must not leak into deploy trees.
fs.chmodSync(dst, 0o644);
}
} else if (op.type === 'Write') {

10
tools/deploy/test/api.test.ts
View File

@@ -27,12 +27,12 @@ function shortHash(data : Buffer | string) : string {
// Stage `content` as a built artifact in a scratch CAS.
function makeArtifact(casDir : string, content : string, ext : string) {
const digest = createHash('sha256').update(content).digest('hex');
const [artifact] = rule('in.png', ['t %f %o'], [`art.${ext}`]);
artifact!.resolve(digest);
const artifact = rule('in.png', ['t %f %o'], `art.${ext}`);
artifact.resolve(digest);
const obj = casPath(casDir, digest, ext);
fs.mkdirSync(pathlib.dirname(obj), {recursive: true});
fs.writeFileSync(obj, content);
return artifact!;
return artifact;
}
test('ctx.hash matches the historical single-file stamp for artifacts and files', () => {
@@ -73,11 +73,13 @@ test('ctx queues artifact copies from the CAS, writes and reads', () => {
assert.equal((ops[1] as {op : {src : string}}).op.src, casPath(casDir, artifact.hash, 'webp'));
});
test('ctx.list sorts and parses extensions', () => {
test('ctx.list sorts, parses extensions, skips dotfiles and directories', () => {
const dir = tmpdir();
fs.writeFileSync(pathlib.join(dir, 'b.png'), '');
fs.writeFileSync(pathlib.join(dir, 'a.gif'), '');
fs.writeFileSync(pathlib.join(dir, 'noext'), '');
fs.writeFileSync(pathlib.join(dir, '.hidden'), '');
fs.mkdirSync(pathlib.join(dir, 'subdir'));
const ctx = makeCtx('cas', new ActionQueue());
assert.deepEqual(ctx.list(dir), [
{dir, name: 'a', ext: 'gif', path: pathlib.join(dir, 'a.gif')},