diff --git a/README.md b/README.md index 39871ee6..3206ce3f 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,10 @@ $ brew install imagemagick gifsicle advancecomp optipng pngquant webp Install dependencies once with `pnpm install`. Each deploy is a root `*.deploy.ts` module: it declares its build rules -(shared sets live in `rules/`) and a `finish` function that maps the built -artifacts to their published names. Build outputs are content-addressed: +(shared sets are plain functions in `rules/`; declaring an identical rule +twice is a no-op returning the existing artifacts, so any number of deploys +can call the same set) and a `finish` function that maps the built artifacts +to their published names. Build outputs are content-addressed: rules declare nominal output filenames but the store names every object by the hash of its bytes (under `.build/cas/`), so incrementality keys on content, same-byte renames rebuild nothing, and hash-stamped publishing diff --git a/assets.deploy.ts b/assets.deploy.ts index b827ab51..f069765d 100644 --- a/assets.deploy.ts +++ b/assets.deploy.ts @@ -1,6 +1,6 @@ import {gen6Padded, itemPadded} from './rules/minisprites.ts'; -import {itemspritecopy, type Manifest, newspritecopy, spritecopy, writeManifest} from './rules/publish.ts'; +import {Manifest, itemspritecopy, newspritecopy, spritecopy} from './rules/publish.ts'; import {forEachRule, rule} from './tools/build/artifact.ts'; import {spriteglob} from './tools/build/helpers.ts'; import {defineDeploy} from './tools/deploy/api.ts'; @@ -30,7 +30,7 @@ const [sheetPng, sheetCss] = rule(minispriteInputs, { cmds: ["node tools/smogdexspritesheet/index.ts --image %o1 --stylesheet %o2 -- %f"], }, ["spritesheet.png", "spritesheet.css"]); -const [sheetWebp] = rule(sheetPng!, ["cwebp -z 9 %f -o %o"], ["spritesheet.webp"]); +const sheetWebp = rule(sheetPng, ["cwebp -z 9 %f -o %o"], "spritesheet.webp"); // Forumsprites sources. Rules must be declared at import time (the build // runs before finish), so these calls cannot live inside finish(). @@ -47,9 +47,9 @@ export default defineDeploy({ // Dex spritesheet assets: hash-stamped css + webp. The css suffix // pointer rides in __meta/ for the dex to read. { - const wh = ctx.hash(sheetWebp!); - ctx.copy(sheetWebp!, `spritesheet-${wh}.webp`); - const src = ctx.read(sheetCss!); + const wh = ctx.hash(sheetWebp); + ctx.copy(sheetWebp, `spritesheet-${wh}.webp`); + const src = ctx.read(sheetCss); const css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`); if (css === src) { throw new Error("spritesheet.css: no webp urls rewritten"); @@ -57,7 +57,7 @@ export default defineDeploy({ // Suffix from source content: the rewritten css is a pure function // of (css, webp), so this changes exactly when the served bytes // change. - const ch = ctx.hash(sheetCss!, sheetWebp!); + const ch = ctx.hash(sheetCss, sheetWebp); ctx.write(`spritesheet-${ch}.css`, css); ctx.write("__meta/spritesheet_css_suffix.txt", `-${ch}\n`); } @@ -71,22 +71,22 @@ export default defineDeploy({ } { - const manifest : Manifest = {}; + const manifest = new Manifest(ctx); for (const f of forumItems) { - itemspritecopy(ctx, f, {dir: "forumsprites"}, manifest); + itemspritecopy(manifest, f, {dir: "forumsprites"}); } for (const f of forumG6) { - spritecopy(ctx, f, {dir: "forumsprites"}, true, manifest); + spritecopy(manifest, f, {dir: "forumsprites"}, true); } - writeManifest(ctx, "__meta/forumsprites/manifest.json", manifest); + manifest.write("__meta/forumsprites/manifest.json"); } { - const manifest : Manifest = {}; + const manifest = new Manifest(ctx); for (const f of ctx.list("src/pmd")) { - spritecopy(ctx, f, {dir: "pmd"}, false, manifest); + spritecopy(manifest, f, {dir: "pmd"}); } - writeManifest(ctx, "__meta/pmd/manifest.json", manifest); + manifest.write("__meta/pmd/manifest.json"); } }, }); diff --git a/ps.deploy.ts b/ps.deploy.ts index e2d468f9..61efffb3 100644 --- a/ps.deploy.ts +++ b/ps.deploy.ts @@ -1,9 +1,9 @@ import * as spritedata from '@smogon/sprite-data/index.ts'; -import {gen10Modelslike, gen5Gifs} from './rules/modelslike.ts'; +import {gen10Modelslike} from './rules/modelslike.ts'; import {type Sprite, toPSID} from './rules/publish.ts'; -import {forEachRule, memo, rule} from './tools/build/artifact.ts'; +import {forEachRule, rule} from './tools/build/artifact.ts'; import {PNG_DETERMINISTIC, base, compresspng, pad, spriteglob} from './tools/build/helpers.ts'; import {type DeployCtx, defineDeploy} from './tools/deploy/api.ts'; @@ -24,21 +24,21 @@ rule("ps-pokemon.sheet.mjs", { nameSensitive: true, deps: ["src/minisprites/pokemon/gen6/*", ...sheetDeps], cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], -}, ["pokemonicons-sheet.png"]); +}, "pokemonicons-sheet.png"); // TODO: reenable when trainers are moved // rule("ps-trainers.sheet.mjs", { // display: "ps trainers sheet", // nameSensitive: true, // cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], -// }, ["trainers-sheet.png"]); +// }, "trainers-sheet.png"); rule("ps-items.sheet.mjs", { display: "ps items sheet", nameSensitive: true, deps: ["src/minisprites/items/*", ...sheetDeps], cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], -}, ["itemicons-sheet.png"]); +}, "itemicons-sheet.png"); // PS pokeball icons; input order is the sheet order. @@ -52,38 +52,34 @@ rule([ `magick convert ${PNG_DETERMINISTIC} -background transparent -gravity center -extent 40x30 %f +append %o`, compresspng({config: "SPRITESHEET"}), ], -}, ["pokemonicons-pokeball-sheet.png"]); +}, "pokemonicons-pokeball-sheet.png"); // Padded Dex, plus missing CAPs backfilled from the gen5/model gifs. -const paddedDex = memo(() => { - const dex = forEachRule("src/dex/*", { - display: "pad dex %f", - cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})], - }, "%b"); +const dex = forEachRule("src/dex/*", { + display: "pad dex %f", + cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})], +}, "%b"); - const dexSet = new Set(dex.map(base)); - const dexMissing = []; - for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) { - if (!dexSet.has(base(file))) { - dexMissing.push(file); - dexSet.add(base(file)); - } +const dexSet = new Set(dex.map(base)); +const dexMissing = []; +for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) { + if (!dexSet.has(base(file))) { + dexMissing.push(file); + dexSet.add(base(file)); } +} - return dex.concat(forEachRule(dexMissing, { - display: "missing dex %B", - cmds: [ - `magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim %o`, - `magick mogrify ${PNG_DETERMINISTIC} -background transparent -gravity center -resize "120x120>" -extent 120x120 %o`, - compresspng({config: "DEX"}), - ], - }, "%B.png")); -}); -paddedDex(); +forEachRule(dexMissing, { + display: "missing dex %B", + cmds: [ + `magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim %o`, + `magick mogrify ${PNG_DETERMINISTIC} -background transparent -gravity center -resize "120x120>" -extent 120x120 %o`, + compresspng({config: "DEX"}), + ], +}, "%B.png"); const aniChampions = gen10Modelslike(); -gen5Gifs(); // PS ids keep the forme dash, unlike the smogon aliases. function psSpritecopy(ctx : DeployCtx, f : Sprite, dir : string) : void { diff --git a/rules/minisprites.ts b/rules/minisprites.ts index 0ee5ec3c..14a7bb36 100644 --- a/rules/minisprites.ts +++ b/rules/minisprites.ts @@ -1,25 +1,33 @@ -import {forEachRule, memo} from '../tools/build/artifact.ts'; +import {type Artifact, forEachRule} from '../tools/build/artifact.ts'; import {compresspng, pad, trimimg} from '../tools/build/helpers.ts'; // Uniform size minisprites -export const gen6Padded = memo(() => forEachRule("src/minisprites/pokemon/gen6/*.png", { - display: "pad g6 minisprite %f", - cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})], -}, "%b")); +export function gen6Padded() : Artifact[] { + return forEachRule("src/minisprites/pokemon/gen6/*.png", { + display: "pad g6 minisprite %f", + cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})], + }, "%b"); +} -export const itemPadded = memo(() => forEachRule("src/minisprites/items/*.png", { - display: "pad item minisprite %f", - cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})], -}, "%b")); +export function itemPadded() : Artifact[] { + return forEachRule("src/minisprites/items/*.png", { + display: "pad item minisprite %f", + cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})], + }, "%b"); +} -export const gen6Trimmed = memo(() => forEachRule("src/minisprites/pokemon/gen6/*.png", { - display: "trim g6 minisprite %f", - cmds: [trimimg(), compresspng({config: "MINISPRITE"})], -}, "%b")); +export function gen6Trimmed() : Artifact[] { + return forEachRule("src/minisprites/pokemon/gen6/*.png", { + display: "trim g6 minisprite %f", + cmds: [trimimg(), compresspng({config: "MINISPRITE"})], + }, "%b"); +} -export const itemTrimmed = memo(() => forEachRule("src/minisprites/items/*.png", { - display: "trim item minisprite %f", - cmds: [trimimg(), compresspng({config: "MINISPRITE"})], -}, "%b")); +export function itemTrimmed() : Artifact[] { + return forEachRule("src/minisprites/items/*.png", { + display: "trim item minisprite %f", + cmds: [trimimg(), compresspng({config: "MINISPRITE"})], + }, "%b"); +} diff --git a/rules/modelslike.ts b/rules/modelslike.ts index cd256b2c..d2a0ddc8 100644 --- a/rules/modelslike.ts +++ b/rules/modelslike.ts @@ -1,35 +1,41 @@ -import {forEachRule, memo} from '../tools/build/artifact.ts'; +import {type Artifact, forEachRule} from '../tools/build/artifact.ts'; // Gen 9 -export const gen9Modelslike = memo(() => forEachRule("src/gen9species/*.png", { - display: "96x96 %f", - // TODO, add customizable compression for gif - // ... or investigate using webp instead of both png/gif here - cmds: [ - "magick convert %f -trim +repage -resize 90x90 %o", - "gifsicle -O3 -b %o", - ], -}, "%B.gif")); +export function gen9Modelslike() : Artifact[] { + return forEachRule("src/gen9species/*.png", { + display: "96x96 %f", + // TODO, add customizable compression for gif + // ... or investigate using webp instead of both png/gif here + cmds: [ + "magick convert %f -trim +repage -resize 90x90 %o", + "gifsicle -O3 -b %o", + ], + }, "%B.gif"); +} // Gen 10 -export const gen10Modelslike = memo(() => forEachRule("src/champions/*.png", { - display: "96x96 %f", - // TODO, add customizable compression for gif - // ... or investigate using webp instead of both png/gif here - cmds: [ - "magick convert %f -trim +repage -resize 90x90 %o", - "gifsicle -O3 -b %o", - ], -}, "%B.gif")); +export function gen10Modelslike() : Artifact[] { + return forEachRule("src/champions/*.png", { + display: "96x96 %f", + // TODO, add customizable compression for gif + // ... or investigate using webp instead of both png/gif here + cmds: [ + "magick convert %f -trim +repage -resize 90x90 %o", + "gifsicle -O3 -b %o", + ], + }, "%B.gif"); +} // Gen 5 CAPs... -export const gen5Gifs = memo(() => forEachRule("src/sprites/gen5/*.png", [ - // TODO, add customizable compression for gif - // ... or investigate using webp instead of both png/gif here - "magick convert %f %o", - "gifsicle -O3 -b %o", -], "%B.gif")); +export function gen5Gifs() : Artifact[] { + return forEachRule("src/sprites/gen5/*.png", [ + // TODO, add customizable compression for gif + // ... or investigate using webp instead of both png/gif here + "magick convert %f %o", + "gifsicle -O3 -b %o", + ], "%B.gif"); +} diff --git a/rules/publish.ts b/rules/publish.ts index 84c6146c..9f9bedab 100644 --- a/rules/publish.ts +++ b/rules/publish.ts @@ -8,7 +8,32 @@ import type {DeployCtx, SrcFile} from '../tools/deploy/api.ts'; // or a raw source file. export type Sprite = Artifact | SrcFile; -export type Manifest = Record; +// The unhashed -> hashed name mapping published beside a stamped set. +export class Manifest { + readonly ctx : DeployCtx; + private entries = new Map(); + + constructor(ctx : DeployCtx) { + this.ctx = ctx; + } + + set(key : string, value : string) : void { + // ActionQueue only dedups final dsts; hashed dsts differ even when + // unhashed names collide, so check the key explicitly. + if (this.entries.has(key)) { + throw new Error(`duplicate sprite name ${key}`); + } + this.entries.set(key, value); + } + + write(dst : string) : void { + const sorted : Record = {}; + for (const k of [...this.entries.keys()].sort()) { + sorted[k] = this.entries.get(k)!; + } + this.ctx.write(dst, JSON.stringify(sorted, null, 4) + "\n"); + } +} export interface Dest { dir : string; @@ -35,38 +60,14 @@ export function toPSID(name : string) : string { // Copy with a content-hash-stamped name and record the unhashed -> hashed // mapping in `manifest`. -export function stampcopy(ctx : DeployCtx, f : Sprite, {dir, ext}: Dest, name : string, - manifest : Manifest) : void { - const key = `${name}.${extOf(f, ext)}`; - // ActionQueue only dedups final dsts; hashed dsts differ even when - // unhashed names collide, so check the manifest key explicitly. - if (manifest[key] !== undefined) { - throw new Error(`duplicate sprite name ${key}`); - } - const h = ctx.hash(f); - manifest[key] = `${name}-${h}.${extOf(f, ext)}`; - ctx.copy(f, `${dir}/${name}-${h}.${extOf(f, ext)}`); +export function stampcopy(manifest : Manifest, f : Sprite, {dir, ext}: Dest, name : string) : void { + const h = manifest.ctx.hash(f); + manifest.set(`${name}.${extOf(f, ext)}`, `${name}-${h}.${extOf(f, ext)}`); + manifest.ctx.copy(f, `${dir}/${name}-${h}.${extOf(f, ext)}`); } -export function writeManifest(ctx : DeployCtx, dst : string, manifest : Manifest) : void { - const sorted : Manifest = {}; - for (const k of Object.keys(manifest).sort()) { - sorted[k] = manifest[k]!; - } - ctx.write(dst, JSON.stringify(sorted, null, 4) + "\n"); -} - -function emit(ctx : DeployCtx, f : Sprite, dest : Dest, name : string, - manifest : Manifest | null) : void { - if (manifest) { - stampcopy(ctx, f, dest, name, manifest); - } else { - ctx.copy(f, `${dest.dir}/${name}.${extOf(f, dest.ext)}`); - } -} - -export function spritecopy(ctx : DeployCtx, f : Sprite, dest : Dest, - allowUnknown = false, manifest : Manifest | null = null) : void { +export function spritecopy(manifest : Manifest, f : Sprite, dest : Dest, + allowUnknown = false) : void { const sn = spritedata.parseFilename(f.name); let name : string; @@ -99,12 +100,11 @@ export function spritecopy(ctx : DeployCtx, f : Sprite, dest : Dest, name += "-gmax"; } - emit(ctx, f, dest, name, manifest); + stampcopy(manifest, f, dest, name); } // TODO: merge with above -export function itemspritecopy(ctx : DeployCtx, f : Sprite, dest : Dest, - manifest : Manifest | null = null) : void { +export function itemspritecopy(manifest : Manifest, f : Sprite, dest : Dest) : void { const sn = spritedata.parseFilename(f.name); if (sn.extension) { throw new Error(`Not an item sprite: ${f.name}`); @@ -114,7 +114,7 @@ export function itemspritecopy(ctx : DeployCtx, f : Sprite, dest : Dest, throw new Error(`Not an item sprite: ${f.name}`); } for (const n of sd.names) { - emit(ctx, f, dest, toSmogonAlias(n), manifest); + stampcopy(manifest, f, dest, toSmogonAlias(n)); } } diff --git a/rules/social.ts b/rules/social.ts index c3fd5cd3..eefc1d8e 100644 --- a/rules/social.ts +++ b/rules/social.ts @@ -1,11 +1,11 @@ -import {forEachRule, memo} from '../tools/build/artifact.ts'; +import {type Artifact, forEachRule} from '../tools/build/artifact.ts'; import {PNG_DETERMINISTIC, base, compresspng, spriteglob} from '../tools/build/helpers.ts'; // Smogdex social images: models, backfilled with gen9 species not yet in // models (first source wins). -const socialInputs = memo(() => { +function socialInputs() : string[] { const social = spriteglob(["src/models/*"], {b: false, s: false}); const socialSeen = new Set(social.map(base)); for (const file of spriteglob(["src/gen9species/*"], {b: false, s: false})) { @@ -15,20 +15,24 @@ const socialInputs = memo(() => { } } return social; -}); +} -export const fbSprites = memo(() => forEachRule(socialInputs(), { - display: "fbsprite %f", - cmds: [ - `magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o`, - compresspng({config: "MODELS"}), - ], -}, "%B.png")); +export function fbSprites() : Artifact[] { + return forEachRule(socialInputs(), { + display: "fbsprite %f", + cmds: [ + `magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o`, + compresspng({config: "MODELS"}), + ], + }, "%B.png"); +} -export const twitterSprites = memo(() => forEachRule(socialInputs(), { - display: "twittersprite %f", - cmds: [ - `magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim -resize 115x115 -background white -gravity center -extent 120x120 %o`, - compresspng({config: "MODELS"}), - ], -}, "%B.png")); +export function twitterSprites() : Artifact[] { + return forEachRule(socialInputs(), { + display: "twittersprite %f", + cmds: [ + `magick convert "%f[0]" ${PNG_DETERMINISTIC} -trim -resize 115x115 -background white -gravity center -extent 120x120 %o`, + compresspng({config: "MODELS"}), + ], + }, "%B.png"); +} diff --git a/smogon.deploy.ts b/smogon.deploy.ts index cd79c0ce..ec017211 100644 --- a/smogon.deploy.ts +++ b/smogon.deploy.ts @@ -1,29 +1,30 @@ -import {gen6Trimmed, itemTrimmed} from './rules/minisprites.ts'; +import {gen6Trimmed} from './rules/minisprites.ts'; import {gen10Modelslike, gen5Gifs, gen9Modelslike} from './rules/modelslike.ts'; -import {itemspritecopy, type Manifest, type Sprite, spritecopy, writeManifest} from './rules/publish.ts'; -import {fbSprites, twitterSprites} from './rules/social.ts'; +import {Manifest, type Sprite, spritecopy} from './rules/publish.ts'; import {defineDeploy} from './tools/deploy/api.ts'; const xyModels = gen9Modelslike(); const xyChampions = gen10Modelslike(); const xyGen5 = gen5Gifs(); const xyIcons = gen6Trimmed(); -const xyItems = itemTrimmed(); -const fb = fbSprites(); -const twitter = twitterSprites(); + +// Deprecated, unstamped sets: +// const xyItems = itemTrimmed(); (rules/minisprites.ts) +// const fb = fbSprites(); (rules/social.ts) +// const twitter = twitterSprites(); (rules/social.ts) export default defineDeploy({ finish(ctx) { // xy/ animations: first source wins per sprite name. const seenModels = new Set(); - const xyManifest : Manifest = {}; + const xyManifest = new Manifest(ctx); const xycopy = (f : Sprite) => { if (seenModels.has(f.name)) { return; } seenModels.add(f.name); - spritecopy(ctx, f, {dir: "xy"}, false, xyManifest); + spritecopy(xyManifest, f, {dir: "xy"}); }; for (const f of ctx.list("src/models")) { @@ -44,26 +45,25 @@ export default defineDeploy({ for (const f of xyGen5) { xycopy(f); } - writeManifest(ctx, "xy/manifest.json", xyManifest); + xyManifest.write("xy/manifest.json"); { - const manifest : Manifest = {}; + const manifest = new Manifest(ctx); for (const f of xyIcons) { - spritecopy(ctx, f, {dir: "xyicons"}, false, manifest); + spritecopy(manifest, f, {dir: "xyicons"}); } - writeManifest(ctx, "xyicons/manifest.json", manifest); + manifest.write("xyicons/manifest.json"); } - for (const f of xyItems) { - itemspritecopy(ctx, f, {dir: "xyitems"}); - } - - for (const f of fb) { - spritecopy(ctx, f, {dir: "fbsprites/xy"}); - } - - for (const f of twitter) { - spritecopy(ctx, f, {dir: "twittersprites/xy"}); - } + // Deprecated, unstamped: + // for (const f of xyItems) { + // itemspritecopy(?, f, {dir: "xyitems"}); + // } + // for (const f of fb) { + // spritecopy(?, f, {dir: "fbsprites/xy"}); + // } + // for (const f of twitter) { + // spritecopy(?, f, {dir: "twittersprites/xy"}); + // } }, }); diff --git a/tools/build/artifact.ts b/tools/build/artifact.ts index e49c1da1..01ae0e6f 100644 --- a/tools/build/artifact.ts +++ b/tools/build/artifact.ts @@ -3,7 +3,7 @@ import pathlib from 'path'; import {createHash} from 'crypto'; import {astable, glob} from './helpers.ts'; -import {type Cmd, basenameNoExt, flattenCmds, substitute} from './subst.ts'; +import {type Cmd, basenameNoExt, flattenCmds, substitute, substituteNames} from './subst.ts'; // A rule's output: content-addressed bytes with a nominal name. The nominal // name exists for provenance, inspection, and deploy-time naming; it is NOT @@ -65,6 +65,7 @@ export interface CmdSpec { } export interface RuleDecl { + id : number; // registration order; identity for artifact inputs inputs : Input[]; // ordered (%f order; some rules are order-sensitive) deps : Input[]; outputs : Artifact[]; @@ -75,6 +76,9 @@ export interface RuleDecl { } let decls : RuleDecl[] = []; +// Declaring an identical rule twice returns the existing artifacts, so +// shared rule sets are plain functions that any number of deploys may call. +let declIndex = new Map(); export function getDecls() : RuleDecl[] { return decls; @@ -82,20 +86,7 @@ export function getDecls() : RuleDecl[] { export function resetDecls() : void { decls = []; -} - -// Share a rule set between deploys: importing declares nothing, the first -// call declares once, later calls return the same artifacts. -export function memo(fn : () => T) : () => T { - let called = false; - let value : T; - return () => { - if (!called) { - value = fn(); - called = true; - } - return value!; - }; + declIndex = new Map(); } // Nominal path of an input, for %b/%B and displays. @@ -123,7 +114,9 @@ export function computeKey(decl : RuleDecl, digestOf : (i : Input) => string) : const h = createHash('sha256'); h.update([ 'v2', - decl.cmds.join('\n'), + // \0, not \n: a single multi-line command executes differently from + // the same lines as separate ' && '-joined commands. + decl.cmds.join('\0'), decl.inputs.map(digestOf).join('\0'), decl.inputs.map(extOf).join('\0'), decl.deps.map(digestOf).join('\0'), @@ -146,11 +139,36 @@ function resolveInputs(input : Input | Input[] | undefined) : Input[] { } function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : string[]) : RuleDecl { - const cmds = flattenCmds(spec.cmds); + const nominalInputs = inputs.map(nominal); + // %b/%B expand to nominal input names, never CAS paths, so they are + // resolved at declaration. This also lands them in the identity key: a + // command embedding input names is name-dependent by construction. + const cmds = flattenCmds(spec.cmds).map(c => substituteNames(c, nominalInputs)); if (cmds.length === 0) { throw new Error(`Rule with no commands (outputs: ${outputs.join(' ')})`); } + if (outputs.length === 0) { + throw new Error(`Rule with no outputs (cmds: ${cmds[0]!})`); + } + + // An identical declaration returns the already-registered rule. Artifact + // inputs are keyed by their producing rule's id, so chains dedupe too. + const token = (i : Input) => typeof i === 'string' ? `s:${i}` : `a:${i.decl.id}:${i.index}`; + const identity = [ + cmds.join('\0'), + inputs.map(token).join('\0'), + deps.map(token).join('\0'), + outputs.join('\0'), + spec.display ?? '', + spec.nameSensitive ? '1' : '0', + ].join('\x01'); + const existing = declIndex.get(identity); + if (existing !== undefined) { + return existing; + } + const decl : RuleDecl = { + id: decls.length, inputs, deps, outputs: [], @@ -168,6 +186,10 @@ function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : st if (out.includes('/') || out.includes('%')) { throw new Error(`Rule outputs are nominal filenames, no paths or substitutions: ${out}`); } + if (/[\s'"\\]/.test(out)) { + // Output names are substituted into shell commands unquoted. + throw new Error(`Rule output name has shell-hostile characters: ${out}`); + } if (outputs.indexOf(out) !== index) { throw new Error(`Duplicate rule output: ${out}`); } @@ -179,7 +201,6 @@ function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : st }); // Validate substitutions now (unknown escapes, out-of-range %oN) rather // than at execution; the results are discarded. - const nominalInputs = inputs.map(nominal); const nominalOutputs = decl.outputs.map(o => o.filename); for (const cmd of cmds) { substitute(cmd, nominalInputs, nominalOutputs); @@ -188,13 +209,23 @@ function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : st decl.display = substitute(spec.display, nominalInputs, nominalOutputs); } decls.push(decl); + declIndex.set(identity, decl); return decl; } +// One Artifact per declared output, as a tuple when the output list is a +// literal, so `const [png, css] = rule(...)` needs no undefined checks. A +// single string output returns its Artifact directly. export function rule(input : Input | Input[], spec : CmdSpec | Cmd[], - output : string | string[]) : Artifact[] { + output : string) : Artifact; +export function rule( + input : Input | Input[], spec : CmdSpec | Cmd[], + output : T) : {[K in keyof T] : Artifact}; +export function rule(input : Input | Input[], spec : CmdSpec | Cmd[], + output : string | readonly string[]) : Artifact | Artifact[] { const s = normalizeSpec(spec); - return makeDecl(resolveInputs(input), resolveInputs(s.deps), s, astable(output)).outputs; + const decl = makeDecl(resolveInputs(input), resolveInputs(s.deps), s, astable(output)); + return typeof output === 'string' ? decl.outputs[0]! : decl.outputs; } export function forEachRule(input : Input | Input[], spec : CmdSpec | Cmd[], diff --git a/tools/build/cas.ts b/tools/build/cas.ts index ec6bcffa..16e71705 100644 --- a/tools/build/cas.ts +++ b/tools/build/cas.ts @@ -16,26 +16,49 @@ export function casPath(casDir : string, digest : string, ext : string) : string } export function casExists(casDir : string, digest : string, ext : string) : boolean { + return casStat(casDir, digest, ext) !== null; +} + +// Size of an object, or null if absent. Callers verify it against the +// recorded size: a crash between rename and data flush can leave a +// truncated object, which must read as dirty, not clean. +export function casStat(casDir : string, digest : string, ext : string) : bigint | null { try { - return fs.statSync(casPath(casDir, digest, ext)).isFile(); + const st = fs.statSync(casPath(casDir, digest, ext), {bigint: true}); + return st.isFile() ? st.size : null; } catch { - return false; + return null; } } -// Move tmpPath into the store, returning the content digest (hex). If the -// object already exists the temp file is simply discarded. -export function casInsert(casDir : string, tmpPath : string, ext : string) : string { +export interface CasObject { + digest : string; // sha256 hex of the bytes + size : bigint; +} + +// Move tmpPath into the store, returning the content digest and size. An +// existing object is trusted only if its bytes actually hash to the digest; +// otherwise (crash-truncated object) the fresh bytes replace it. +export function casInsert(casDir : string, tmpPath : string, ext : string) : CasObject { const digest = hashFileSync(tmpPath).toString('hex'); + const size = fs.statSync(tmpPath, {bigint: true}).size; const target = casPath(casDir, digest, ext); - if (fs.existsSync(target)) { + if (fs.existsSync(target) && hashFileSync(target).toString('hex') === digest) { fs.unlinkSync(tmpPath); - return digest; + return {digest, size}; } fs.mkdirSync(pathlib.dirname(target), {recursive: true}); fs.chmodSync(tmpPath, 0o444); + // Flush the bytes before the rename becomes visible, so a power loss + // cannot journal the rename while dropping the data pages. + const fd = fs.openSync(tmpPath, 'r'); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } fs.renameSync(tmpPath, target); - return digest; + return {digest, size}; } // Remove every object not in `live` (keys are ".", the object diff --git a/tools/build/driver.ts b/tools/build/driver.ts index c386de40..a1b27377 100644 --- a/tools/build/driver.ts +++ b/tools/build/driver.ts @@ -34,14 +34,24 @@ export async function build(decls : readonly RuleDecl[], opts : BuildOpts) : Pro const logError = opts.logError ?? console.error; const {store} = opts; + // Hash sources over the producer closure: demanding a rule demands the + // producers of its artifact inputs, even ones outside `decls`. + const closure = new Set(); const sources = new Set(); - for (const decl of decls) { + const add = (decl : RuleDecl) => { + if (closure.has(decl)) { + return; + } + closure.add(decl); for (const input of [...decl.inputs, ...decl.deps]) { if (typeof input === 'string') { sources.add(input); + } else { + add(input.decl); } } - } + }; + decls.forEach(add); const {hashes, updated, missing} = reconcileHashes(sources, store.loadFileCache()); if (missing.length > 0) { throw new BuildError(`Missing input files:\n ${missing.slice(0, 20).join('\n ')}` diff --git a/tools/build/executor.ts b/tools/build/executor.ts index 386c9e77..2d47d525 100644 --- a/tools/build/executor.ts +++ b/tools/build/executor.ts @@ -3,7 +3,7 @@ import fs from 'fs'; import pathlib from 'path'; import {type Input, type RuleDecl, computeKey} from './artifact.ts'; -import {casExists, casInsert, casPath} from './cas.ts'; +import {casInsert, casPath, casStat} from './cas.ts'; import {BuildError} from './errors.ts'; import {runShell} from './exec.ts'; import {type Store} from './store.ts'; @@ -118,7 +118,19 @@ export class Executor { private demand(decl : RuleDecl) : Promise { let p = this.memo.get(decl); if (p === undefined) { - p = this.demandInner(decl); + // Any non-sentinel escape (a store error, a resolve conflict, a + // bug) must surface as a reported failure, not vanish into the + // allSettled in build(). + p = this.demandInner(decl).catch(err => { + if (err instanceof RuleFailed || err instanceof DryDirty || err instanceof Aborted) { + throw err; + } + this.outcomes.set(decl, {status: 'failed', message: 'internal error'}); + this.logError(`FAILED (internal error): ${label(decl)}`); + this.logError(indent(err instanceof Error ? err.stack ?? err.message : String(err))); + this.failAc.abort(); + throw new RuleFailed(); + }); this.memo.set(decl, p); } return p; @@ -185,7 +197,7 @@ export class Executor { if (stored !== null && stored.length === decl.outputs.length && stored.every((o, n) => o.ext === decl.outputs[n]!.ext) - && stored.every(o => casExists(casDir, o.digest, o.ext))) { + && stored.every(o => casStat(casDir, o.digest, o.ext) === o.size)) { this.outcomes.set(decl, {status: 'clean'}); return stored.map(o => o.digest); } @@ -203,17 +215,6 @@ export class Executor { throw new Aborted(); } return await this.execute(decl, key, reason); - } catch (err) { - if (err instanceof Aborted || err instanceof RuleFailed) { - throw err; - } - // Unexpected (infrastructure) error: count the rule failed and - // stop scheduling; something systemic is wrong. - this.outcomes.set(decl, {status: 'failed', message: 'internal error'}); - this.logError(`FAILED (internal error): ${label(decl)}`); - this.logError(indent(err instanceof Error ? err.stack ?? err.message : String(err))); - this.failAc.abort(); - throw new RuleFailed(); } finally { this.semaphore.release(); } @@ -236,10 +237,10 @@ export class Executor { if (result.code === 0) { const missing = tempOutputs.filter(p => !fs.existsSync(p)); if (missing.length === 0) { - const outputs = decl.outputs.map((o, n) => ({ - digest: casInsert(casDir, tempOutputs[n]!, o.ext), - ext: o.ext, - })); + const outputs = decl.outputs.map((o, n) => { + const object = casInsert(casDir, tempOutputs[n]!, o.ext); + return {digest: object.digest, ext: o.ext, size: object.size}; + }); store.recordRule(key, decl.cmds, outputs); this.outcomes.set(decl, {status: 'ran', reason}); this.log(`[${++this.counter}] ${label(decl)}` diff --git a/tools/build/helpers.ts b/tools/build/helpers.ts index 6bc6d0f0..a5b9094c 100644 --- a/tools/build/helpers.ts +++ b/tools/build/helpers.ts @@ -16,11 +16,11 @@ export function getconfig(name : string) : string | undefined { return value === '' ? undefined : value; } -export function astable(x : string | string[] | undefined) : string[] { +export function astable(x : string | readonly string[] | undefined) : string[] { if (x === undefined) { return []; } - return typeof x === 'string' ? [x] : x; + return typeof x === 'string' ? [x] : [...x]; } // Single-directory, single-'*' glob (all Tupfile patterns were of this form). diff --git a/tools/build/store.ts b/tools/build/store.ts index f87fdf23..13dc6e21 100644 --- a/tools/build/store.ts +++ b/tools/build/store.ts @@ -9,6 +9,7 @@ import type {FileStat} from './hash.ts'; export interface StoredOutput { digest : string; // sha256 hex of the bytes; the CAS object is . ext : string; + size : bigint; // verified against the object on every clean check } const DDL = ` @@ -33,6 +34,7 @@ CREATE TABLE IF NOT EXISTS rule_outputs ( ord INTEGER NOT NULL, digest TEXT NOT NULL, ext TEXT NOT NULL, + size INTEGER NOT NULL, PRIMARY KEY (rule_id, ord) ); `; @@ -111,8 +113,8 @@ export class Store { if (rule === undefined) { return null; } - return this.db.prepare<[bigint], {digest : string, ext : string}>( - 'SELECT digest, ext FROM rule_outputs WHERE rule_id = ? ORDER BY ord').all(rule.id); + return this.db.prepare<[bigint], StoredOutput>( + 'SELECT digest, ext, size FROM rule_outputs WHERE rule_id = ? ORDER BY ord').all(rule.id); })(); } @@ -125,11 +127,11 @@ export class Store { RETURNING id`); const delOutputs = this.db.prepare('DELETE FROM rule_outputs WHERE rule_id = ?'); const insOutput = this.db.prepare( - 'INSERT INTO rule_outputs (rule_id, ord, digest, ext) VALUES (?, ?, ?, ?)'); + 'INSERT INTO rule_outputs (rule_id, ord, digest, ext, size) VALUES (?, ?, ?, ?, ?)'); this.db.transaction(() => { const {id} = upsert.get(key, cmds.join('\n'))!; delOutputs.run(id); - outputs.forEach((o, i) => insOutput.run(id, i, o.digest, o.ext)); + outputs.forEach((o, i) => insOutput.run(id, i, o.digest, o.ext, o.size)); })(); } diff --git a/tools/build/subst.ts b/tools/build/subst.ts index 30f07096..59a0bec5 100644 --- a/tools/build/subst.ts +++ b/tools/build/subst.ts @@ -19,6 +19,15 @@ export function basenameNoExt(path : string) : string { return dot > 0 ? base.slice(0, dot) : base; } +// Expand only the name substitutions (%b/%B). They are static per rule, so +// declarations expand them eagerly; %f/%o/%oN wait for execution, when +// concrete paths exist. +export function substituteNames(s : string, inputs : string[]) : string { + return s.replace(/%([bB])/g, (_, c : string) => + c === 'b' ? inputs.map(p => pathlib.basename(p)).join(' ') + : inputs.map(basenameNoExt).join(' ')); +} + // Tup-style substitutions: // %f inputs, space-joined %b input basenames // %o outputs, space-joined %B input basenames without extension diff --git a/tools/build/test/artifact.test.ts b/tools/build/test/artifact.test.ts index a6053def..45f99dc0 100644 --- a/tools/build/test/artifact.test.ts +++ b/tools/build/test/artifact.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import {beforeEach, test} from 'node:test'; -import {type Input, computeKey, forEachRule, memo, resetDecls, rule} from '../artifact.ts'; +import {type Input, computeKey, forEachRule, getDecls, resetDecls, rule} from '../artifact.ts'; beforeEach(resetDecls); @@ -19,27 +19,56 @@ function digests(map : Record) : (i : Input) => string { test('rule declares artifacts with nominal names', () => { const [png, css] = rule('src/a.png', ['tool %f %o1 %o2'], ['sheet.png', 'sheet.css']); - assert.equal(png!.name, 'sheet'); - assert.equal(png!.ext, 'png'); - assert.equal(css!.filename, 'sheet.css'); - assert.deepEqual(png!.sources, ['src/a.png']); - assert.equal(png!.decl, css!.decl); - assert.throws(() => png!.hash, /not been built/); - png!.resolve('d1'); - assert.equal(png!.hash, 'd1'); - assert.throws(() => png!.resolve('d2'), /resolved twice/); + assert.equal(png.name, 'sheet'); + assert.equal(png.ext, 'png'); + assert.equal(css.filename, 'sheet.css'); + assert.deepEqual(png.sources, ['src/a.png']); + assert.equal(png.decl, css.decl); + assert.throws(() => png.hash, /not been built/); + png.resolve('d1'); + assert.equal(png.hash, 'd1'); + assert.throws(() => png.resolve('d2'), /resolved twice/); +}); + +test('a single string output returns its artifact directly', () => { + const out = rule('src/a.png', ['tool %f %o'], 'only.png'); + assert.equal(out.filename, 'only.png'); }); test('rule rejects paths, substitutions, missing extensions in outputs', () => { assert.throws(() => rule('a.png', ['c'], ['dir/x.png']), /nominal filenames/); assert.throws(() => rule('a.png', ['c'], ['%B.png']), /nominal filenames/); assert.throws(() => rule('a.png', ['c'], ['noext']), /needs an extension/); + assert.throws(() => rule('a.png', ['c'], []), /no outputs/); + assert.throws(() => rule('a.png', ['c'], ['has space.png']), /shell-hostile/); + assert.throws(() => rule('a.png', ['c'], ['quote"d.png']), /shell-hostile/); +}); + +test('%b/%B in commands expand to nominal names at declaration', () => { + const art = rule('src/dir/a.png', ['tool %f %o'], 'mid.png'); + const out = rule(art, ['emit --name %B %f %o'], 'x.css'); + // Expanded eagerly (never a CAS basename), and thus part of the key. + assert.deepEqual(out.decl.cmds, ['emit --name mid %f %o']); + art.resolve('d1'); + const renamed = rule('src/dir/b.png', ['tool %f %o'], 'other.png'); + const out2 = rule(renamed, ['emit --name %B %f %o'], 'x.css'); + renamed.resolve('d1'); + assert.notEqual( + computeKey(out.decl, digests({})), + computeKey(out2.decl, digests({}))); +}); + +test('a multi-line command and split commands get different keys', () => { + const a = rule('a.png', ['one\ntwo'], 'x.png'); + const b = rule('a.png', ['one', 'two'], 'x.png'); + const map = {'a.png': 'd1'}; + assert.notEqual(computeKey(a.decl, digests(map)), computeKey(b.decl, digests(map))); }); test('rule validates command substitutions at declaration', () => { - assert.throws(() => rule('a.png', ['tool %q'], ['x.png']), /Unknown substitution/); - assert.throws(() => rule('a.png', ['tool %o2'], ['x.png']), /out of range/); - assert.throws(() => rule('a.png', [''], ['x.png']), /no commands/); + assert.throws(() => rule('a.png', ['tool %q'], 'x.png'), /Unknown substitution/); + assert.throws(() => rule('a.png', ['tool %o2'], 'x.png'), /out of range/); + assert.throws(() => rule('a.png', [''], 'x.png'), /no commands/); }); test('forEachRule declares one rule per input, %b/%B templates', () => { @@ -50,54 +79,78 @@ test('forEachRule declares one rule per input, %b/%B templates', () => { }); test('chained rules accept artifacts as inputs', () => { - const [png] = rule('src/a.png', ['tool %f %o'], ['x.png']); - const [webp] = rule(png!, ['cwebp %f -o %o'], ['x.webp']); - assert.equal(webp!.decl.inputs[0], png); - assert.deepEqual(webp!.sources, []); - png!.resolve('dp'); - const key = computeKey(webp!.decl, digests({})); + const png = rule('src/a.png', ['tool %f %o'], 'x.png'); + const webp = rule(png, ['cwebp %f -o %o'], 'x.webp'); + assert.equal(webp.decl.inputs[0], png); + assert.deepEqual(webp.sources, []); + png.resolve('dp'); + const key = computeKey(webp.decl, digests({})); assert.equal(typeof key, 'string'); }); test('key ignores input paths but not bytes, order, exts, or commands', () => { - const [a] = rule('src/a.png', ['convert %f %o'], ['out.png']); - const [b] = rule('src/elsewhere/z.png', ['convert %f %o'], ['other.png']); - const kA = computeKey(a!.decl, digests({'src/a.png': 'd1'})); + const a = rule('src/a.png', ['convert %f %o'], 'out.png'); + const b = rule('src/elsewhere/z.png', ['convert %f %o'], 'other.png'); + const kA = computeKey(a.decl, digests({'src/a.png': 'd1'})); // Renamed source, same bytes, different nominal output: same key. - assert.equal(kA, computeKey(b!.decl, digests({'src/elsewhere/z.png': 'd1'}))); + assert.equal(kA, computeKey(b.decl, digests({'src/elsewhere/z.png': 'd1'}))); // Different bytes: different key. - assert.notEqual(kA, computeKey(b!.decl, digests({'src/elsewhere/z.png': 'd2'}))); + assert.notEqual(kA, computeKey(b.decl, digests({'src/elsewhere/z.png': 'd2'}))); - const [two] = rule(['x.png', 'y.png'], ['join %f %o'], ['out.png']); - const [reversed] = rule(['y.png', 'x.png'], ['join %f %o'], ['out.png']); + const two = rule(['x.png', 'y.png'], ['join %f %o'], 'out.png'); + const reversed = rule(['y.png', 'x.png'], ['join %f %o'], 'out.png'); const map = {'x.png': 'dx', 'y.png': 'dy'}; - assert.notEqual(computeKey(two!.decl, digests(map)), computeKey(reversed!.decl, digests(map))); + assert.notEqual(computeKey(two.decl, digests(map)), computeKey(reversed.decl, digests(map))); - const [gif] = rule('src/a.png', ['convert %f %o'], ['out.gif']); - assert.notEqual(kA, computeKey(gif!.decl, digests({'src/a.png': 'd1'}))); + const gif = rule('src/a.png', ['convert %f %o'], 'out.gif'); + assert.notEqual(kA, computeKey(gif.decl, digests({'src/a.png': 'd1'}))); }); test('nameSensitive keys on paths; rejects artifact inputs', () => { const spec = {nameSensitive: true, cmds: ['tool %f %o']}; - const [a] = rule('src/a.png', spec, ['out.png']); - const [b] = rule('src/b.png', spec, ['out.png']); + const a = rule('src/a.png', spec, 'out.png'); + const b = rule('src/b.png', spec, 'out.png'); assert.notEqual( - computeKey(a!.decl, digests({'src/a.png': 'd1'})), - computeKey(b!.decl, digests({'src/b.png': 'd1'}))); - const [art] = rule('src/a.png', ['t %f %o'], ['x.png']); - assert.throws(() => rule(art!, spec, ['y.png']), /not yet supported/); + computeKey(a.decl, digests({'src/a.png': 'd1'})), + computeKey(b.decl, digests({'src/b.png': 'd1'}))); + const art = rule('src/a.png', ['t %f %o'], 'x.png'); + assert.throws(() => rule(art, spec, 'y.png'), /not yet supported/); }); test('deps are part of identity', () => { - const [a] = rule('src/a.png', {deps: 'data/d.json', cmds: ['tool %f %o']}, ['out.png']); - const kOld = computeKey(a!.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j1'})); - const kNew = computeKey(a!.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j2'})); + const a = rule('src/a.png', {deps: 'data/d.json', cmds: ['tool %f %o']}, 'out.png'); + const kOld = computeKey(a.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j1'})); + const kNew = computeKey(a.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j2'})); assert.notEqual(kOld, kNew); }); -test('memo runs once', () => { - let calls = 0; - const f = memo(() => { calls++; return rule('a.png', ['c %f %o'], ['x.png']); }); - assert.equal(f(), f()); - assert.equal(calls, 1); +test('identical declarations return the existing artifacts', () => { + const first = rule('a.png', ['c %f %o'], 'x.png'); + const second = rule('a.png', ['c %f %o'], 'x.png'); + assert.equal(first, second); + assert.equal(getDecls().length, 1); +}); + +test('forEachRule dedupes per input, sharing overlap', () => { + const spec = {display: 'd %f', cmds: ['c %f %o']}; + const first = forEachRule(['src/a.png', 'src/b.png'], spec, '%B.gif'); + const second = forEachRule(['src/b.png', 'src/c.png'], spec, '%B.gif'); + assert.equal(first[1], second[0]); + assert.equal(getDecls().length, 3); +}); + +test('declarations differing in display, cmds, or outputs stay distinct', () => { + rule('a.png', {display: 'one', cmds: ['c %f %o']}, 'x.png'); + rule('a.png', {display: 'two', cmds: ['c %f %o']}, 'x.png'); + rule('a.png', {display: 'one', cmds: ['other %f %o']}, 'x.png'); + rule('a.png', {display: 'one', cmds: ['c %f %o']}, 'y.png'); + assert.equal(getDecls().length, 4); +}); + +test('chained declarations dedupe through artifact inputs', () => { + const mid = rule('a.png', ['c %f %o'], 'mid.png'); + const first = rule(mid, ['convert %f %o'], 'out.webp'); + const second = rule(mid, ['convert %f %o'], 'out.webp'); + assert.equal(first, second); + assert.equal(getDecls().length, 2); }); diff --git a/tools/build/test/cas.test.ts b/tools/build/test/cas.test.ts index 9c19b396..9051b5f8 100644 --- a/tools/build/test/cas.test.ts +++ b/tools/build/test/cas.test.ts @@ -6,7 +6,7 @@ import os from 'node:os'; import pathlib from 'node:path'; import {test} from 'node:test'; -import {casExists, casInsert, casPath, casSweep} from '../cas.ts'; +import {casExists, casInsert, casPath, casStat, casSweep} from '../cas.ts'; function makeTmpRoot() : string { return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'cas-test-')); @@ -21,37 +21,54 @@ function stage(root : string, data : string) : string { test('casInsert stores by content digest, read-only', () => { const root = makeTmpRoot(); const cas = pathlib.join(root, 'cas'); - const digest = casInsert(cas, stage(root, 'hello'), 'png'); + const {digest, size} = casInsert(cas, stage(root, 'hello'), 'png'); assert.equal(digest, createHash('sha256').update('hello').digest('hex')); + assert.equal(size, 5n); const obj = casPath(cas, digest, 'png'); assert.equal(fs.readFileSync(obj, 'utf8'), 'hello'); assert.equal(fs.statSync(obj).mode & 0o777, 0o444); assert.ok(casExists(cas, digest, 'png')); - assert.ok(!casExists(cas, digest, 'gif')); + assert.equal(casStat(cas, digest, 'png'), 5n); + assert.equal(casStat(cas, digest, 'gif'), null); fs.rmSync(root, {recursive: true, force: true}); }); test('casInsert dedupes an existing object and discards the temp', () => { const root = makeTmpRoot(); const cas = pathlib.join(root, 'cas'); - const d1 = casInsert(cas, stage(root, 'same'), 'png'); + const d1 = casInsert(cas, stage(root, 'same'), 'png').digest; const tmp2 = stage(root, 'same'); - const d2 = casInsert(cas, tmp2, 'png'); + const d2 = casInsert(cas, tmp2, 'png').digest; assert.equal(d1, d2); assert.ok(!fs.existsSync(tmp2)); // Same bytes under a different extension is a distinct object. - const d3 = casInsert(cas, stage(root, 'same'), 'gif'); + const d3 = casInsert(cas, stage(root, 'same'), 'gif').digest; assert.equal(d1, d3); assert.ok(casExists(cas, d1, 'png')); assert.ok(casExists(cas, d1, 'gif')); fs.rmSync(root, {recursive: true, force: true}); }); +test('casInsert replaces a corrupt object instead of trusting it', () => { + const root = makeTmpRoot(); + const cas = pathlib.join(root, 'cas'); + const {digest} = casInsert(cas, stage(root, 'good bytes'), 'png'); + const obj = casPath(cas, digest, 'png'); + // Simulate a crash-truncated object under the same digest name. + fs.chmodSync(obj, 0o644); + fs.truncateSync(obj); + const again = casInsert(cas, stage(root, 'good bytes'), 'png'); + assert.equal(again.digest, digest); + assert.equal(fs.readFileSync(obj, 'utf8'), 'good bytes'); + assert.equal(fs.statSync(obj).mode & 0o777, 0o444); + fs.rmSync(root, {recursive: true, force: true}); +}); + test('casSweep removes non-live objects and prunes empty fanout dirs', () => { const root = makeTmpRoot(); const cas = pathlib.join(root, 'cas'); - const keep = casInsert(cas, stage(root, 'keep'), 'png'); - const drop = casInsert(cas, stage(root, 'drop'), 'png'); + const keep = casInsert(cas, stage(root, 'keep'), 'png').digest; + const drop = casInsert(cas, stage(root, 'drop'), 'png').digest; const removed = casSweep(cas, new Set([`${keep}.png`])); assert.equal(removed, 1); assert.ok(casExists(cas, keep, 'png')); diff --git a/tools/build/test/executor.test.ts b/tools/build/test/executor.test.ts index 63294fe2..09dbf31d 100644 --- a/tools/build/test/executor.test.ts +++ b/tools/build/test/executor.test.ts @@ -42,7 +42,7 @@ function src(env : Env, name : string, content : string) : string { // A copy rule that also counts its executions in env.execLog. function copyRule(env : Env, input : string | Artifact, out : string, extra : Partial = {}) : Artifact { - return rule(input, {cmds: [`cat %f > %o && echo x >> ${env.execLog}`], ...extra}, [out])[0]!; + return rule(input, {cmds: [`cat %f > %o && echo x >> ${env.execLog}`], ...extra}, out); } function execCount(env : Env) : number { @@ -53,10 +53,11 @@ function execCount(env : Env) : number { } } -async function runBuild(env : Env, opts : {dryRun? : boolean, gc? : boolean} = {}) : Promise { +async function runBuild(env : Env, opts : {dryRun? : boolean, gc? : boolean} = {}, + decls = getDecls()) : Promise { const store = new Store(env.dbPath); try { - return await build(getDecls(), { + return await build(decls, { root: env.root, store, casDir: env.casDir, @@ -147,7 +148,7 @@ test('byte-identical inputs share one execution across two declarations', async // The consumer must be a *different* computation: an identical command over // identical bytes would share the producer's key (by design). function upcaseRule(env : Env, input : Artifact, out : string) : Artifact { - return rule(input, [`tr a-z A-Z < %f > %o && echo x >> ${env.execLog}`], [out])[0]!; + return rule(input, [`tr a-z A-Z < %f > %o && echo x >> ${env.execLog}`], out); } test('chained rules: consumer follows producer, cas-missing reruns alone', async () => { @@ -177,7 +178,7 @@ test('chained rules: consumer follows producer, cas-missing reruns alone', async test('failed producer blocks the consumer; nothing is recorded', async () => { const env = setup(); - const bad = rule(src(env, 'a.txt', 'x'), ['false'], ['mid.txt'])[0]!; + const bad = rule(src(env, 'a.txt', 'x'), ['false'], 'mid.txt'); copyRule(env, bad, 'final.txt'); const result = await runBuild(env); assert.ok(!result.ok); @@ -194,11 +195,11 @@ test('multi-output rules route %oN and skip all-or-nothing', async () => { ['x.txt', 'y.css']); const [x1, y1] = declare(); await runBuild(env); - assert.equal(fs.readFileSync(casPath(env.casDir, x1!.hash, 'txt'), 'utf8'), 'one'); - assert.equal(fs.readFileSync(casPath(env.casDir, y1!.hash, 'css'), 'utf8'), 'two'); + assert.equal(fs.readFileSync(casPath(env.casDir, x1.hash, 'txt'), 'utf8'), 'one'); + assert.equal(fs.readFileSync(casPath(env.casDir, y1.hash, 'css'), 'utf8'), 'two'); resetDecls(); - fs.rmSync(casPath(env.casDir, y1!.hash, 'css')); + fs.rmSync(casPath(env.casDir, y1.hash, 'css')); declare(); const rerun = await runBuild(env); assert.deepEqual(statuses(rerun), ['ran']); @@ -233,6 +234,32 @@ test('dry run reports without writing state', async () => { store.close(); }); +test('a truncated CAS object reads as dirty and is repaired', async () => { + const env = setup(); + const out = copyRule(env, src(env, 'a.txt', 'truncate-me'), 'out.txt'); + await runBuild(env); + const obj = casPath(env.casDir, out.hash, 'txt'); + fs.chmodSync(obj, 0o644); + fs.truncateSync(obj); + + resetDecls(); + copyRule(env, pathlib.join(env.root, 'src/a.txt'), 'out.txt'); + const rerun = await runBuild(env); + assert.deepEqual(statuses(rerun), ['ran']); + assert.equal(fs.readFileSync(obj, 'utf8'), 'truncate-me'); + assert.equal(fs.statSync(obj).mode & 0o777, 0o444); +}); + +test('building a consumer alone pulls in and hashes its producer', async () => { + const env = setup(); + const mid = copyRule(env, src(env, 'a.txt', 'solo'), 'mid.txt'); + const final = rule(mid, [`tr a-z A-Z < %f > %o && echo x >> ${env.execLog}`], 'final.txt'); + const result = await runBuild(env, {}, [final.decl]); + assert.ok(result.ok); + assert.equal(execCount(env), 2); + assert.equal(fs.readFileSync(casPath(env.casDir, final.hash, 'txt'), 'utf8'), 'SOLO'); +}); + test('missing sources fail upfront', async () => { const env = setup(); copyRule(env, pathlib.join(env.root, 'src/nope.txt'), 'out.txt'); diff --git a/tools/build/test/store.test.ts b/tools/build/test/store.test.ts index 7eb5a809..c75709f0 100644 --- a/tools/build/test/store.test.ts +++ b/tools/build/test/store.test.ts @@ -16,17 +16,17 @@ function makeDbPath() : string { test('recordRule/lookupRule roundtrip, upsert replaces outputs', () => { const store = new Store(makeDbPath()); assert.equal(store.lookupRule('k1'), null); - store.recordRule('k1', ['cmd a'], [{digest: 'd1', ext: 'png'}, {digest: 'd2', ext: 'css'}]); - assert.deepEqual(store.lookupRule('k1'), [{digest: 'd1', ext: 'png'}, {digest: 'd2', ext: 'css'}]); - store.recordRule('k1', ['cmd a'], [{digest: 'd3', ext: 'png'}, {digest: 'd4', ext: 'css'}]); - assert.deepEqual(store.lookupRule('k1'), [{digest: 'd3', ext: 'png'}, {digest: 'd4', ext: 'css'}]); + store.recordRule('k1', ['cmd a'], [{digest: 'd1', ext: 'png', size: 1n}, {digest: 'd2', ext: 'css', size: 2n}]); + assert.deepEqual(store.lookupRule('k1'), [{digest: 'd1', ext: 'png', size: 1n}, {digest: 'd2', ext: 'css', size: 2n}]); + store.recordRule('k1', ['cmd a'], [{digest: 'd3', ext: 'png', size: 3n}, {digest: 'd4', ext: 'css', size: 4n}]); + assert.deepEqual(store.lookupRule('k1'), [{digest: 'd3', ext: 'png', size: 3n}, {digest: 'd4', ext: 'css', size: 4n}]); store.close(); }); test('deleteKeysNotIn cascades outputs; liveObjects reflects survivors', () => { const store = new Store(makeDbPath()); - store.recordRule('keep', ['c'], [{digest: 'da', ext: 'png'}]); - store.recordRule('drop', ['c'], [{digest: 'db', ext: 'gif'}]); + store.recordRule('keep', ['c'], [{digest: 'da', ext: 'png', size: 1n}]); + store.recordRule('drop', ['c'], [{digest: 'db', ext: 'gif', size: 1n}]); assert.deepEqual(store.liveObjects(), new Set(['da.png', 'db.gif'])); assert.equal(store.deleteKeysNotIn(new Set(['keep'])), 1); assert.equal(store.lookupRule('drop'), null); @@ -77,8 +77,8 @@ test('migrates a v1 db: drops rule tables, keeps file_cache', () => { const store = new Store(dbPath); assert.deepEqual([...store.loadFileCache().keys()], ['src/a.png']); assert.equal(store.lookupRule('old'), null); - store.recordRule('new', ['c'], [{digest: 'd', ext: 'png'}]); - assert.deepEqual(store.lookupRule('new'), [{digest: 'd', ext: 'png'}]); + store.recordRule('new', ['c'], [{digest: 'd', ext: 'png', size: 1n}]); + assert.deepEqual(store.lookupRule('new'), [{digest: 'd', ext: 'png', size: 1n}]); store.close(); const check = new Database(dbPath); diff --git a/tools/deploy/api.ts b/tools/deploy/api.ts index a3f011e4..d3d7e76c 100644 --- a/tools/deploy/api.ts +++ b/tools/deploy/api.ts @@ -71,8 +71,13 @@ export function makeCtx(casDir : string, queue : ActionQueue) : DeployCtx { }, list(dir : string) : SrcFile[] { const result = []; - for (const filename of fs.readdirSync(dir).sort()) { - const p = pathlib.path(filename, {dir}); + // Files only, no dotfiles: the same filtering the build-side + // glob applies to rule inputs. + for (const ent of fs.readdirSync(dir, {withFileTypes: true}).sort((a, b) => a.name < b.name ? -1 : 1)) { + if (ent.name.startsWith('.') || (!ent.isFile() && !ent.isSymbolicLink())) { + continue; + } + const p = pathlib.path(ent.name, {dir}); result.push({...p, path: pathlib.format(p)}); } return result; diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index 8e9fbdaf..a45cd57a 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -19,6 +19,7 @@ import {type DeploySpec, makeCtx} from './api.ts'; import {ActionQueue} from './queue.ts'; const root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..'); +const invocationCwd = process.cwd(); process.chdir(root); const DB_PATH = '.build/db.sqlite'; @@ -235,7 +236,15 @@ common(program.command('inspect ')) .action(async (paths : string[], opts : CommonOpts & {output : string}) => { setConfig(loadConfig(opts.config)); await importDeploys(discoverDeployFiles()); - const targets = paths.map(p => nodePath.normalize(p).replace(/\/+$/, '')); + // Accept absolute paths and paths relative to where the user ran the + // command; rules declare repo-root-relative paths. + const targets = paths.map(p => { + const target = nodePath.relative(root, nodePath.resolve(invocationCwd, p)); + if (target.startsWith('..')) { + throw new BuildError(`Not under the repo root: ${p}`); + } + return target; + }); const closure = new Set(); for (const decl of getDecls()) { diff --git a/tools/deploy/queue.ts b/tools/deploy/queue.ts index 2a7ae153..ff244fea 100644 --- a/tools/deploy/queue.ts +++ b/tools/deploy/queue.ts @@ -136,12 +136,13 @@ export class ActionQueue { const dst = nodePath.join(dir, entry.dst); fs.mkdirSync(nodePath.dirname(dst), {recursive: true}); if (op.type === 'Copy'){ - if (mode === 'link') { + // Read-only sources are CAS objects; their mode must not + // leak into deploy trees (rsync -a would ship it), and a + // hardlink cannot carry its own mode, so copy those. + if (mode === 'link' && (fs.statSync(op.src).mode & 0o200) !== 0) { fs.linkSync(op.src, dst); } else { fs.copyFileSync(op.src, dst); - // The copy inherits the source mode; CAS objects are - // read-only, which must not leak into deploy trees. fs.chmodSync(dst, 0o644); } } else if (op.type === 'Write') { diff --git a/tools/deploy/test/api.test.ts b/tools/deploy/test/api.test.ts index 042ea620..7360e2b8 100644 --- a/tools/deploy/test/api.test.ts +++ b/tools/deploy/test/api.test.ts @@ -27,12 +27,12 @@ function shortHash(data : Buffer | string) : string { // Stage `content` as a built artifact in a scratch CAS. function makeArtifact(casDir : string, content : string, ext : string) { const digest = createHash('sha256').update(content).digest('hex'); - const [artifact] = rule('in.png', ['t %f %o'], [`art.${ext}`]); - artifact!.resolve(digest); + const artifact = rule('in.png', ['t %f %o'], `art.${ext}`); + artifact.resolve(digest); const obj = casPath(casDir, digest, ext); fs.mkdirSync(pathlib.dirname(obj), {recursive: true}); fs.writeFileSync(obj, content); - return artifact!; + return artifact; } test('ctx.hash matches the historical single-file stamp for artifacts and files', () => { @@ -73,11 +73,13 @@ test('ctx queues artifact copies from the CAS, writes and reads', () => { assert.equal((ops[1] as {op : {src : string}}).op.src, casPath(casDir, artifact.hash, 'webp')); }); -test('ctx.list sorts and parses extensions', () => { +test('ctx.list sorts, parses extensions, skips dotfiles and directories', () => { const dir = tmpdir(); fs.writeFileSync(pathlib.join(dir, 'b.png'), ''); fs.writeFileSync(pathlib.join(dir, 'a.gif'), ''); fs.writeFileSync(pathlib.join(dir, 'noext'), ''); + fs.writeFileSync(pathlib.join(dir, '.hidden'), ''); + fs.mkdirSync(pathlib.join(dir, 'subdir')); const ctx = makeCtx('cas', new ActionQueue()); assert.deepEqual(ctx.list(dir), [ {dir, name: 'a', ext: 'gif', path: pathlib.join(dir, 'a.gif')},