diff --git a/tools/build/api.ts b/tools/build/api.ts index 61a143ea..cb305d79 100644 --- a/tools/build/api.ts +++ b/tools/build/api.ts @@ -1,11 +1,13 @@ -import fs from 'fs'; import pathlib from 'path'; import {createHash} from 'crypto'; -import {type Cmd, flattenCmds, substitute, basenameNoExt} from './subst.ts'; +import {astable, glob} from './helpers.ts'; +import {type Cmd, flattenCmds, substitute} from './subst.ts'; export type {Cmd}; +export {base, compresspng, getconfig, glob, pad, setConfig, + spritedata, spriteglob, trimimg, type SpriteData} from './helpers.ts'; export interface CmdSpec { display? : string; @@ -27,11 +29,6 @@ export interface RuleDecl { } let rules : RuleDecl[] = []; -let config = new Map(); - -export function setConfig(cfg : Map) : void { - config = cfg; -} export function getRules() : RuleDecl[] { return rules; @@ -41,134 +38,6 @@ export function resetRules() : void { rules = []; } -export function getconfig(name : string) : string | undefined { - const value = config.get(name); - return value === '' ? undefined : value; -} - -function astable(x : string | string[] | undefined) : string[] { - if (x === undefined) { - return []; - } - return typeof x === 'string' ? [x] : x; -} - -// Single-directory, single-'*' glob (all Tupfile patterns were of this form). -// Non-glob strings pass through literally; existence is checked at hash time. -// Results are sorted within a pattern; declared order is preserved across -// patterns (some rules, e.g. the pokeball sheet, are input-order-sensitive). -function globOne(pat : string) : string[] { - if (!pat.includes('*')) { - return [pat]; - } - const dir = pathlib.dirname(pat); - const base = pathlib.basename(pat); - if (dir.includes('*') || base.indexOf('*') !== base.lastIndexOf('*')) { - throw new Error(`Unsupported glob pattern: ${pat}`); - } - const [prefix, suffix] = base.split('*') as [string, string]; - const results = []; - for (const ent of fs.readdirSync(dir, {withFileTypes: true})) { - if (!ent.isFile() && !ent.isSymbolicLink()) { - continue; - } - const name = ent.name; - if (name.startsWith('.')) { - continue; // tup.glob ignored dotfiles - } - if (name.length >= prefix.length + suffix.length - && name.startsWith(prefix) && name.endsWith(suffix)) { - results.push(dir === '.' ? name : `${dir}/${name}`); - } - } - results.sort(); - return results; -} - -export function glob(pats : string | string[]) : string[] { - return astable(pats).flatMap(globOne); -} - -// tup.base: basename without directory or final extension -export function base(path : string) : string { - return basenameNoExt(path); -} - -export interface SpriteData { - id : string; - data : Record; -} - -// Port of util/sprites.lua spritedata. Lua used gmatch("[^-]+"), which skips -// empty segments, hence the filter. -export function spritedata(basename : string) : SpriteData { - const parts = basename.split('-').filter(p => p !== ''); - const data : Record = {}; - for (const part of parts.slice(1)) { - if (part.length === 1) { - data[part] = true; - } else { - data[part[0]!] = part.slice(1); - } - } - return {id: parts[0] ?? '', data}; -} - -export function spriteglob(pats : string | string[], flagspec? : Record) : string[] { - return glob(pats).filter(filename => { - const sd = spritedata(base(filename)); - for (const [k, v] of Object.entries(flagspec ?? {})) { - if (Boolean(v) !== Boolean(sd.data[k])) { - return false; - } - } - return true; - }); -} - -export function pad(opts : {w : number, h : number, input? : string, output? : string}) : string { - const input = opts.input ?? '%f'; - const output = opts.output ?? '%o'; - return `magick convert ${input} -background transparent -gravity center -extent ${opts.w}x${opts.h} ${output}`; -} - -export function trimimg(opts : {input? : string, output? : string} = {}) : string { - return `magick convert ${opts.input ?? '%f'} -trim ${opts.output ?? '%o'}`; -} - -interface CompressOpts { - pngquant? : string; - optipng? : string; - advpng? : string; -} - -function compressopts(program : string, copts : CompressOpts) : void { - copts.pngquant = getconfig(`${program}_PNGQUANT`) ?? copts.pngquant; - copts.optipng = getconfig(`${program}_OPTIPNG`) ?? copts.optipng; - copts.advpng = getconfig(`${program}_ADVPNG`) ?? copts.advpng; -} - -export function compresspng(opts : {config? : string, output? : string} = {}) : Cmd[] { - const output = opts.output ?? '%o'; - const copts : CompressOpts = {}; - compressopts('DEFAULT', copts); - if (opts.config) { - compressopts(opts.config, copts); - } - const cmds = []; - if (copts.pngquant !== undefined) { - // -f -o necessary to overwrite existing file - cmds.push(`pngquant -f -o ${output} ${copts.pngquant} ${output}`); - } - if (copts.optipng !== undefined) { - cmds.push(`optipng -q ${copts.optipng} ${output}`); - } - if (copts.advpng !== undefined) { - cmds.push(`advpng -q ${copts.advpng} ${output}`); - } - return cmds; -} - function normalizeSpec(spec : CmdSpec | Cmd[]) : CmdSpec { return Array.isArray(spec) ? {cmds: spec} : spec; } diff --git a/tools/build/artifact.ts b/tools/build/artifact.ts new file mode 100644 index 00000000..abe3cb10 --- /dev/null +++ b/tools/build/artifact.ts @@ -0,0 +1,208 @@ + +import pathlib from 'path'; +import {createHash} from 'crypto'; + +import {astable, glob} from './helpers.ts'; +import {type Cmd, basenameNoExt, flattenCmds, substitute} from './subst.ts'; + +// A rule's output: content-addressed bytes with a nominal name. The nominal +// name exists for provenance, inspection, and deploy-time naming; it is NOT +// part of the rule's identity, so renaming an output (or its source) never +// rebuilds anything. The digest is resolved by the executor (from the store +// on a clean hit, or by running the rule). +export class Artifact { + readonly name : string; // nominal basename sans ext ("abra", "spritesheet") + readonly ext : string; // "png", "gif", ... (no dot) + readonly decl : RuleDecl; + readonly index : number; // position among decl.outputs + private digest : string | null = null; + + constructor(name : string, ext : string, decl : RuleDecl, index : number) { + this.name = name; + this.ext = ext; + this.decl = decl; + this.index = index; + } + + get filename() : string { + return `${this.name}.${this.ext}`; + } + + // The producing rule's source-path inputs (provenance). + get sources() : string[] { + return this.decl.inputs.filter(i => typeof i === 'string'); + } + + get hash() : string { + if (this.digest === null) { + throw new Error(`Artifact ${this.filename} has not been built yet`); + } + return this.digest; + } + + resolve(digest : string) : void { + if (this.digest !== null && this.digest !== digest) { + throw new Error(`Artifact ${this.filename} resolved twice with different digests`); + } + this.digest = digest; + } +} + +export type Input = string | Artifact; // string = source path relative to repo root + +export interface CmdSpec { + display? : string; + // Tracked-but-not-substituted inputs: part of rule identity, but never + // expanded into %f. Use for files a tool reads on its own (e.g. + // tools/sheet readdirs the minisprite directories). + deps? : Input | Input[]; + // IMPORTANT: set this on any rule whose output BYTES depend on input + // NAMES (a tool that readdirs, or parses ids out of %f filenames, e.g. + // the spritesheet builders). Identity is normally content-only, so + // without this flag a same-bytes rename would leave the output stale. + nameSensitive? : boolean; + cmds : Cmd[]; +} + +export interface RuleDecl { + inputs : Input[]; // ordered (%f order; some rules are order-sensitive) + deps : Input[]; + outputs : Artifact[]; + cmds : string[]; // flattened PRE-substitution templates + display : string | null; // nominally substituted; cosmetic + nameSensitive : boolean; +} + +let decls : RuleDecl[] = []; + +export function getDecls() : RuleDecl[] { + return decls; +} + +export function resetDecls() : void { + decls = []; +} + +// Share a rule set between deploys: importing declares nothing, the first +// call declares once, later calls return the same artifacts. +export function memo(fn : () => T) : () => T { + let called = false; + let value : T; + return () => { + if (!called) { + value = fn(); + called = true; + } + return value!; + }; +} + +// Nominal path of an input, for %b/%B and displays. +function nominal(i : Input) : string { + return typeof i === 'string' ? i : i.filename; +} + +function extOf(i : Input) : string { + return typeof i === 'string' ? pathlib.extname(i) : `.${i.ext}`; +} + +function pathOrThrow(i : Input, what : string) : string { + if (typeof i !== 'string') { + throw new Error(`nameSensitive rules with artifact ${what} are not yet supported`); + } + return i; +} + +// Rule identity. Input paths are deliberately excluded (unless +// nameSensitive): identity is the computation and the bytes it consumes, so +// source renames and byte-identical duplicates dedupe for free. Output exts +// are included because tools pick output formats from extensions; nominal +// names are not. +export function computeKey(decl : RuleDecl, digestOf : (i : Input) => string) : string { + const h = createHash('sha256'); + h.update([ + 'v2', + decl.cmds.join('\n'), + decl.inputs.map(digestOf).join('\0'), + decl.inputs.map(extOf).join('\0'), + decl.deps.map(digestOf).join('\0'), + decl.outputs.map(o => o.ext).join('\0'), + ...(decl.nameSensitive + ? [[...decl.inputs, ...decl.deps].map(i => pathOrThrow(i, 'inputs')).join('\0')] + : []), + ].join('\x01')); + return h.digest('hex'); +} + +function normalizeSpec(spec : CmdSpec | Cmd[]) : CmdSpec { + return Array.isArray(spec) ? {cmds: spec} : spec; +} + +// Glob string patterns; artifacts pass through. Order is preserved. +function resolveInputs(input : Input | Input[] | undefined) : Input[] { + const list = input === undefined ? [] : Array.isArray(input) ? input : [input]; + return list.flatMap((i) : Input[] => typeof i === 'string' ? glob(i) : [i]); +} + +function makeDecl(inputs : Input[], deps : Input[], spec : CmdSpec, outputs : string[]) : RuleDecl { + const cmds = flattenCmds(spec.cmds); + if (cmds.length === 0) { + throw new Error(`Rule with no commands (outputs: ${outputs.join(' ')})`); + } + const decl : RuleDecl = { + inputs, + deps, + outputs: [], + cmds, + display: null, + nameSensitive: spec.nameSensitive ?? false, + }; + if (decl.nameSensitive) { + for (const i of [...inputs, ...deps]) { + pathOrThrow(i, 'inputs'); + } + } + outputs.forEach((out, index) => { + if (out.includes('/') || out.includes('%')) { + throw new Error(`Rule outputs are nominal filenames, no paths or substitutions: ${out}`); + } + const ext = pathlib.extname(out); + if (ext === '' || ext === '.') { + throw new Error(`Rule output needs an extension: ${out}`); + } + decl.outputs.push(new Artifact(basenameNoExt(out), ext.slice(1), decl, index)); + }); + // Validate substitutions now (unknown escapes, out-of-range %oN) rather + // than at execution; the results are discarded. + const nominalInputs = inputs.map(nominal); + const nominalOutputs = decl.outputs.map(o => o.filename); + for (const cmd of cmds) { + substitute(cmd, nominalInputs, nominalOutputs); + } + if (spec.display !== undefined) { + decl.display = substitute(spec.display, nominalInputs, nominalOutputs); + } + decls.push(decl); + return decl; +} + +export function rule(input : Input | Input[], spec : CmdSpec | Cmd[], + output : string | string[]) : Artifact[] { + const s = normalizeSpec(spec); + return makeDecl(resolveInputs(input), resolveInputs(s.deps), s, astable(output)).outputs; +} + +export function forEachRule(input : Input | Input[], spec : CmdSpec | Cmd[], + output : string) : Artifact[] { + const s = normalizeSpec(spec); + if (/%[fo]/.test(output)) { + throw new Error(`forEachRule output template may only use %b/%B: ${output}`); + } + const deps = resolveInputs(s.deps); + const outputs = []; + for (const file of resolveInputs(input)) { + const decl = makeDecl([file], deps, s, [substitute(output, [nominal(file)], [])]); + outputs.push(...decl.outputs); + } + return outputs; +} diff --git a/tools/build/helpers.ts b/tools/build/helpers.ts new file mode 100644 index 00000000..493f5fff --- /dev/null +++ b/tools/build/helpers.ts @@ -0,0 +1,141 @@ + +import fs from 'fs'; +import pathlib from 'path'; + +import type {Artifact} from './artifact.ts'; +import {type Cmd, basenameNoExt} from './subst.ts'; + +let config = new Map(); + +export function setConfig(cfg : Map) : void { + config = cfg; +} + +export function getconfig(name : string) : string | undefined { + const value = config.get(name); + return value === '' ? undefined : value; +} + +export function astable(x : string | string[] | undefined) : string[] { + if (x === undefined) { + return []; + } + return typeof x === 'string' ? [x] : x; +} + +// Single-directory, single-'*' glob (all Tupfile patterns were of this form). +// Non-glob strings pass through literally; existence is checked at hash time. +// Results are sorted within a pattern; declared order is preserved across +// patterns (some rules, e.g. the pokeball sheet, are input-order-sensitive). +function globOne(pat : string) : string[] { + if (!pat.includes('*')) { + return [pat]; + } + const dir = pathlib.dirname(pat); + const base = pathlib.basename(pat); + if (dir.includes('*') || base.indexOf('*') !== base.lastIndexOf('*')) { + throw new Error(`Unsupported glob pattern: ${pat}`); + } + const [prefix, suffix] = base.split('*') as [string, string]; + const results = []; + for (const ent of fs.readdirSync(dir, {withFileTypes: true})) { + if (!ent.isFile() && !ent.isSymbolicLink()) { + continue; + } + const name = ent.name; + if (name.startsWith('.')) { + continue; // tup.glob ignored dotfiles + } + if (name.length >= prefix.length + suffix.length + && name.startsWith(prefix) && name.endsWith(suffix)) { + results.push(dir === '.' ? name : `${dir}/${name}`); + } + } + results.sort(); + return results; +} + +export function glob(pats : string | string[]) : string[] { + return astable(pats).flatMap(globOne); +} + +// tup.base: basename without directory or final extension. For an artifact, +// its nominal name. +export function base(x : string | Artifact) : string { + return typeof x === 'string' ? basenameNoExt(x) : x.name; +} + +export interface SpriteData { + id : string; + data : Record; +} + +// Port of util/sprites.lua spritedata. Lua used gmatch("[^-]+"), which skips +// empty segments, hence the filter. +export function spritedata(basename : string) : SpriteData { + const parts = basename.split('-').filter(p => p !== ''); + const data : Record = {}; + for (const part of parts.slice(1)) { + if (part.length === 1) { + data[part] = true; + } else { + data[part[0]!] = part.slice(1); + } + } + return {id: parts[0] ?? '', data}; +} + +export function spriteglob(pats : string | string[], flagspec? : Record) : string[] { + return glob(pats).filter(filename => { + const sd = spritedata(base(filename)); + for (const [k, v] of Object.entries(flagspec ?? {})) { + if (Boolean(v) !== Boolean(sd.data[k])) { + return false; + } + } + return true; + }); +} + +export function pad(opts : {w : number, h : number, input? : string, output? : string}) : string { + const input = opts.input ?? '%f'; + const output = opts.output ?? '%o'; + return `magick convert ${input} -background transparent -gravity center -extent ${opts.w}x${opts.h} ${output}`; +} + +export function trimimg(opts : {input? : string, output? : string} = {}) : string { + return `magick convert ${opts.input ?? '%f'} -trim ${opts.output ?? '%o'}`; +} + +interface CompressOpts { + pngquant? : string; + optipng? : string; + advpng? : string; +} + +function compressopts(program : string, copts : CompressOpts) : void { + copts.pngquant = getconfig(`${program}_PNGQUANT`) ?? copts.pngquant; + copts.optipng = getconfig(`${program}_OPTIPNG`) ?? copts.optipng; + copts.advpng = getconfig(`${program}_ADVPNG`) ?? copts.advpng; +} + +export function compresspng(opts : {config? : string, output? : string} = {}) : Cmd[] { + const output = opts.output ?? '%o'; + const copts : CompressOpts = {}; + compressopts('DEFAULT', copts); + if (opts.config) { + compressopts(opts.config, copts); + } + const cmds = []; + if (copts.pngquant !== undefined) { + // -f -o necessary to overwrite existing file + cmds.push(`pngquant -f -o ${output} ${copts.pngquant} ${output}`); + } + if (copts.optipng !== undefined) { + cmds.push(`optipng -q ${copts.optipng} ${output}`); + } + if (copts.advpng !== undefined) { + cmds.push(`advpng -q ${copts.advpng} ${output}`); + } + return cmds; +} diff --git a/tools/build/test/artifact.test.ts b/tools/build/test/artifact.test.ts new file mode 100644 index 00000000..a6053def --- /dev/null +++ b/tools/build/test/artifact.test.ts @@ -0,0 +1,103 @@ + +import assert from 'node:assert/strict'; +import {beforeEach, test} from 'node:test'; + +import {type Input, computeKey, forEachRule, memo, resetDecls, rule} from '../artifact.ts'; + +beforeEach(resetDecls); + +function digests(map : Record) : (i : Input) => string { + return i => { + if (typeof i !== 'string') { + return i.hash; + } + const d = map[i]; + assert.ok(d !== undefined, `no digest for ${i}`); + return d; + }; +} + +test('rule declares artifacts with nominal names', () => { + const [png, css] = rule('src/a.png', ['tool %f %o1 %o2'], ['sheet.png', 'sheet.css']); + assert.equal(png!.name, 'sheet'); + assert.equal(png!.ext, 'png'); + assert.equal(css!.filename, 'sheet.css'); + assert.deepEqual(png!.sources, ['src/a.png']); + assert.equal(png!.decl, css!.decl); + assert.throws(() => png!.hash, /not been built/); + png!.resolve('d1'); + assert.equal(png!.hash, 'd1'); + assert.throws(() => png!.resolve('d2'), /resolved twice/); +}); + +test('rule rejects paths, substitutions, missing extensions in outputs', () => { + assert.throws(() => rule('a.png', ['c'], ['dir/x.png']), /nominal filenames/); + assert.throws(() => rule('a.png', ['c'], ['%B.png']), /nominal filenames/); + assert.throws(() => rule('a.png', ['c'], ['noext']), /needs an extension/); +}); + +test('rule validates command substitutions at declaration', () => { + assert.throws(() => rule('a.png', ['tool %q'], ['x.png']), /Unknown substitution/); + assert.throws(() => rule('a.png', ['tool %o2'], ['x.png']), /out of range/); + assert.throws(() => rule('a.png', [''], ['x.png']), /no commands/); +}); + +test('forEachRule declares one rule per input, %b/%B templates', () => { + const outs = forEachRule(['src/a.png', 'src/b.png'], ['convert %f %o'], '%B.gif'); + assert.deepEqual(outs.map(o => o.filename), ['a.gif', 'b.gif']); + assert.notEqual(outs[0]!.decl, outs[1]!.decl); + assert.throws(() => forEachRule('src/a.png', ['c'], '%f.gif'), /only use %b\/%B/); +}); + +test('chained rules accept artifacts as inputs', () => { + const [png] = rule('src/a.png', ['tool %f %o'], ['x.png']); + const [webp] = rule(png!, ['cwebp %f -o %o'], ['x.webp']); + assert.equal(webp!.decl.inputs[0], png); + assert.deepEqual(webp!.sources, []); + png!.resolve('dp'); + const key = computeKey(webp!.decl, digests({})); + assert.equal(typeof key, 'string'); +}); + +test('key ignores input paths but not bytes, order, exts, or commands', () => { + const [a] = rule('src/a.png', ['convert %f %o'], ['out.png']); + const [b] = rule('src/elsewhere/z.png', ['convert %f %o'], ['other.png']); + const kA = computeKey(a!.decl, digests({'src/a.png': 'd1'})); + // Renamed source, same bytes, different nominal output: same key. + assert.equal(kA, computeKey(b!.decl, digests({'src/elsewhere/z.png': 'd1'}))); + // Different bytes: different key. + assert.notEqual(kA, computeKey(b!.decl, digests({'src/elsewhere/z.png': 'd2'}))); + + const [two] = rule(['x.png', 'y.png'], ['join %f %o'], ['out.png']); + const [reversed] = rule(['y.png', 'x.png'], ['join %f %o'], ['out.png']); + const map = {'x.png': 'dx', 'y.png': 'dy'}; + assert.notEqual(computeKey(two!.decl, digests(map)), computeKey(reversed!.decl, digests(map))); + + const [gif] = rule('src/a.png', ['convert %f %o'], ['out.gif']); + assert.notEqual(kA, computeKey(gif!.decl, digests({'src/a.png': 'd1'}))); +}); + +test('nameSensitive keys on paths; rejects artifact inputs', () => { + const spec = {nameSensitive: true, cmds: ['tool %f %o']}; + const [a] = rule('src/a.png', spec, ['out.png']); + const [b] = rule('src/b.png', spec, ['out.png']); + assert.notEqual( + computeKey(a!.decl, digests({'src/a.png': 'd1'})), + computeKey(b!.decl, digests({'src/b.png': 'd1'}))); + const [art] = rule('src/a.png', ['t %f %o'], ['x.png']); + assert.throws(() => rule(art!, spec, ['y.png']), /not yet supported/); +}); + +test('deps are part of identity', () => { + const [a] = rule('src/a.png', {deps: 'data/d.json', cmds: ['tool %f %o']}, ['out.png']); + const kOld = computeKey(a!.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j1'})); + const kNew = computeKey(a!.decl, digests({'src/a.png': 'd1', 'data/d.json': 'j2'})); + assert.notEqual(kOld, kNew); +}); + +test('memo runs once', () => { + let calls = 0; + const f = memo(() => { calls++; return rule('a.png', ['c %f %o'], ['x.png']); }); + assert.equal(f(), f()); + assert.equal(calls, 1); +});