diff --git a/Buildfile.ts b/Buildfile.ts new file mode 100644 index 00000000..b38263cd --- /dev/null +++ b/Buildfile.ts @@ -0,0 +1,190 @@ + +import {base, compresspng, forEachRule, pad, rule, spriteglob, trimimg} from './tools/build/api.ts'; + +// Generate uniform size minisprites + +forEachRule("src/minisprites/pokemon/gen6/*.png", { + display: "pad g6 minisprite %f", + cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})], +}, "build/gen6-minisprites-padded/%b"); + +forEachRule("src/minisprites/items/*.png", { + display: "pad item minisprite %f", + cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})], +}, "build/item-minisprites-padded/%b"); + +forEachRule("src/minisprites/pokemon/gen6/*.png", { + display: "trim g6 minisprite %f", + cmds: [trimimg(), compresspng({config: "MINISPRITE"})], +}, "build/gen6-minisprites-trimmed/%b"); + +forEachRule("src/minisprites/items/*.png", { + display: "trim item minisprite %f", + cmds: [trimimg(), compresspng({config: "MINISPRITE"})], +}, "build/item-minisprites-trimmed/%b"); + +// Gen 9 + +forEachRule("src/gen9species/*.png", { + display: "96x96 %f", + // TODO, add customizable compression for gif + // ... or investigate using webp instead of both png/gif here + cmds: [ + "magick convert %f -trim +repage -resize 90x90 %o", + "gifsicle -O3 -b %o", + ], +}, "build/gen9-modelslike/%B.gif"); + +// Gen 10 + +forEachRule("src/champions/*.png", { + display: "96x96 %f", + // TODO, add customizable compression for gif + // ... or investigate using webp instead of both png/gif here + cmds: [ + "magick convert %f -trim +repage -resize 90x90 %o", + "gifsicle -O3 -b %o", + ], +}, "build/gen10-modelslike/%B.gif"); + +// Gen 5 CAPs... + +forEachRule("src/sprites/gen5/*.png", [ + // TODO, add customizable compression for gif + // ... or investigate using webp instead of both png/gif here + "magick convert %f %o", + "gifsicle -O3 -b %o", +], "build/gen5-gif/%B.gif"); + +// PS spritesheet + +rule("ps-pokemon.sheet.mjs", { + display: "ps pokemon sheet", + // The sheet tool reads the minisprites (via readdir, which tup couldn't + // track) and the sprite data; declare them so changes rebuild the sheet. + deps: [ + "src/minisprites/pokemon/gen6/*", + "data/species.json", + "data/items.json", + "tools/sheet/index.ts", + ], + cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], +}, "build/ps/pokemonicons-sheet.png"); + +// TODO: reenable when trainers are moved +// rule("ps-trainers.sheet.mjs", { +// display: "ps trainers sheet", +// cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], +// }, "build/ps/trainers-sheet.png"); + +rule("ps-items.sheet.mjs", { + display: "ps items sheet", + deps: [ + "src/minisprites/items/*", + "data/species.json", + "data/items.json", + "tools/sheet/index.ts", + ], + cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})], +}, "build/ps/itemicons-sheet.png"); + +// PS pokeball icons + +const balls = [ + "src/_uncategorized/noncanonical/ui/battle/Ball-Normal.png", + "src/_uncategorized/noncanonical/ui/battle/Ball-Sick.png", + "src/_uncategorized/noncanonical/ui/battle/Ball-Null.png", +]; + +rule(balls, { + display: "pokemonicons-pokeball-sheet", + cmds: [ + "magick convert -background transparent -gravity center -extent 40x30 %f +append %o", + compresspng({config: "SPRITESHEET"}), + ], +}, "build/ps/pokemonicons-pokeball-sheet.png"); + +// Smogdex minisprites (webp) + +forEachRule(spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false}), { + display: "webp minisprite %f", + cmds: ["cwebp -z 9 %f -o %o"], +}, "build/smogon/minisprites/%B.webp"); + +// Smogdex spritesheet + +rule(spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false}), { + display: "smogdex sheet", + deps: ["data/species.json", "data/items.json", "tools/smogdexspritesheet/index.ts"], + // build/smogon/spritesheet.png is an undeclared temporary; it is removed + // before the rule finishes. + cmds: [ + "node tools/smogdexspritesheet/index.ts --image build/smogon/spritesheet.png --stylesheet build/smogon/spritesheet.css -- %f", + "cwebp -z 9 build/smogon/spritesheet.png -o build/smogon/spritesheet.webp", + "rm build/smogon/spritesheet.png", + ], +}, ["build/smogon/spritesheet.webp", "build/smogon/spritesheet.css"]); + +// Smogdex social images + +const social = spriteglob(["src/models/*"], {b: false, s: false}); + +const socialSeen = new Set(social.map(base)); +for (const file of spriteglob(["src/gen9species/*"], {b: false, s: false})) { + if (!socialSeen.has(base(file))) { + social.push(file); + socialSeen.add(base(file)); + } +} + +forEachRule(social, { + display: "fbsprite %f", + cmds: [ + 'magick convert "%f[0]" -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o', + compresspng({config: "MODELS"}), + ], +}, "build/smogon/fbsprites/xy/%B.png"); + +forEachRule(social, { + display: "twittersprite %f", + cmds: [ + 'magick convert "%f[0]" -trim -resize 115x115 -background white -gravity center -extent 120x120 %o', + compresspng({config: "MODELS"}), + ], +}, "build/smogon/twittersprites/xy/%B.png"); + +// Trainers + +// TODO: reenable when trainers are moved +// forEachRule("src/canonical/trainers/*", { +// display: "pad trainer %f", +// cmds: [pad({w: 80, h: 80}), compresspng({config: "TRAINERS"})], +// }, "build/padded-trainers/canonical/%b"); + +// Padded Dex + +const dexOutput = forEachRule("src/dex/*", { + display: "pad dex %f", + cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})], +}, "build/padded-dex/%b"); + +// Build missing CAP dex + +const dexSet = new Set(dexOutput.map(base)); + +const dexMissing = []; +for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) { + if (!dexSet.has(base(file))) { + dexMissing.push(file); + dexSet.add(base(file)); + } +} + +forEachRule(dexMissing, { + display: "missing dex %B", + cmds: [ + 'magick convert "%f[0]" -trim %o', + 'magick mogrify -background transparent -gravity center -resize "120x120>" -extent 120x120 %o', + compresspng({config: "DEX"}), + ], +}, "build/padded-dex/%B.png"); diff --git a/package.json b/package.json index 70aa6824..f46268d7 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "build": "tsc --build tsconfig-workspace.json" }, "devDependencies": { + "@types/node": "^24.10.7", "typescript": "~7.0.2" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a93935d6..892df403 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: devDependencies: + '@types/node': + specifier: ^24.10.7 + version: 24.13.3 typescript: specifier: ~7.0.2 version: 7.0.2 diff --git a/tools/build/api.ts b/tools/build/api.ts new file mode 100644 index 00000000..c9d6b013 --- /dev/null +++ b/tools/build/api.ts @@ -0,0 +1,228 @@ + +import fs from 'fs'; +import pathlib from 'path'; +import {createHash} from 'crypto'; + +import {type Cmd, flattenCmds, substitute, basenameNoExt} from './subst.ts'; + +export type {Cmd}; + +export interface CmdSpec { + display? : string; + // Tracked-but-not-substituted inputs: hashed and part of rule identity, + // but never expanded into %f. Use for files a tool reads on its own + // (e.g. tools/sheet readdirs the minisprite directories). + deps? : string | string[]; + cmds : Cmd[]; +} + +export interface RuleDecl { + inputs : string[]; // %f source, ordered + deps : string[]; // hashed, never substituted + outputs : string[]; // post-substitution paths + command : string; // final ' && '-joined shell command + display : string | null; // post-substitution; cosmetic, not part of identity + template : string; // pre-substitution cmds + output basename template(s) + key : string; // identity for incremental skip +} + +let rules : RuleDecl[] = []; +let config = new Map(); + +export function setConfig(cfg : Map) : void { + config = cfg; +} + +export function getRules() : RuleDecl[] { + return rules; +} + +export function resetRules() : void { + rules = []; +} + +export function getconfig(name : string) : string | undefined { + const value = config.get(name); + return value === '' ? undefined : value; +} + +function astable(x : string | string[] | undefined) : string[] { + if (x === undefined) { + return []; + } + return typeof x === 'string' ? [x] : x; +} + +// Single-directory, single-'*' glob (all Tupfile patterns were of this form). +// Non-glob strings pass through literally; existence is checked at hash time. +// Results are sorted within a pattern; declared order is preserved across +// patterns (some rules, e.g. the pokeball sheet, are input-order-sensitive). +function globOne(pat : string) : string[] { + if (!pat.includes('*')) { + return [pat]; + } + const dir = pathlib.dirname(pat); + const base = pathlib.basename(pat); + if (dir.includes('*') || base.indexOf('*') !== base.lastIndexOf('*')) { + throw new Error(`Unsupported glob pattern: ${pat}`); + } + const [prefix, suffix] = base.split('*') as [string, string]; + const results = []; + for (const ent of fs.readdirSync(dir, {withFileTypes: true})) { + if (!ent.isFile() && !ent.isSymbolicLink()) { + continue; + } + const name = ent.name; + if (name.length >= prefix.length + suffix.length + && name.startsWith(prefix) && name.endsWith(suffix)) { + results.push(dir === '.' ? name : `${dir}/${name}`); + } + } + results.sort(); + return results; +} + +export function glob(pats : string | string[]) : string[] { + return astable(pats).flatMap(globOne); +} + +// tup.base: basename without directory or final extension +export function base(path : string) : string { + return basenameNoExt(path); +} + +export interface SpriteData { + id : string; + data : Record; +} + +// Port of util/sprites.lua spritedata. Lua used gmatch("[^-]+"), which skips +// empty segments, hence the filter. +export function spritedata(basename : string) : SpriteData { + const parts = basename.split('-').filter(p => p !== ''); + const data : Record = {}; + for (const part of parts.slice(1)) { + if (part.length === 1) { + data[part] = true; + } else { + data[part[0]!] = part.slice(1); + } + } + return {id: parts[0] ?? '', data}; +} + +export function spriteglob(pats : string | string[], flagspec? : Record) : string[] { + return glob(pats).filter(filename => { + const sd = spritedata(base(filename)); + for (const [k, v] of Object.entries(flagspec ?? {})) { + if (Boolean(v) !== Boolean(sd.data[k])) { + return false; + } + } + return true; + }); +} + +export function pad(opts : {w : number, h : number, input? : string, output? : string}) : string { + const input = opts.input ?? '%f'; + const output = opts.output ?? '%o'; + return `magick convert ${input} -background transparent -gravity center -extent ${opts.w}x${opts.h} ${output}`; +} + +export function trimimg(opts : {input? : string, output? : string} = {}) : string { + return `magick convert ${opts.input ?? '%f'} -trim ${opts.output ?? '%o'}`; +} + +interface CompressOpts { + pngquant? : string; + optipng? : string; + advpng? : string; +} + +function compressopts(program : string, copts : CompressOpts) : void { + copts.pngquant = getconfig(`${program}_PNGQUANT`) ?? copts.pngquant; + copts.optipng = getconfig(`${program}_OPTIPNG`) ?? copts.optipng; + copts.advpng = getconfig(`${program}_ADVPNG`) ?? copts.advpng; +} + +export function compresspng(opts : {config? : string, output? : string} = {}) : Cmd[] { + const output = opts.output ?? '%o'; + const copts : CompressOpts = {}; + compressopts('DEFAULT', copts); + if (opts.config) { + compressopts(opts.config, copts); + } + const cmds = []; + if (copts.pngquant !== undefined) { + // -f -o necessary to overwrite existing file + cmds.push(`pngquant -f -o ${output} ${copts.pngquant} ${output}`); + } + if (copts.optipng !== undefined) { + cmds.push(`optipng -q ${copts.optipng} ${output}`); + } + if (copts.advpng !== undefined) { + cmds.push(`advpng -q ${copts.advpng} ${output}`); + } + return cmds; +} + +function normalizeSpec(spec : CmdSpec | Cmd[]) : CmdSpec { + return Array.isArray(spec) ? {cmds: spec} : spec; +} + +// The rename-detection template deliberately excludes output directories +// (basename only) so that content-preserving moves across directories with +// identical processing still match, but extension changes (which change the +// bytes tools like magick produce) do not. +function templateOf(cmds : string[], outputTemplates : string[]) : string { + return cmds.join('\n') + '\0' + outputTemplates.map(t => pathlib.basename(t)).join('\0'); +} + +function keyOf(command : string, inputs : string[], deps : string[], outputs : string[]) : string { + const h = createHash('sha256'); + h.update([command, inputs.join('\0'), deps.join('\0'), outputs.join('\0')].join('\x01')); + return h.digest('hex'); +} + +function makeRule(inputs : string[], deps : string[], spec : CmdSpec, + outputs : string[], outputTemplates : string[]) : RuleDecl { + const cmds = flattenCmds(spec.cmds).map(c => substitute(c, inputs, outputs)); + if (cmds.length === 0) { + throw new Error(`Rule with no commands (outputs: ${outputs.join(' ')})`); + } + const command = cmds.join(' && '); + const decl : RuleDecl = { + inputs, + deps, + outputs, + command, + display: spec.display !== undefined ? substitute(spec.display, inputs, outputs) : null, + template: templateOf(flattenCmds(spec.cmds), outputTemplates), + key: keyOf(command, inputs, deps, outputs), + }; + rules.push(decl); + return decl; +} + +export function rule(input : string | string[], spec : CmdSpec | Cmd[], + output : string | string[]) : string[] { + const s = normalizeSpec(spec); + const outputs = astable(output); + const decl = makeRule(glob(input), glob(astable(s.deps)), s, outputs, outputs); + return decl.outputs; +} + +export function forEachRule(input : string | string[], spec : CmdSpec | Cmd[], + output : string) : string[] { + const s = normalizeSpec(spec); + if (/%[fo]/.test(output)) { + throw new Error(`forEachRule output template may only use %b/%B: ${output}`); + } + const deps = glob(astable(s.deps)); + const outputs = []; + for (const file of glob(input)) { + const decl = makeRule([file], deps, s, [substitute(output, [file], [])], [output]); + outputs.push(...decl.outputs); + } + return outputs; +} diff --git a/tools/build/config.ts b/tools/build/config.ts new file mode 100644 index 00000000..86adc1df --- /dev/null +++ b/tools/build/config.ts @@ -0,0 +1,25 @@ + +import fs from 'fs'; + +export function parseConfig(text : string) : Map { + const result = new Map(); + for (let line of text.split('\n')) { + line = line.trim(); + if (line === '' || line.startsWith('#')) { + continue; + } + const eq = line.indexOf('='); + if (eq === -1) { + throw new Error(`Invalid config line: ${line}`); + } + result.set(line.slice(0, eq).trim(), line.slice(eq + 1).trim()); + } + return result; +} + +export function loadConfig(path : string) : Map { + if (!fs.existsSync(path)) { + return new Map(); + } + return parseConfig(fs.readFileSync(path, 'utf8')); +} diff --git a/tools/build/db.ts b/tools/build/db.ts new file mode 100644 index 00000000..a175ab68 --- /dev/null +++ b/tools/build/db.ts @@ -0,0 +1,234 @@ + +import fs from 'fs'; +import pathlib from 'path'; +import Database from 'better-sqlite3'; + +import type {RuleDecl} from './api.ts'; +import type {FileStat} from './hash.ts'; +import {BuildError} from './graph.ts'; + +export interface StoredRuleInput { + path : string; + isDep : boolean; + hash : Buffer; +} + +export interface StoredRuleOutput { + path : string; + size : bigint | null; // null when the rule last failed + mtimeNs : bigint | null; +} + +export interface StoredRule { + id : bigint; + key : string; + command : string; + display : string | null; + template : string; + inputSig : Buffer; + ok : boolean; + inputs : StoredRuleInput[]; // ordered: inputs (in %f order), then deps + outputs : StoredRuleOutput[]; // ordered: position-mapped for renames +} + +export interface RecordedOutput { + path : string; + size : bigint; + mtimeNs : bigint; +} + +const DDL = ` +CREATE TABLE file_cache ( + path TEXT PRIMARY KEY, + size INTEGER NOT NULL, + mtime_ns INTEGER NOT NULL, + hash BLOB NOT NULL +) WITHOUT ROWID; + +CREATE TABLE rules ( + id INTEGER PRIMARY KEY, + key TEXT NOT NULL UNIQUE, + command TEXT NOT NULL, + display TEXT, + template TEXT NOT NULL, + input_sig BLOB NOT NULL, + ok INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX rules_rename ON rules(template, input_sig); + +CREATE TABLE rule_inputs ( + rule_id INTEGER NOT NULL REFERENCES rules(id) ON DELETE CASCADE, + ord INTEGER NOT NULL, + path TEXT NOT NULL, + is_dep INTEGER NOT NULL DEFAULT 0, + hash BLOB NOT NULL, + PRIMARY KEY (rule_id, ord) +); +CREATE INDEX rule_inputs_path ON rule_inputs(path); + +CREATE TABLE rule_outputs ( + rule_id INTEGER NOT NULL REFERENCES rules(id) ON DELETE CASCADE, + ord INTEGER NOT NULL, + path TEXT NOT NULL, + size INTEGER, + mtime_ns INTEGER, + PRIMARY KEY (rule_id, ord) +); +CREATE INDEX rule_outputs_path ON rule_outputs(path); +`; + +export class BuildDb { + private db : Database.Database; + + constructor(dbPath : string) { + fs.mkdirSync(pathlib.dirname(dbPath), {recursive: true}); + this.db = new Database(dbPath); + this.db.defaultSafeIntegers(true); + this.db.pragma('journal_mode = WAL'); + this.db.pragma('foreign_keys = ON'); + this.db.pragma('synchronous = NORMAL'); + this.migrate(); + } + + private migrate() : void { + const version = Number(this.db.pragma('user_version', {simple: true})); + if (version === 0) { + this.db.exec('BEGIN;' + DDL + 'PRAGMA user_version = 1; COMMIT;'); + } else if (version !== 1) { + throw new BuildError( + `Unknown build db schema version ${version}; delete .build/ and re-run with --adopt`); + } + } + + loadFileCache() : Map { + const result = new Map(); + const rows = this.db.prepare<[], {path : string, size : bigint, mtime_ns : bigint, hash : Buffer}>( + 'SELECT path, size, mtime_ns, hash FROM file_cache').all(); + for (const row of rows) { + result.set(row.path, {size: row.size, mtimeNs: row.mtime_ns, hash: row.hash}); + } + return result; + } + + saveFileCache(entries : Map) : void { + const upsert = this.db.prepare(` + INSERT INTO file_cache (path, size, mtime_ns, hash) VALUES (?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET + size = excluded.size, mtime_ns = excluded.mtime_ns, hash = excluded.hash`); + this.db.transaction(() => { + for (const [path, stat] of entries) { + upsert.run(path, stat.size, stat.mtimeNs, stat.hash); + } + })(); + } + + pruneFileCache(live : Set) : void { + const paths = this.db.prepare<[], {path : string}>('SELECT path FROM file_cache').all(); + const del = this.db.prepare('DELETE FROM file_cache WHERE path = ?'); + this.db.transaction(() => { + for (const {path} of paths) { + if (!live.has(path)) { + del.run(path); + } + } + })(); + } + + loadStoredRules() : StoredRule[] { + const byId = new Map(); + const ruleRows = this.db.prepare<[], { + id : bigint, key : string, command : string, display : string | null, + template : string, input_sig : Buffer, ok : bigint, + }>('SELECT id, key, command, display, template, input_sig, ok FROM rules').all(); + for (const row of ruleRows) { + byId.set(row.id, { + id: row.id, + key: row.key, + command: row.command, + display: row.display, + template: row.template, + inputSig: row.input_sig, + ok: row.ok !== 0n, + inputs: [], + outputs: [], + }); + } + const inputRows = this.db.prepare<[], {rule_id : bigint, path : string, is_dep : bigint, hash : Buffer}>( + 'SELECT rule_id, path, is_dep, hash FROM rule_inputs ORDER BY rule_id, ord').all(); + for (const row of inputRows) { + byId.get(row.rule_id)!.inputs.push({path: row.path, isDep: row.is_dep !== 0n, hash: row.hash}); + } + const outputRows = this.db.prepare<[], {rule_id : bigint, path : string, size : bigint | null, mtime_ns : bigint | null}>( + 'SELECT rule_id, path, size, mtime_ns FROM rule_outputs ORDER BY rule_id, ord').all(); + for (const row of outputRows) { + byId.get(row.rule_id)!.outputs.push({path: row.path, size: row.size, mtimeNs: row.mtime_ns}); + } + return [...byId.values()]; + } + + // One transaction per completed rule: an interrupted build only ever + // contains fully-recorded rules. outputs === null records a failure (ok=0, + // output paths kept for GC, stats nulled so the rule stays dirty). + recordRuleResult(decl : RuleDecl, inputHashes : Map, sig : Buffer, + outputs : RecordedOutput[] | null) : void { + const ok = outputs !== null; + const upsert = this.db.prepare<[string, string, string | null, string, Buffer, bigint], {id : bigint}>(` + INSERT INTO rules (key, command, display, template, input_sig, ok) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(key) DO UPDATE SET + display = excluded.display, input_sig = excluded.input_sig, ok = excluded.ok + RETURNING id`); + const delInputs = this.db.prepare('DELETE FROM rule_inputs WHERE rule_id = ?'); + const delOutputs = this.db.prepare('DELETE FROM rule_outputs WHERE rule_id = ?'); + const insInput = this.db.prepare( + 'INSERT INTO rule_inputs (rule_id, ord, path, is_dep, hash) VALUES (?, ?, ?, ?, ?)'); + const insOutput = this.db.prepare( + 'INSERT INTO rule_outputs (rule_id, ord, path, size, mtime_ns) VALUES (?, ?, ?, ?, ?)'); + this.db.transaction(() => { + const {id} = upsert.get(decl.key, decl.command, decl.display, decl.template, + sig, ok ? 1n : 0n)!; + delInputs.run(id); + delOutputs.run(id); + let ord = 0; + for (const path of decl.inputs) { + insInput.run(id, ord++, path, 0, inputHashes.get(path)!); + } + for (const path of decl.deps) { + insInput.run(id, ord++, path, 1, inputHashes.get(path)!); + } + const outputRows = outputs ?? decl.outputs.map(path => ({path, size: null, mtimeNs: null})); + outputRows.forEach((o, i) => insOutput.run(id, i, o.path, o.size, o.mtimeNs)); + })(); + } + + deleteRule(id : bigint) : void { + this.db.prepare('DELETE FROM rules WHERE id = ?').run(id); + } + + close() : void { + this.db.pragma('wal_checkpoint(TRUNCATE)'); + this.db.close(); + } +} + +// Guards against two concurrent builds; the exclusive transaction is released +// by the OS on any crash, so there are no stale lock files. +export function acquireLock(lockPath : string) : () => void { + fs.mkdirSync(pathlib.dirname(lockPath), {recursive: true}); + const lock = new Database(lockPath, {timeout: 0}); + try { + lock.exec('BEGIN EXCLUSIVE'); + } catch (err) { + lock.close(); + if ((err as {code? : string}).code === 'SQLITE_BUSY') { + throw new BuildError('Another build is already running.'); + } + throw err; + } + return () => { + try { + lock.exec('COMMIT'); + } catch {} + lock.close(); + }; +} diff --git a/tools/build/exec.ts b/tools/build/exec.ts new file mode 100644 index 00000000..578831e8 --- /dev/null +++ b/tools/build/exec.ts @@ -0,0 +1,78 @@ + +import {spawn} from 'child_process'; + +export interface ExecResult { + code : number | null; + signal : NodeJS.Signals | null; + output : string; + durationMs : number; +} + +// The command script is fed to sh via stdin rather than -c: a single argv +// entry is capped by the kernel (MAX_ARG_STRLEN, ~128KB) and the largest %f +// expansion is already 80KB+. +export function runShell(command : string, opts : {cwd : string, signal : AbortSignal}) : Promise { + return new Promise((resolve, reject) => { + const start = performance.now(); + // detached: own process group, so an abort kills grandchildren + // (magick, optipng, ...) with one signal + const child = spawn('sh', [], {cwd: opts.cwd, detached: true, stdio: ['pipe', 'pipe', 'pipe']}); + const chunks : Buffer[] = []; + child.stdout.on('data', c => chunks.push(c)); + child.stderr.on('data', c => chunks.push(c)); + child.stdin.on('error', () => {}); // EPIPE if the shell exits early + child.stdin.end(command + '\n'); + + let killTimer : NodeJS.Timeout | undefined; + const kill = (sig : NodeJS.Signals) => { + try { + process.kill(-child.pid!, sig); + } catch {} + }; + const onAbort = () => { + kill('SIGTERM'); + killTimer = setTimeout(() => kill('SIGKILL'), 5000); + killTimer.unref(); + }; + if (opts.signal.aborted) { + onAbort(); + } else { + opts.signal.addEventListener('abort', onAbort, {once: true}); + } + + const cleanup = () => { + opts.signal.removeEventListener('abort', onAbort); + if (killTimer !== undefined) { + clearTimeout(killTimer); + } + }; + child.on('error', err => { + cleanup(); + reject(err); + }); + child.on('close', (code, signal) => { + cleanup(); + resolve({ + code, + signal, + output: Buffer.concat(chunks).toString(), + durationMs: performance.now() - start, + }); + }); + }); +} + +export async function workerPool(items : readonly T[], jobs : number, + fn : (item : T, index : number) => Promise) : Promise { + let next = 0; + const workers = []; + for (let i = 0; i < Math.max(1, Math.min(jobs, items.length)); i++) { + workers.push((async () => { + while (next < items.length) { + const index = next++; + await fn(items[index]!, index); + } + })()); + } + await Promise.all(workers); +} diff --git a/tools/build/graph.ts b/tools/build/graph.ts new file mode 100644 index 00000000..d7008170 --- /dev/null +++ b/tools/build/graph.ts @@ -0,0 +1,62 @@ + +import type {RuleDecl} from './api.ts'; + +export class BuildError extends Error {} + +export interface GraphResult { + order : RuleDecl[]; // topological, stable w.r.t. declaration order + generated : Set; // every path produced by some rule +} + +export function checkGraph(rules : RuleDecl[]) : GraphResult { + const owner = new Map(); + for (const rule of rules) { + for (const output of rule.outputs) { + const other = owner.get(output); + if (other !== undefined) { + throw new BuildError( + `Output ${output} produced by multiple rules:\n ${other.command}\n ${rule.command}`); + } + owner.set(output, rule); + } + } + + const consumers = new Map(); + const indegree = new Map(); + for (const rule of rules) { + indegree.set(rule, 0); + } + for (const rule of rules) { + for (const input of [...rule.inputs, ...rule.deps]) { + const producer = owner.get(input); + if (producer !== undefined && producer !== rule) { + let list = consumers.get(producer); + if (list === undefined) { + consumers.set(producer, list = []); + } + list.push(rule); + indegree.set(rule, indegree.get(rule)! + 1); + } + } + } + + const queue = rules.filter(r => indegree.get(r) === 0); + const order = []; + for (let i = 0; i < queue.length; i++) { + const rule = queue[i]!; + order.push(rule); + for (const consumer of consumers.get(rule) ?? []) { + const deg = indegree.get(consumer)! - 1; + indegree.set(consumer, deg); + if (deg === 0) { + queue.push(consumer); + } + } + } + if (order.length !== rules.length) { + const stuck = rules.filter(r => indegree.get(r)! > 0); + throw new BuildError(`Dependency cycle involving:\n ${stuck.map(r => r.command).join('\n ')}`); + } + + return {order, generated: new Set(owner.keys())}; +} diff --git a/tools/build/hash.ts b/tools/build/hash.ts new file mode 100644 index 00000000..34669d5b --- /dev/null +++ b/tools/build/hash.ts @@ -0,0 +1,51 @@ + +import fs from 'fs'; +import {createHash} from 'crypto'; + +export interface FileStat { + size : bigint; + mtimeNs : bigint; + hash : Buffer; +} + +export function hashFileSync(path : string) : Buffer { + return createHash('sha256').update(fs.readFileSync(path)).digest(); +} + +export interface ReconcileResult { + hashes : Map; // current content hash for every extant path + updated : Map; // cache entries that changed (to persist) + missing : string[]; // paths that don't exist or aren't files +} + +// Content hashes with a stat cache: only files whose (size, mtime_ns) changed +// since the recorded cache entry are rehashed. mtime_ns exceeds 2^53, hence +// bigint stats throughout. +export function reconcileHashes(paths : Iterable, cache : Map) : ReconcileResult { + const hashes = new Map(); + const updated = new Map(); + const missing = []; + for (const path of paths) { + let st; + try { + st = fs.statSync(path, {bigint: true}); + } catch { + missing.push(path); + continue; + } + if (!st.isFile()) { + missing.push(path); + continue; + } + const cached = cache.get(path); + let hash; + if (cached !== undefined && cached.size === st.size && cached.mtimeNs === st.mtimeNs) { + hash = cached.hash; + } else { + hash = hashFileSync(path); + updated.set(path, {size: st.size, mtimeNs: st.mtimeNs, hash}); + } + hashes.set(path, hash); + } + return {hashes, updated, missing}; +} diff --git a/tools/build/index.ts b/tools/build/index.ts new file mode 100644 index 00000000..60e4a547 --- /dev/null +++ b/tools/build/index.ts @@ -0,0 +1,307 @@ + +import fs from 'fs'; +import os from 'os'; +import pathlib from 'path'; +import {fileURLToPath, pathToFileURL} from 'url'; +import {program} from 'commander'; +import debugfn from 'debug'; + +import {getRules, type RuleDecl, setConfig} from './api.ts'; +import {loadConfig} from './config.ts'; +import {acquireLock, BuildDb, type RecordedOutput} from './db.ts'; +import {runShell, workerPool} from './exec.ts'; +import {BuildError, checkGraph} from './graph.ts'; +import {reconcileHashes} from './hash.ts'; +import {computePlan, type OutputStat, ruleInputSig} from './plan.ts'; + +const debug = debugfn('build'); + +program + .option('-j, --jobs ', 'number of parallel jobs', String(os.availableParallelism())) + .option('-n, --dry-run', 'print the plan without changing anything') + .option('--adopt', 'record existing outputs as up to date instead of running (migration)') + .option('--fail-fast', 'stop scheduling new rules after the first failure') + .option('--config ', 'config file', 'build.config') + .option('-v, --verbose', 'print more detail'); +program.parse(process.argv); +const opts = program.opts(); + +const root = pathlib.resolve(fileURLToPath(import.meta.url), '../../..'); +process.chdir(root); + +function statPath(path : string) : OutputStat | null { + try { + const st = fs.statSync(path, {bigint: true}); + return st.isFile() ? {size: st.size, mtimeNs: st.mtimeNs} : null; + } catch { + return null; + } +} + +function label(decl : RuleDecl) : string { + return decl.display ?? decl.command.split(' && ')[0]!; +} + +function indent(text : string) : string { + return text.replace(/\n$/, '').split('\n').map(l => ' ' + l).join('\n'); +} + +async function main() : Promise { + const dryRun = Boolean(opts.dryRun); + const releaseLock = dryRun ? null : acquireLock('.build/lock.sqlite'); + const db = new BuildDb('.build/db.sqlite'); + try { + // Phase 1: evaluate the rule set + setConfig(loadConfig(opts.config)); + await import(pathToFileURL(pathlib.join(root, 'Buildfile.ts')).href); + const rules = getRules(); + const {order, generated} = checkGraph(rules); + for (const rule of rules) { + for (const path of [...rule.inputs, ...rule.deps]) { + if (generated.has(path)) { + // The planner assumes all inputs are hashable before + // execution; support this when a rule needs it. + throw new BuildError(`Rules consuming generated files are not yet supported: ${path}`); + } + } + } + debug('%d rules', rules.length); + + // Phase 2: hash source files (stat-cached) + const sources = new Set(); + for (const rule of rules) { + for (const path of [...rule.inputs, ...rule.deps]) { + sources.add(path); + } + } + const {hashes, updated, missing} = reconcileHashes(sources, db.loadFileCache()); + if (missing.length > 0) { + throw new BuildError(`Missing input files:\n ${missing.slice(0, 20).join('\n ')}` + + (missing.length > 20 ? `\n ... and ${missing.length - 20} more` : '')); + } + if (!dryRun && updated.size > 0) { + db.saveFileCache(updated); + } + debug('hashed %d files (%d cached)', sources.size, sources.size - updated.size); + + // Phase 3: plan + const statMemo = new Map(); + const statOutput = (path : string) => { + let st = statMemo.get(path); + if (st === undefined) { + statMemo.set(path, st = statPath(path)); + } + return st; + }; + const plan = computePlan({ + current: rules, + stored: db.loadStoredRules(), + hashes, + statOutput, + adopt: Boolean(opts.adopt), + }); + + const staleOutputs = []; + const currentOutputs = new Set(rules.flatMap(r => r.outputs)); + for (const s of plan.stale) { + for (const o of s.outputs) { + if (!currentOutputs.has(o.path)) { + staleOutputs.push(o.path); + } + } + } + + if (plan.run.length + plan.renames.length + plan.adopt.length + plan.stale.length === 0) { + console.log(`${plan.clean.length} rules up to date.`); + return 0; + } + if (opts.verbose || dryRun) { + for (const {decl, reason} of plan.run) { + console.log(`run (${reason}): ${label(decl)}`); + } + for (const {decl, from} of plan.renames) { + console.log(`rename: ${from.outputs.map(o => o.path).join(' ')} -> ${decl.outputs.join(' ')}`); + } + for (const decl of plan.adopt) { + console.log(`adopt: ${label(decl)}`); + } + for (const path of staleOutputs) { + console.log(`delete: ${path}`); + } + } + if (dryRun) { + console.log(`would run ${plan.run.length}, rename ${plan.renames.length}, ` + + `adopt ${plan.adopt.length}, delete ${staleOutputs.length} outputs ` + + `(${plan.clean.length} up to date)`); + return 0; + } + + // Phase 4: renames (before stale deletion: sources must still exist) + for (const {decl, from} of plan.renames) { + const recorded : RecordedOutput[] = []; + for (let i = 0; i < decl.outputs.length; i++) { + const src = from.outputs[i]!.path; + const dst = decl.outputs[i]!; + if (src !== dst) { + fs.mkdirSync(pathlib.dirname(dst), {recursive: true}); + fs.copyFileSync(src, dst); + } + const st = statPath(dst); + if (st === null) { + throw new BuildError(`Rename copy failed: ${src} -> ${dst}`); + } + recorded.push({path: dst, size: st.size, mtimeNs: st.mtimeNs}); + } + db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded); + } + + // Phase 5: delete outputs of removed rules, prune empty dirs + const staleDirs = new Set(); + for (const s of plan.stale) { + for (const o of s.outputs) { + if (!currentOutputs.has(o.path)) { + try { + fs.unlinkSync(o.path); + } catch (err) { + if ((err as {code? : string}).code !== 'ENOENT') { + throw err; + } + } + staleDirs.add(pathlib.dirname(o.path)); + } + } + db.deleteRule(s.id); + } + for (let dir of staleDirs) { + while (dir.startsWith('build/')) { + try { + fs.rmdirSync(dir); + } catch { + break; + } + dir = pathlib.dirname(dir); + } + } + db.pruneFileCache(sources); + + // Phase 6: adoption (migration): trust existing outputs + for (const decl of plan.adopt) { + const recorded = decl.outputs.map(path => { + const st = statPath(path)!; + return {path, size: st.size, mtimeNs: st.mtimeNs}; + }); + db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded); + } + + // Phase 7: execute + // The worker pool does not serialize producers before consumers; that + // is safe because rules consuming generated files are rejected above. + // `order` is used for a stable, declaration-ordered schedule. + const orderIndex = new Map(order.map((r, i) => [r, i])); + const runList = [...plan.run].sort((a, b) => orderIndex.get(a.decl)! - orderIndex.get(b.decl)!); + const ac = new AbortController(); + let interrupted = false; + const onSignal = () => { + if (interrupted) { + process.exit(130); + } + interrupted = true; + console.error('\nInterrupted; waiting for running rules to stop...'); + ac.abort(); + }; + process.on('SIGINT', onSignal); + process.on('SIGTERM', onSignal); + + const failures : RuleDecl[] = []; + let done = 0; + await workerPool(runList, parseInt(opts.jobs, 10), async ({decl}) => { + if (ac.signal.aborted) { + return; + } + for (const out of decl.outputs) { + fs.mkdirSync(pathlib.dirname(out), {recursive: true}); + } + const result = await runShell(decl.command, {cwd: root, signal: ac.signal}); + if (ac.signal.aborted && result.code !== 0) { + return; // killed by the abort, not a real failure; stays dirty + } + let recorded : RecordedOutput[] | null = null; + const missingOutputs = []; + if (result.code === 0) { + recorded = []; + for (const path of decl.outputs) { + const st = statPath(path); + if (st === null) { + missingOutputs.push(path); + recorded = null; + break; + } + recorded.push({path, size: st.size, mtimeNs: st.mtimeNs}); + } + } + db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded); + done++; + if (recorded !== null) { + console.log(`[${done}/${runList.length}] ${label(decl)}`); + if (result.output !== '') { + console.log(indent(result.output)); + } + } else { + failures.push(decl); + console.error(`[${done}/${runList.length}] FAILED: ${label(decl)}`); + console.error(` command: ${decl.command}`); + if (result.output !== '') { + console.error(indent(result.output)); + } + if (missingOutputs.length > 0) { + console.error(` command succeeded but did not produce: ${missingOutputs.join(' ')}`); + } + if (opts.failFast) { + ac.abort(); + } + } + }); + process.off('SIGINT', onSignal); + process.off('SIGTERM', onSignal); + + // Phase 8: summary + const parts = [`${plan.clean.length} up to date`]; + if (runList.length > 0) { + parts.push(`${done - failures.length} ran`); + } + if (plan.renames.length > 0) { + parts.push(`${plan.renames.length} renamed`); + } + if (plan.adopt.length > 0) { + parts.push(`${plan.adopt.length} adopted`); + } + if (plan.stale.length > 0) { + parts.push(`${plan.stale.length} removed`); + } + if (failures.length > 0) { + parts.push(`${failures.length} FAILED`); + } + console.log(parts.join(', ') + '.'); + if (failures.length > 0) { + console.error('Failed rules:'); + for (const decl of failures) { + console.error(` ${label(decl)}`); + } + } + return interrupted ? 130 : failures.length > 0 ? 1 : 0; + } finally { + db.close(); + releaseLock?.(); + } +} + +try { + process.exitCode = await main(); +} catch (err) { + if (err instanceof BuildError) { + console.error(err.message); + process.exitCode = 1; + } else { + throw err; + } +} diff --git a/tools/build/plan.ts b/tools/build/plan.ts new file mode 100644 index 00000000..30f712c0 --- /dev/null +++ b/tools/build/plan.ts @@ -0,0 +1,132 @@ + +import {createHash} from 'crypto'; + +import type {RuleDecl} from './api.ts'; +import type {StoredRule} from './db.ts'; + +export type DirtyReason = 'new' | 'failed-last-run' | 'input-changed' + | 'output-missing' | 'output-tampered'; + +export interface OutputStat { + size : bigint; + mtimeNs : bigint; +} + +export interface Plan { + clean : RuleDecl[]; + run : {decl : RuleDecl, reason : DirtyReason}[]; + renames : {decl : RuleDecl, from : StoredRule}[]; + stale : StoredRule[]; + adopt : RuleDecl[]; +} + +export function inputSig(hashes : Buffer[]) : Buffer { + const h = createHash('sha256'); + for (const hash of hashes) { + h.update(hash); + } + return h.digest(); +} + +export function ruleInputSig(decl : RuleDecl, hashes : Map) : Buffer { + return inputSig([...decl.inputs, ...decl.deps].map(p => hashes.get(p)!)); +} + +function renameKey(template : string, sig : Buffer) : string { + return template + '\0' + sig.toString('hex'); +} + +export function computePlan(opts : { + current : RuleDecl[], + stored : StoredRule[], + hashes : Map, + statOutput : (path : string) => OutputStat | null, + adopt : boolean, +}) : Plan { + const {current, stored, hashes, statOutput, adopt} = opts; + + const storedByKey = new Map(); + const renameIndex = new Map(); + for (const s of stored) { + storedByKey.set(s.key, s); + if (s.ok) { + const key = renameKey(s.template, s.inputSig); + let list = renameIndex.get(key); + if (list === undefined) { + renameIndex.set(key, list = []); + } + list.push(s); + } + } + + const currentKeys = new Set(current.map(r => r.key)); + const plan : Plan = { + clean: [], + run: [], + renames: [], + stale: stored.filter(s => !currentKeys.has(s.key)), + adopt: [], + }; + + // A stored rule's outputs are intact iff every recorded output exists on + // disk with its recorded stat. Rename sources must pass this (never copy + // tampered or unverified bytes). + const outputsIntact = (s : StoredRule) : boolean => + s.outputs.every(o => { + if (o.size === null || o.mtimeNs === null) { + return false; + } + const st = statOutput(o.path); + return st !== null && st.size === o.size && st.mtimeNs === o.mtimeNs; + }); + + const dirtyReason = (s : StoredRule) : DirtyReason | null => { + if (!s.ok) { + return 'failed-last-run'; + } + for (const inp of s.inputs) { + if (!hashes.get(inp.path)?.equals(inp.hash)) { + return 'input-changed'; + } + } + for (const o of s.outputs) { + const st = statOutput(o.path); + if (st === null) { + return 'output-missing'; + } + if (st.size !== o.size || st.mtimeNs !== o.mtimeNs) { + return 'output-tampered'; + } + } + return null; + }; + + for (const decl of current) { + const s = storedByKey.get(decl.key); + let reason : DirtyReason | null; + if (s !== undefined) { + reason = dirtyReason(s); + if (reason === null) { + plan.clean.push(decl); + continue; + } + } else { + reason = 'new'; + const sig = ruleInputSig(decl, hashes); + const candidates = renameIndex.get(renameKey(decl.template, sig)) ?? []; + const from = candidates.find(c => + c.outputs.length === decl.outputs.length && outputsIntact(c)); + if (from !== undefined) { + plan.renames.push({decl, from}); + continue; + } + } + if (adopt && decl.outputs.every(p => statOutput(p) !== null)) { + plan.adopt.push(decl); + } else { + plan.run.push({decl, reason}); + } + } + + return plan; +} diff --git a/tools/build/subst.ts b/tools/build/subst.ts new file mode 100644 index 00000000..1a3fc9fb --- /dev/null +++ b/tools/build/subst.ts @@ -0,0 +1,35 @@ + +import pathlib from 'path'; + +// A command spec entry: a shell command string, or an arbitrarily nested list +// of them (flattened, like the Lua flatten()). +export type Cmd = string | Cmd[]; + +export function flattenCmds(cmds : Cmd) : string[] { + if (typeof cmds === 'string') { + const trimmed = cmds.trim(); + return trimmed === '' ? [] : [trimmed]; + } + return cmds.flatMap(flattenCmds); +} + +export function basenameNoExt(path : string) : string { + const base = pathlib.basename(path); + const dot = base.lastIndexOf('.'); + return dot > 0 ? base.slice(0, dot) : base; +} + +// Tup-style substitutions: +// %f inputs, space-joined %b input basenames +// %o outputs, space-joined %B input basenames without extension +export function substitute(s : string, inputs : string[], outputs : string[]) : string { + return s.replace(/%([a-zA-Z])/g, (match, c : string) => { + switch (c) { + case 'f': return inputs.join(' '); + case 'o': return outputs.join(' '); + case 'b': return inputs.map(p => pathlib.basename(p)).join(' '); + case 'B': return inputs.map(basenameNoExt).join(' '); + default: throw new Error(`Unknown substitution ${match} in: ${s}`); + } + }); +}