Leahnaya/sprites
1
0
Fork 0
mirror of https://github.com/smogon/sprites.git synced 2026-08-20 00:34:22 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Implement tools/build core and Buildfile.ts

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Christopher Monsanto
2026-08-15 22:01:04 -04:00
parent 4ded7eea86
commit edda6cb310
12 changed files with 1346 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

190
Buildfile.ts Normal file
View File

@@ -0,0 +1,190 @@
import {base, compresspng, forEachRule, pad, rule, spriteglob, trimimg} from './tools/build/api.ts';
// Generate uniform size minisprites
forEachRule("src/minisprites/pokemon/gen6/*.png", {
display: "pad g6 minisprite %f",
cmds: [pad({w: 40, h: 30}), compresspng({config: "MINISPRITE"})],
}, "build/gen6-minisprites-padded/%b");
forEachRule("src/minisprites/items/*.png", {
display: "pad item minisprite %f",
cmds: [pad({w: 24, h: 24}), compresspng({config: "MINISPRITE"})],
}, "build/item-minisprites-padded/%b");
forEachRule("src/minisprites/pokemon/gen6/*.png", {
display: "trim g6 minisprite %f",
cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
}, "build/gen6-minisprites-trimmed/%b");
forEachRule("src/minisprites/items/*.png", {
display: "trim item minisprite %f",
cmds: [trimimg(), compresspng({config: "MINISPRITE"})],
}, "build/item-minisprites-trimmed/%b");
// Gen 9
forEachRule("src/gen9species/*.png", {
display: "96x96 %f",
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
cmds: [
"magick convert %f -trim +repage -resize 90x90 %o",
"gifsicle -O3 -b %o",
],
}, "build/gen9-modelslike/%B.gif");
// Gen 10
forEachRule("src/champions/*.png", {
display: "96x96 %f",
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
cmds: [
"magick convert %f -trim +repage -resize 90x90 %o",
"gifsicle -O3 -b %o",
],
}, "build/gen10-modelslike/%B.gif");
// Gen 5 CAPs...
forEachRule("src/sprites/gen5/*.png", [
// TODO, add customizable compression for gif
// ... or investigate using webp instead of both png/gif here
"magick convert %f %o",
"gifsicle -O3 -b %o",
], "build/gen5-gif/%B.gif");
// PS spritesheet
rule("ps-pokemon.sheet.mjs", {
display: "ps pokemon sheet",
// The sheet tool reads the minisprites (via readdir, which tup couldn't
// track) and the sprite data; declare them so changes rebuild the sheet.
deps: [
"src/minisprites/pokemon/gen6/*",
"data/species.json",
"data/items.json",
"tools/sheet/index.ts",
],
cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
}, "build/ps/pokemonicons-sheet.png");
// TODO: reenable when trainers are moved
// rule("ps-trainers.sheet.mjs", {
// display: "ps trainers sheet",
// cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
// }, "build/ps/trainers-sheet.png");
rule("ps-items.sheet.mjs", {
display: "ps items sheet",
deps: [
"src/minisprites/items/*",
"data/species.json",
"data/items.json",
"tools/sheet/index.ts",
],
cmds: ["node tools/sheet/index.ts %f %o", compresspng({config: "SPRITESHEET"})],
}, "build/ps/itemicons-sheet.png");
// PS pokeball icons
const balls = [
"src/_uncategorized/noncanonical/ui/battle/Ball-Normal.png",
"src/_uncategorized/noncanonical/ui/battle/Ball-Sick.png",
"src/_uncategorized/noncanonical/ui/battle/Ball-Null.png",
];
rule(balls, {
display: "pokemonicons-pokeball-sheet",
cmds: [
"magick convert -background transparent -gravity center -extent 40x30 %f +append %o",
compresspng({config: "SPRITESHEET"}),
],
}, "build/ps/pokemonicons-pokeball-sheet.png");
// Smogdex minisprites (webp)
forEachRule(spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false}), {
display: "webp minisprite %f",
cmds: ["cwebp -z 9 %f -o %o"],
}, "build/smogon/minisprites/%B.webp");
// Smogdex spritesheet
rule(spriteglob(["src/minisprites/pokemon/gen6/*", "src/minisprites/items/*"], {a: false}), {
display: "smogdex sheet",
deps: ["data/species.json", "data/items.json", "tools/smogdexspritesheet/index.ts"],
// build/smogon/spritesheet.png is an undeclared temporary; it is removed
// before the rule finishes.
cmds: [
"node tools/smogdexspritesheet/index.ts --image build/smogon/spritesheet.png --stylesheet build/smogon/spritesheet.css -- %f",
"cwebp -z 9 build/smogon/spritesheet.png -o build/smogon/spritesheet.webp",
"rm build/smogon/spritesheet.png",
],
}, ["build/smogon/spritesheet.webp", "build/smogon/spritesheet.css"]);
// Smogdex social images
const social = spriteglob(["src/models/*"], {b: false, s: false});
const socialSeen = new Set(social.map(base));
for (const file of spriteglob(["src/gen9species/*"], {b: false, s: false})) {
if (!socialSeen.has(base(file))) {
social.push(file);
socialSeen.add(base(file));
}
}
forEachRule(social, {
display: "fbsprite %f",
cmds: [
'magick convert "%f[0]" -trim -resize 150x150 -background white -gravity center -extent 198x198 -bordercolor black -border 1 %o',
compresspng({config: "MODELS"}),
],
}, "build/smogon/fbsprites/xy/%B.png");
forEachRule(social, {
display: "twittersprite %f",
cmds: [
'magick convert "%f[0]" -trim -resize 115x115 -background white -gravity center -extent 120x120 %o',
compresspng({config: "MODELS"}),
],
}, "build/smogon/twittersprites/xy/%B.png");
// Trainers
// TODO: reenable when trainers are moved
// forEachRule("src/canonical/trainers/*", {
// display: "pad trainer %f",
// cmds: [pad({w: 80, h: 80}), compresspng({config: "TRAINERS"})],
// }, "build/padded-trainers/canonical/%b");
// Padded Dex
const dexOutput = forEachRule("src/dex/*", {
display: "pad dex %f",
cmds: [pad({w: 120, h: 120}), compresspng({config: "DEX"})],
}, "build/padded-dex/%b");
// Build missing CAP dex
const dexSet = new Set(dexOutput.map(base));
const dexMissing = [];
for (const file of spriteglob(["src/sprites/gen5/*.gif", "src/models/*.gif"], {b: false, s: false})) {
if (!dexSet.has(base(file))) {
dexMissing.push(file);
dexSet.add(base(file));
}
}
forEachRule(dexMissing, {
display: "missing dex %B",
cmds: [
'magick convert "%f[0]" -trim %o',
'magick mogrify -background transparent -gravity center -resize "120x120>" -extent 120x120 %o',
compresspng({config: "DEX"}),
],
}, "build/padded-dex/%B.png");

1
package.json
View File

@@ -4,6 +4,7 @@
"build": "tsc --build tsconfig-workspace.json"
},
"devDependencies": {
"@types/node": "^24.10.7",
"typescript": "~7.0.2"
}
}

3
pnpm-lock.yaml generated
View File

@@ -8,6 +8,9 @@ importers:
.:
devDependencies:
'@types/node':
specifier: ^24.10.7
version: 24.13.3
typescript:
specifier: ~7.0.2
version: 7.0.2

228
tools/build/api.ts Normal file
View File

@@ -0,0 +1,228 @@
import fs from 'fs';
import pathlib from 'path';
import {createHash} from 'crypto';
import {type Cmd, flattenCmds, substitute, basenameNoExt} from './subst.ts';
export type {Cmd};
export interface CmdSpec {
display? : string;
// Tracked-but-not-substituted inputs: hashed and part of rule identity,
// but never expanded into %f. Use for files a tool reads on its own
// (e.g. tools/sheet readdirs the minisprite directories).
deps? : string | string[];
cmds : Cmd[];
}
export interface RuleDecl {
inputs : string[]; // %f source, ordered
deps : string[]; // hashed, never substituted
outputs : string[]; // post-substitution paths
command : string; // final ' && '-joined shell command
display : string | null; // post-substitution; cosmetic, not part of identity
template : string; // pre-substitution cmds + output basename template(s)
key : string; // identity for incremental skip
}
let rules : RuleDecl[] = [];
let config = new Map<string, string>();
export function setConfig(cfg : Map<string, string>) : void {
config = cfg;
}
export function getRules() : RuleDecl[] {
return rules;
}
export function resetRules() : void {
rules = [];
}
export function getconfig(name : string) : string | undefined {
const value = config.get(name);
return value === '' ? undefined : value;
}
function astable(x : string | string[] | undefined) : string[] {
if (x === undefined) {
return [];
}
return typeof x === 'string' ? [x] : x;
}
// Single-directory, single-'*' glob (all Tupfile patterns were of this form).
// Non-glob strings pass through literally; existence is checked at hash time.
// Results are sorted within a pattern; declared order is preserved across
// patterns (some rules, e.g. the pokeball sheet, are input-order-sensitive).
function globOne(pat : string) : string[] {
if (!pat.includes('*')) {
return [pat];
}
const dir = pathlib.dirname(pat);
const base = pathlib.basename(pat);
if (dir.includes('*') || base.indexOf('*') !== base.lastIndexOf('*')) {
throw new Error(`Unsupported glob pattern: ${pat}`);
}
const [prefix, suffix] = base.split('*') as [string, string];
const results = [];
for (const ent of fs.readdirSync(dir, {withFileTypes: true})) {
if (!ent.isFile() && !ent.isSymbolicLink()) {
continue;
}
const name = ent.name;
if (name.length >= prefix.length + suffix.length
&& name.startsWith(prefix) && name.endsWith(suffix)) {
results.push(dir === '.' ? name : `${dir}/${name}`);
}
}
results.sort();
return results;
}
export function glob(pats : string | string[]) : string[] {
return astable(pats).flatMap(globOne);
}
// tup.base: basename without directory or final extension
export function base(path : string) : string {
return basenameNoExt(path);
}
export interface SpriteData {
id : string;
data : Record<string, string | true>;
}
// Port of util/sprites.lua spritedata. Lua used gmatch("[^-]+"), which skips
// empty segments, hence the filter.
export function spritedata(basename : string) : SpriteData {
const parts = basename.split('-').filter(p => p !== '');
const data : Record<string, string | true> = {};
for (const part of parts.slice(1)) {
if (part.length === 1) {
data[part] = true;
} else {
data[part[0]!] = part.slice(1);
}
}
return {id: parts[0] ?? '', data};
}
export function spriteglob(pats : string | string[], flagspec? : Record<string, unknown>) : string[] {
return glob(pats).filter(filename => {
const sd = spritedata(base(filename));
for (const [k, v] of Object.entries(flagspec ?? {})) {
if (Boolean(v) !== Boolean(sd.data[k])) {
return false;
}
}
return true;
});
}
export function pad(opts : {w : number, h : number, input? : string, output? : string}) : string {
const input = opts.input ?? '%f';
const output = opts.output ?? '%o';
return `magick convert ${input} -background transparent -gravity center -extent ${opts.w}x${opts.h} ${output}`;
}
export function trimimg(opts : {input? : string, output? : string} = {}) : string {
return `magick convert ${opts.input ?? '%f'} -trim ${opts.output ?? '%o'}`;
}
interface CompressOpts {
pngquant? : string;
optipng? : string;
advpng? : string;
}
function compressopts(program : string, copts : CompressOpts) : void {
copts.pngquant = getconfig(`${program}_PNGQUANT`) ?? copts.pngquant;
copts.optipng = getconfig(`${program}_OPTIPNG`) ?? copts.optipng;
copts.advpng = getconfig(`${program}_ADVPNG`) ?? copts.advpng;
}
export function compresspng(opts : {config? : string, output? : string} = {}) : Cmd[] {
const output = opts.output ?? '%o';
const copts : CompressOpts = {};
compressopts('DEFAULT', copts);
if (opts.config) {
compressopts(opts.config, copts);
}
const cmds = [];
if (copts.pngquant !== undefined) {
// -f -o necessary to overwrite existing file
cmds.push(`pngquant -f -o ${output} ${copts.pngquant} ${output}`);
}
if (copts.optipng !== undefined) {
cmds.push(`optipng -q ${copts.optipng} ${output}`);
}
if (copts.advpng !== undefined) {
cmds.push(`advpng -q ${copts.advpng} ${output}`);
}
return cmds;
}
function normalizeSpec(spec : CmdSpec | Cmd[]) : CmdSpec {
return Array.isArray(spec) ? {cmds: spec} : spec;
}
// The rename-detection template deliberately excludes output directories
// (basename only) so that content-preserving moves across directories with
// identical processing still match, but extension changes (which change the
// bytes tools like magick produce) do not.
function templateOf(cmds : string[], outputTemplates : string[]) : string {
return cmds.join('\n') + '\0' + outputTemplates.map(t => pathlib.basename(t)).join('\0');
}
function keyOf(command : string, inputs : string[], deps : string[], outputs : string[]) : string {
const h = createHash('sha256');
h.update([command, inputs.join('\0'), deps.join('\0'), outputs.join('\0')].join('\x01'));
return h.digest('hex');
}
function makeRule(inputs : string[], deps : string[], spec : CmdSpec,
outputs : string[], outputTemplates : string[]) : RuleDecl {
const cmds = flattenCmds(spec.cmds).map(c => substitute(c, inputs, outputs));
if (cmds.length === 0) {
throw new Error(`Rule with no commands (outputs: ${outputs.join(' ')})`);
}
const command = cmds.join(' && ');
const decl : RuleDecl = {
inputs,
deps,
outputs,
command,
display: spec.display !== undefined ? substitute(spec.display, inputs, outputs) : null,
template: templateOf(flattenCmds(spec.cmds), outputTemplates),
key: keyOf(command, inputs, deps, outputs),
};
rules.push(decl);
return decl;
}
export function rule(input : string | string[], spec : CmdSpec | Cmd[],
output : string | string[]) : string[] {
const s = normalizeSpec(spec);
const outputs = astable(output);
const decl = makeRule(glob(input), glob(astable(s.deps)), s, outputs, outputs);
return decl.outputs;
}
export function forEachRule(input : string | string[], spec : CmdSpec | Cmd[],
output : string) : string[] {
const s = normalizeSpec(spec);
if (/%[fo]/.test(output)) {
throw new Error(`forEachRule output template may only use %b/%B: ${output}`);
}
const deps = glob(astable(s.deps));
const outputs = [];
for (const file of glob(input)) {
const decl = makeRule([file], deps, s, [substitute(output, [file], [])], [output]);
outputs.push(...decl.outputs);
}
return outputs;
}

25
tools/build/config.ts Normal file
View File

@@ -0,0 +1,25 @@
import fs from 'fs';
export function parseConfig(text : string) : Map<string, string> {
const result = new Map<string, string>();
for (let line of text.split('\n')) {
line = line.trim();
if (line === '' || line.startsWith('#')) {
continue;
}
const eq = line.indexOf('=');
if (eq === -1) {
throw new Error(`Invalid config line: ${line}`);
}
result.set(line.slice(0, eq).trim(), line.slice(eq + 1).trim());
}
return result;
}
export function loadConfig(path : string) : Map<string, string> {
if (!fs.existsSync(path)) {
return new Map();
}
return parseConfig(fs.readFileSync(path, 'utf8'));
}

234
tools/build/db.ts Normal file
View File

@@ -0,0 +1,234 @@
import fs from 'fs';
import pathlib from 'path';
import Database from 'better-sqlite3';
import type {RuleDecl} from './api.ts';
import type {FileStat} from './hash.ts';
import {BuildError} from './graph.ts';
export interface StoredRuleInput {
path : string;
isDep : boolean;
hash : Buffer;
}
export interface StoredRuleOutput {
path : string;
size : bigint | null; // null when the rule last failed
mtimeNs : bigint | null;
}
export interface StoredRule {
id : bigint;
key : string;
command : string;
display : string | null;
template : string;
inputSig : Buffer;
ok : boolean;
inputs : StoredRuleInput[]; // ordered: inputs (in %f order), then deps
outputs : StoredRuleOutput[]; // ordered: position-mapped for renames
}
export interface RecordedOutput {
path : string;
size : bigint;
mtimeNs : bigint;
}
const DDL = `
CREATE TABLE file_cache (
path TEXT PRIMARY KEY,
size INTEGER NOT NULL,
mtime_ns INTEGER NOT NULL,
hash BLOB NOT NULL
) WITHOUT ROWID;
CREATE TABLE rules (
id INTEGER PRIMARY KEY,
key TEXT NOT NULL UNIQUE,
command TEXT NOT NULL,
display TEXT,
template TEXT NOT NULL,
input_sig BLOB NOT NULL,
ok INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX rules_rename ON rules(template, input_sig);
CREATE TABLE rule_inputs (
rule_id INTEGER NOT NULL REFERENCES rules(id) ON DELETE CASCADE,
ord INTEGER NOT NULL,
path TEXT NOT NULL,
is_dep INTEGER NOT NULL DEFAULT 0,
hash BLOB NOT NULL,
PRIMARY KEY (rule_id, ord)
);
CREATE INDEX rule_inputs_path ON rule_inputs(path);
CREATE TABLE rule_outputs (
rule_id INTEGER NOT NULL REFERENCES rules(id) ON DELETE CASCADE,
ord INTEGER NOT NULL,
path TEXT NOT NULL,
size INTEGER,
mtime_ns INTEGER,
PRIMARY KEY (rule_id, ord)
);
CREATE INDEX rule_outputs_path ON rule_outputs(path);
`;
export class BuildDb {
private db : Database.Database;
constructor(dbPath : string) {
fs.mkdirSync(pathlib.dirname(dbPath), {recursive: true});
this.db = new Database(dbPath);
this.db.defaultSafeIntegers(true);
this.db.pragma('journal_mode = WAL');
this.db.pragma('foreign_keys = ON');
this.db.pragma('synchronous = NORMAL');
this.migrate();
}
private migrate() : void {
const version = Number(this.db.pragma('user_version', {simple: true}));
if (version === 0) {
this.db.exec('BEGIN;' + DDL + 'PRAGMA user_version = 1; COMMIT;');
} else if (version !== 1) {
throw new BuildError(
`Unknown build db schema version ${version}; delete .build/ and re-run with --adopt`);
}
}
loadFileCache() : Map<string, FileStat> {
const result = new Map<string, FileStat>();
const rows = this.db.prepare<[], {path : string, size : bigint, mtime_ns : bigint, hash : Buffer}>(
'SELECT path, size, mtime_ns, hash FROM file_cache').all();
for (const row of rows) {
result.set(row.path, {size: row.size, mtimeNs: row.mtime_ns, hash: row.hash});
}
return result;
}
saveFileCache(entries : Map<string, FileStat>) : void {
const upsert = this.db.prepare(`
INSERT INTO file_cache (path, size, mtime_ns, hash) VALUES (?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET
size = excluded.size, mtime_ns = excluded.mtime_ns, hash = excluded.hash`);
this.db.transaction(() => {
for (const [path, stat] of entries) {
upsert.run(path, stat.size, stat.mtimeNs, stat.hash);
}
})();
}
pruneFileCache(live : Set<string>) : void {
const paths = this.db.prepare<[], {path : string}>('SELECT path FROM file_cache').all();
const del = this.db.prepare('DELETE FROM file_cache WHERE path = ?');
this.db.transaction(() => {
for (const {path} of paths) {
if (!live.has(path)) {
del.run(path);
}
}
})();
}
loadStoredRules() : StoredRule[] {
const byId = new Map<bigint, StoredRule>();
const ruleRows = this.db.prepare<[], {
id : bigint, key : string, command : string, display : string | null,
template : string, input_sig : Buffer, ok : bigint,
}>('SELECT id, key, command, display, template, input_sig, ok FROM rules').all();
for (const row of ruleRows) {
byId.set(row.id, {
id: row.id,
key: row.key,
command: row.command,
display: row.display,
template: row.template,
inputSig: row.input_sig,
ok: row.ok !== 0n,
inputs: [],
outputs: [],
});
}
const inputRows = this.db.prepare<[], {rule_id : bigint, path : string, is_dep : bigint, hash : Buffer}>(
'SELECT rule_id, path, is_dep, hash FROM rule_inputs ORDER BY rule_id, ord').all();
for (const row of inputRows) {
byId.get(row.rule_id)!.inputs.push({path: row.path, isDep: row.is_dep !== 0n, hash: row.hash});
}
const outputRows = this.db.prepare<[], {rule_id : bigint, path : string, size : bigint | null, mtime_ns : bigint | null}>(
'SELECT rule_id, path, size, mtime_ns FROM rule_outputs ORDER BY rule_id, ord').all();
for (const row of outputRows) {
byId.get(row.rule_id)!.outputs.push({path: row.path, size: row.size, mtimeNs: row.mtime_ns});
}
return [...byId.values()];
}
// One transaction per completed rule: an interrupted build only ever
// contains fully-recorded rules. outputs === null records a failure (ok=0,
// output paths kept for GC, stats nulled so the rule stays dirty).
recordRuleResult(decl : RuleDecl, inputHashes : Map<string, Buffer>, sig : Buffer,
outputs : RecordedOutput[] | null) : void {
const ok = outputs !== null;
const upsert = this.db.prepare<[string, string, string | null, string, Buffer, bigint], {id : bigint}>(`
INSERT INTO rules (key, command, display, template, input_sig, ok)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(key) DO UPDATE SET
display = excluded.display, input_sig = excluded.input_sig, ok = excluded.ok
RETURNING id`);
const delInputs = this.db.prepare('DELETE FROM rule_inputs WHERE rule_id = ?');
const delOutputs = this.db.prepare('DELETE FROM rule_outputs WHERE rule_id = ?');
const insInput = this.db.prepare(
'INSERT INTO rule_inputs (rule_id, ord, path, is_dep, hash) VALUES (?, ?, ?, ?, ?)');
const insOutput = this.db.prepare(
'INSERT INTO rule_outputs (rule_id, ord, path, size, mtime_ns) VALUES (?, ?, ?, ?, ?)');
this.db.transaction(() => {
const {id} = upsert.get(decl.key, decl.command, decl.display, decl.template,
sig, ok ? 1n : 0n)!;
delInputs.run(id);
delOutputs.run(id);
let ord = 0;
for (const path of decl.inputs) {
insInput.run(id, ord++, path, 0, inputHashes.get(path)!);
}
for (const path of decl.deps) {
insInput.run(id, ord++, path, 1, inputHashes.get(path)!);
}
const outputRows = outputs ?? decl.outputs.map(path => ({path, size: null, mtimeNs: null}));
outputRows.forEach((o, i) => insOutput.run(id, i, o.path, o.size, o.mtimeNs));
})();
}
deleteRule(id : bigint) : void {
this.db.prepare('DELETE FROM rules WHERE id = ?').run(id);
}
close() : void {
this.db.pragma('wal_checkpoint(TRUNCATE)');
this.db.close();
}
}
// Guards against two concurrent builds; the exclusive transaction is released
// by the OS on any crash, so there are no stale lock files.
export function acquireLock(lockPath : string) : () => void {
fs.mkdirSync(pathlib.dirname(lockPath), {recursive: true});
const lock = new Database(lockPath, {timeout: 0});
try {
lock.exec('BEGIN EXCLUSIVE');
} catch (err) {
lock.close();
if ((err as {code? : string}).code === 'SQLITE_BUSY') {
throw new BuildError('Another build is already running.');
}
throw err;
}
return () => {
try {
lock.exec('COMMIT');
} catch {}
lock.close();
};
}

78
tools/build/exec.ts Normal file
View File

@@ -0,0 +1,78 @@
import {spawn} from 'child_process';
export interface ExecResult {
code : number | null;
signal : NodeJS.Signals | null;
output : string;
durationMs : number;
}
// The command script is fed to sh via stdin rather than -c: a single argv
// entry is capped by the kernel (MAX_ARG_STRLEN, ~128KB) and the largest %f
// expansion is already 80KB+.
export function runShell(command : string, opts : {cwd : string, signal : AbortSignal}) : Promise<ExecResult> {
return new Promise((resolve, reject) => {
const start = performance.now();
// detached: own process group, so an abort kills grandchildren
// (magick, optipng, ...) with one signal
const child = spawn('sh', [], {cwd: opts.cwd, detached: true, stdio: ['pipe', 'pipe', 'pipe']});
const chunks : Buffer[] = [];
child.stdout.on('data', c => chunks.push(c));
child.stderr.on('data', c => chunks.push(c));
child.stdin.on('error', () => {}); // EPIPE if the shell exits early
child.stdin.end(command + '\n');
let killTimer : NodeJS.Timeout | undefined;
const kill = (sig : NodeJS.Signals) => {
try {
process.kill(-child.pid!, sig);
} catch {}
};
const onAbort = () => {
kill('SIGTERM');
killTimer = setTimeout(() => kill('SIGKILL'), 5000);
killTimer.unref();
};
if (opts.signal.aborted) {
onAbort();
} else {
opts.signal.addEventListener('abort', onAbort, {once: true});
}
const cleanup = () => {
opts.signal.removeEventListener('abort', onAbort);
if (killTimer !== undefined) {
clearTimeout(killTimer);
}
};
child.on('error', err => {
cleanup();
reject(err);
});
child.on('close', (code, signal) => {
cleanup();
resolve({
code,
signal,
output: Buffer.concat(chunks).toString(),
durationMs: performance.now() - start,
});
});
});
}
export async function workerPool<T>(items : readonly T[], jobs : number,
fn : (item : T, index : number) => Promise<void>) : Promise<void> {
let next = 0;
const workers = [];
for (let i = 0; i < Math.max(1, Math.min(jobs, items.length)); i++) {
workers.push((async () => {
while (next < items.length) {
const index = next++;
await fn(items[index]!, index);
}
})());
}
await Promise.all(workers);
}

62
tools/build/graph.ts Normal file
View File

@@ -0,0 +1,62 @@
import type {RuleDecl} from './api.ts';
export class BuildError extends Error {}
export interface GraphResult {
order : RuleDecl[]; // topological, stable w.r.t. declaration order
generated : Set<string>; // every path produced by some rule
}
export function checkGraph(rules : RuleDecl[]) : GraphResult {
const owner = new Map<string, RuleDecl>();
for (const rule of rules) {
for (const output of rule.outputs) {
const other = owner.get(output);
if (other !== undefined) {
throw new BuildError(
`Output ${output} produced by multiple rules:\n ${other.command}\n ${rule.command}`);
}
owner.set(output, rule);
}
}
const consumers = new Map<RuleDecl, RuleDecl[]>();
const indegree = new Map<RuleDecl, number>();
for (const rule of rules) {
indegree.set(rule, 0);
}
for (const rule of rules) {
for (const input of [...rule.inputs, ...rule.deps]) {
const producer = owner.get(input);
if (producer !== undefined && producer !== rule) {
let list = consumers.get(producer);
if (list === undefined) {
consumers.set(producer, list = []);
}
list.push(rule);
indegree.set(rule, indegree.get(rule)! + 1);
}
}
}
const queue = rules.filter(r => indegree.get(r) === 0);
const order = [];
for (let i = 0; i < queue.length; i++) {
const rule = queue[i]!;
order.push(rule);
for (const consumer of consumers.get(rule) ?? []) {
const deg = indegree.get(consumer)! - 1;
indegree.set(consumer, deg);
if (deg === 0) {
queue.push(consumer);
}
}
}
if (order.length !== rules.length) {
const stuck = rules.filter(r => indegree.get(r)! > 0);
throw new BuildError(`Dependency cycle involving:\n ${stuck.map(r => r.command).join('\n ')}`);
}
return {order, generated: new Set(owner.keys())};
}

51
tools/build/hash.ts Normal file
View File

@@ -0,0 +1,51 @@
import fs from 'fs';
import {createHash} from 'crypto';
export interface FileStat {
size : bigint;
mtimeNs : bigint;
hash : Buffer;
}
export function hashFileSync(path : string) : Buffer {
return createHash('sha256').update(fs.readFileSync(path)).digest();
}
export interface ReconcileResult {
hashes : Map<string, Buffer>; // current content hash for every extant path
updated : Map<string, FileStat>; // cache entries that changed (to persist)
missing : string[]; // paths that don't exist or aren't files
}
// Content hashes with a stat cache: only files whose (size, mtime_ns) changed
// since the recorded cache entry are rehashed. mtime_ns exceeds 2^53, hence
// bigint stats throughout.
export function reconcileHashes(paths : Iterable<string>, cache : Map<string, FileStat>) : ReconcileResult {
const hashes = new Map<string, Buffer>();
const updated = new Map<string, FileStat>();
const missing = [];
for (const path of paths) {
let st;
try {
st = fs.statSync(path, {bigint: true});
} catch {
missing.push(path);
continue;
}
if (!st.isFile()) {
missing.push(path);
continue;
}
const cached = cache.get(path);
let hash;
if (cached !== undefined && cached.size === st.size && cached.mtimeNs === st.mtimeNs) {
hash = cached.hash;
} else {
hash = hashFileSync(path);
updated.set(path, {size: st.size, mtimeNs: st.mtimeNs, hash});
}
hashes.set(path, hash);
}
return {hashes, updated, missing};
}

307
tools/build/index.ts Normal file
View File

@@ -0,0 +1,307 @@
import fs from 'fs';
import os from 'os';
import pathlib from 'path';
import {fileURLToPath, pathToFileURL} from 'url';
import {program} from 'commander';
import debugfn from 'debug';
import {getRules, type RuleDecl, setConfig} from './api.ts';
import {loadConfig} from './config.ts';
import {acquireLock, BuildDb, type RecordedOutput} from './db.ts';
import {runShell, workerPool} from './exec.ts';
import {BuildError, checkGraph} from './graph.ts';
import {reconcileHashes} from './hash.ts';
import {computePlan, type OutputStat, ruleInputSig} from './plan.ts';
const debug = debugfn('build');
program
.option('-j, --jobs <n>', 'number of parallel jobs', String(os.availableParallelism()))
.option('-n, --dry-run', 'print the plan without changing anything')
.option('--adopt', 'record existing outputs as up to date instead of running (migration)')
.option('--fail-fast', 'stop scheduling new rules after the first failure')
.option('--config <file>', 'config file', 'build.config')
.option('-v, --verbose', 'print more detail');
program.parse(process.argv);
const opts = program.opts();
const root = pathlib.resolve(fileURLToPath(import.meta.url), '../../..');
process.chdir(root);
function statPath(path : string) : OutputStat | null {
try {
const st = fs.statSync(path, {bigint: true});
return st.isFile() ? {size: st.size, mtimeNs: st.mtimeNs} : null;
} catch {
return null;
}
}
function label(decl : RuleDecl) : string {
return decl.display ?? decl.command.split(' && ')[0]!;
}
function indent(text : string) : string {
return text.replace(/\n$/, '').split('\n').map(l => ' ' + l).join('\n');
}
async function main() : Promise<number> {
const dryRun = Boolean(opts.dryRun);
const releaseLock = dryRun ? null : acquireLock('.build/lock.sqlite');
const db = new BuildDb('.build/db.sqlite');
try {
// Phase 1: evaluate the rule set
setConfig(loadConfig(opts.config));
await import(pathToFileURL(pathlib.join(root, 'Buildfile.ts')).href);
const rules = getRules();
const {order, generated} = checkGraph(rules);
for (const rule of rules) {
for (const path of [...rule.inputs, ...rule.deps]) {
if (generated.has(path)) {
// The planner assumes all inputs are hashable before
// execution; support this when a rule needs it.
throw new BuildError(`Rules consuming generated files are not yet supported: ${path}`);
}
}
}
debug('%d rules', rules.length);
// Phase 2: hash source files (stat-cached)
const sources = new Set<string>();
for (const rule of rules) {
for (const path of [...rule.inputs, ...rule.deps]) {
sources.add(path);
}
}
const {hashes, updated, missing} = reconcileHashes(sources, db.loadFileCache());
if (missing.length > 0) {
throw new BuildError(`Missing input files:\n ${missing.slice(0, 20).join('\n ')}`
+ (missing.length > 20 ? `\n ... and ${missing.length - 20} more` : ''));
}
if (!dryRun && updated.size > 0) {
db.saveFileCache(updated);
}
debug('hashed %d files (%d cached)', sources.size, sources.size - updated.size);
// Phase 3: plan
const statMemo = new Map<string, OutputStat | null>();
const statOutput = (path : string) => {
let st = statMemo.get(path);
if (st === undefined) {
statMemo.set(path, st = statPath(path));
}
return st;
};
const plan = computePlan({
current: rules,
stored: db.loadStoredRules(),
hashes,
statOutput,
adopt: Boolean(opts.adopt),
});
const staleOutputs = [];
const currentOutputs = new Set(rules.flatMap(r => r.outputs));
for (const s of plan.stale) {
for (const o of s.outputs) {
if (!currentOutputs.has(o.path)) {
staleOutputs.push(o.path);
}
}
}
if (plan.run.length + plan.renames.length + plan.adopt.length + plan.stale.length === 0) {
console.log(`${plan.clean.length} rules up to date.`);
return 0;
}
if (opts.verbose || dryRun) {
for (const {decl, reason} of plan.run) {
console.log(`run (${reason}): ${label(decl)}`);
}
for (const {decl, from} of plan.renames) {
console.log(`rename: ${from.outputs.map(o => o.path).join(' ')} -> ${decl.outputs.join(' ')}`);
}
for (const decl of plan.adopt) {
console.log(`adopt: ${label(decl)}`);
}
for (const path of staleOutputs) {
console.log(`delete: ${path}`);
}
}
if (dryRun) {
console.log(`would run ${plan.run.length}, rename ${plan.renames.length}, `
+ `adopt ${plan.adopt.length}, delete ${staleOutputs.length} outputs `
+ `(${plan.clean.length} up to date)`);
return 0;
}
// Phase 4: renames (before stale deletion: sources must still exist)
for (const {decl, from} of plan.renames) {
const recorded : RecordedOutput[] = [];
for (let i = 0; i < decl.outputs.length; i++) {
const src = from.outputs[i]!.path;
const dst = decl.outputs[i]!;
if (src !== dst) {
fs.mkdirSync(pathlib.dirname(dst), {recursive: true});
fs.copyFileSync(src, dst);
}
const st = statPath(dst);
if (st === null) {
throw new BuildError(`Rename copy failed: ${src} -> ${dst}`);
}
recorded.push({path: dst, size: st.size, mtimeNs: st.mtimeNs});
}
db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded);
}
// Phase 5: delete outputs of removed rules, prune empty dirs
const staleDirs = new Set<string>();
for (const s of plan.stale) {
for (const o of s.outputs) {
if (!currentOutputs.has(o.path)) {
try {
fs.unlinkSync(o.path);
} catch (err) {
if ((err as {code? : string}).code !== 'ENOENT') {
throw err;
}
}
staleDirs.add(pathlib.dirname(o.path));
}
}
db.deleteRule(s.id);
}
for (let dir of staleDirs) {
while (dir.startsWith('build/')) {
try {
fs.rmdirSync(dir);
} catch {
break;
}
dir = pathlib.dirname(dir);
}
}
db.pruneFileCache(sources);
// Phase 6: adoption (migration): trust existing outputs
for (const decl of plan.adopt) {
const recorded = decl.outputs.map(path => {
const st = statPath(path)!;
return {path, size: st.size, mtimeNs: st.mtimeNs};
});
db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded);
}
// Phase 7: execute
// The worker pool does not serialize producers before consumers; that
// is safe because rules consuming generated files are rejected above.
// `order` is used for a stable, declaration-ordered schedule.
const orderIndex = new Map(order.map((r, i) => [r, i]));
const runList = [...plan.run].sort((a, b) => orderIndex.get(a.decl)! - orderIndex.get(b.decl)!);
const ac = new AbortController();
let interrupted = false;
const onSignal = () => {
if (interrupted) {
process.exit(130);
}
interrupted = true;
console.error('\nInterrupted; waiting for running rules to stop...');
ac.abort();
};
process.on('SIGINT', onSignal);
process.on('SIGTERM', onSignal);
const failures : RuleDecl[] = [];
let done = 0;
await workerPool(runList, parseInt(opts.jobs, 10), async ({decl}) => {
if (ac.signal.aborted) {
return;
}
for (const out of decl.outputs) {
fs.mkdirSync(pathlib.dirname(out), {recursive: true});
}
const result = await runShell(decl.command, {cwd: root, signal: ac.signal});
if (ac.signal.aborted && result.code !== 0) {
return; // killed by the abort, not a real failure; stays dirty
}
let recorded : RecordedOutput[] | null = null;
const missingOutputs = [];
if (result.code === 0) {
recorded = [];
for (const path of decl.outputs) {
const st = statPath(path);
if (st === null) {
missingOutputs.push(path);
recorded = null;
break;
}
recorded.push({path, size: st.size, mtimeNs: st.mtimeNs});
}
}
db.recordRuleResult(decl, hashes, ruleInputSig(decl, hashes), recorded);
done++;
if (recorded !== null) {
console.log(`[${done}/${runList.length}] ${label(decl)}`);
if (result.output !== '') {
console.log(indent(result.output));
}
} else {
failures.push(decl);
console.error(`[${done}/${runList.length}] FAILED: ${label(decl)}`);
console.error(` command: ${decl.command}`);
if (result.output !== '') {
console.error(indent(result.output));
}
if (missingOutputs.length > 0) {
console.error(` command succeeded but did not produce: ${missingOutputs.join(' ')}`);
}
if (opts.failFast) {
ac.abort();
}
}
});
process.off('SIGINT', onSignal);
process.off('SIGTERM', onSignal);
// Phase 8: summary
const parts = [`${plan.clean.length} up to date`];
if (runList.length > 0) {
parts.push(`${done - failures.length} ran`);
}
if (plan.renames.length > 0) {
parts.push(`${plan.renames.length} renamed`);
}
if (plan.adopt.length > 0) {
parts.push(`${plan.adopt.length} adopted`);
}
if (plan.stale.length > 0) {
parts.push(`${plan.stale.length} removed`);
}
if (failures.length > 0) {
parts.push(`${failures.length} FAILED`);
}
console.log(parts.join(', ') + '.');
if (failures.length > 0) {
console.error('Failed rules:');
for (const decl of failures) {
console.error(` ${label(decl)}`);
}
}
return interrupted ? 130 : failures.length > 0 ? 1 : 0;
} finally {
db.close();
releaseLock?.();
}
}
try {
process.exitCode = await main();
} catch (err) {
if (err instanceof BuildError) {
console.error(err.message);
process.exitCode = 1;
} else {
throw err;
}
}

132
tools/build/plan.ts Normal file
View File

@@ -0,0 +1,132 @@
import {createHash} from 'crypto';
import type {RuleDecl} from './api.ts';
import type {StoredRule} from './db.ts';
export type DirtyReason = 'new' | 'failed-last-run' | 'input-changed'
| 'output-missing' | 'output-tampered';
export interface OutputStat {
size : bigint;
mtimeNs : bigint;
}
export interface Plan {
clean : RuleDecl[];
run : {decl : RuleDecl, reason : DirtyReason}[];
renames : {decl : RuleDecl, from : StoredRule}[];
stale : StoredRule[];
adopt : RuleDecl[];
}
export function inputSig(hashes : Buffer[]) : Buffer {
const h = createHash('sha256');
for (const hash of hashes) {
h.update(hash);
}
return h.digest();
}
export function ruleInputSig(decl : RuleDecl, hashes : Map<string, Buffer>) : Buffer {
return inputSig([...decl.inputs, ...decl.deps].map(p => hashes.get(p)!));
}
function renameKey(template : string, sig : Buffer) : string {
return template + '\0' + sig.toString('hex');
}
export function computePlan(opts : {
current : RuleDecl[],
stored : StoredRule[],
hashes : Map<string, Buffer>,
statOutput : (path : string) => OutputStat | null,
adopt : boolean,
}) : Plan {
const {current, stored, hashes, statOutput, adopt} = opts;
const storedByKey = new Map<string, StoredRule>();
const renameIndex = new Map<string, StoredRule[]>();
for (const s of stored) {
storedByKey.set(s.key, s);
if (s.ok) {
const key = renameKey(s.template, s.inputSig);
let list = renameIndex.get(key);
if (list === undefined) {
renameIndex.set(key, list = []);
}
list.push(s);
}
}
const currentKeys = new Set(current.map(r => r.key));
const plan : Plan = {
clean: [],
run: [],
renames: [],
stale: stored.filter(s => !currentKeys.has(s.key)),
adopt: [],
};
// A stored rule's outputs are intact iff every recorded output exists on
// disk with its recorded stat. Rename sources must pass this (never copy
// tampered or unverified bytes).
const outputsIntact = (s : StoredRule) : boolean =>
s.outputs.every(o => {
if (o.size === null || o.mtimeNs === null) {
return false;
}
const st = statOutput(o.path);
return st !== null && st.size === o.size && st.mtimeNs === o.mtimeNs;
});
const dirtyReason = (s : StoredRule) : DirtyReason | null => {
if (!s.ok) {
return 'failed-last-run';
}
for (const inp of s.inputs) {
if (!hashes.get(inp.path)?.equals(inp.hash)) {
return 'input-changed';
}
}
for (const o of s.outputs) {
const st = statOutput(o.path);
if (st === null) {
return 'output-missing';
}
if (st.size !== o.size || st.mtimeNs !== o.mtimeNs) {
return 'output-tampered';
}
}
return null;
};
for (const decl of current) {
const s = storedByKey.get(decl.key);
let reason : DirtyReason | null;
if (s !== undefined) {
reason = dirtyReason(s);
if (reason === null) {
plan.clean.push(decl);
continue;
}
} else {
reason = 'new';
const sig = ruleInputSig(decl, hashes);
const candidates = renameIndex.get(renameKey(decl.template, sig)) ?? [];
const from = candidates.find(c =>
c.outputs.length === decl.outputs.length && outputsIntact(c));
if (from !== undefined) {
plan.renames.push({decl, from});
continue;
}
}
if (adopt && decl.outputs.every(p => statOutput(p) !== null)) {
plan.adopt.push(decl);
} else {
plan.run.push({decl, reason});
}
}
return plan;
}

35
tools/build/subst.ts Normal file
View File

@@ -0,0 +1,35 @@
import pathlib from 'path';
// A command spec entry: a shell command string, or an arbitrarily nested list
// of them (flattened, like the Lua flatten()).
export type Cmd = string | Cmd[];
export function flattenCmds(cmds : Cmd) : string[] {
if (typeof cmds === 'string') {
const trimmed = cmds.trim();
return trimmed === '' ? [] : [trimmed];
}
return cmds.flatMap(flattenCmds);
}
export function basenameNoExt(path : string) : string {
const base = pathlib.basename(path);
const dot = base.lastIndexOf('.');
return dot > 0 ? base.slice(0, dot) : base;
}
// Tup-style substitutions:
// %f inputs, space-joined %b input basenames
// %o outputs, space-joined %B input basenames without extension
export function substitute(s : string, inputs : string[], outputs : string[]) : string {
return s.replace(/%([a-zA-Z])/g, (match, c : string) => {
switch (c) {
case 'f': return inputs.join(' ');
case 'o': return outputs.join(' ');
case 'b': return inputs.map(p => pathlib.basename(p)).join(' ');
case 'B': return inputs.map(basenameNoExt).join(' ');
default: throw new Error(`Unknown substitution ${match} in: ${s}`);
}
});
}