From 8bf512480ade87166f8bfd2d681d6f670978e8db Mon Sep 17 00:00:00 2001 From: Christopher Monsanto Date: Mon, 17 Aug 2026 14:51:08 -0400 Subject: [PATCH] Replace the .env deploy command with deploy.json5 An untracked json5 config maps deploy names to a buildFile plus (subset globs, cmd) entries: the deploy verb builds the named buildFiles, finishes each, routes the outputs through the subset globs, and pipes one tar per entry to its command. Every glob must match an output and every output must be covered by some entry. The tar no longer carries __key; the upload command names the asset set itself (smogonctl assets upload sprites). Co-Authored-By: Claude Fable 5 --- .gitignore | 2 +- README.md | 25 ++++++++- assets.build.ts | 5 -- package.json | 2 +- pnpm-lock.yaml | 10 ++++ tools/deploy/config.ts | 88 ++++++++++++++++++++++++++++++++ tools/deploy/index.ts | 58 ++++++++++++--------- tools/deploy/package.json | 1 + tools/deploy/queue.ts | 4 +- tools/deploy/test/api.test.ts | 21 ++++++-- tools/deploy/test/config.test.ts | 62 ++++++++++++++++++++++ 11 files changed, 237 insertions(+), 41 deletions(-) create mode 100644 tools/deploy/config.ts create mode 100644 tools/deploy/test/config.test.ts diff --git a/.gitignore b/.gitignore index e12159ad..5932f34d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ pnpm-debug.log .cache/ .build/ build.config -.env +deploy.json5 diff --git a/README.md b/README.md index 455c17cf..8c25e06b 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,9 @@ reuses the build's digests. All state lives in `.build/`. ``` $ pnpm build # build every deploy's rules, GC stale state -$ pnpm deploy # assets.build.ts -> tar -> DEPLOY_COMMAND (.env) -$ node tools/deploy/index.ts build ps.build.ts # build one deploy's rules +$ pnpm deploy # every deploy in deploy.json5 +$ node tools/deploy/index.ts deploy assets # one named deploy +$ node tools/deploy/index.ts build ps.build.ts # build one deploy's rules $ node tools/deploy/index.ts run smogon.build.ts -o deploy/smogon $ node tools/deploy/index.ts inspect src/minisprites/items/i1.png -o /tmp/out ``` @@ -76,6 +77,26 @@ copies the outputs out under readable names for eyeballing. Useful flags: `-j ` parallelism, `-n` dry run, `-v` verbose, `--fail-fast` stop after the first failure. +## Deploying + +`deploy` reads `deploy.json5` at the repo root (not tracked by git). It maps +deploy names to a buildFile and a list of (subset, cmd) entries: after +building and finishing the buildFile, each entry's globs select a subset of +the finish outputs, which are tarred and piped to the entry's command on +stdin. Every glob must match something, and every output must be covered by +some entry. + +```json5 +{ + assets: { + buildFile: "assets.build.ts", + deploy: [ + {subset: ["**"], cmd: "ssh smogon smogonctl assets upload sprites"}, + ], + }, +} +``` + ## Configuration Build settings are configurable in `build.config` (not tracked by git). diff --git a/assets.build.ts b/assets.build.ts index f069765d..5783574c 100644 --- a/assets.build.ts +++ b/assets.build.ts @@ -39,11 +39,6 @@ const forumG6 = gen6Padded(); export default defineDeploy({ finish(ctx) { - // The upload contract wants __key first in the tar, naming the asset - // set; the tar packer emits ops in queue order, so it has to be the - // first op. - ctx.write("__key", "sprites"); - // Dex spritesheet assets: hash-stamped css + webp. The css suffix // pointer rides in __meta/ for the dex to read. { diff --git a/package.json b/package.json index 1e5bdb86..83cf5255 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "type": "module", "scripts": { "build": "node tools/deploy/index.ts build", - "deploy": "node tools/deploy/index.ts deploy assets.build.ts", + "deploy": "node tools/deploy/index.ts deploy", "check": "tsc --build tsconfig-workspace.json" }, "dependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8f96ce04..53ad154e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,9 @@ importers: debug: specifier: ^4.1.1 version: 4.4.3 + json5: + specifier: ^2.2.3 + version: 2.2.3 tar-stream: specifier: ^3.0.0 version: 3.2.0 @@ -547,6 +550,11 @@ packages: json-stringify-safe@5.0.1: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + jsprim@1.4.2: resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} engines: {node: '>=0.6.0'} @@ -1161,6 +1169,8 @@ snapshots: json-stringify-safe@5.0.1: {} + json5@2.2.3: {} + jsprim@1.4.2: dependencies: assert-plus: 1.0.0 diff --git a/tools/deploy/config.ts b/tools/deploy/config.ts new file mode 100644 index 00000000..24452bdc --- /dev/null +++ b/tools/deploy/config.ts @@ -0,0 +1,88 @@ + +import fs from 'fs'; +import nodePath from 'path'; + +import JSON5 from 'json5'; + +import {BuildError} from '../build/errors.ts'; + +export interface DeployEntry { + subset : string[]; + cmd : string; +} + +export interface DeployTarget { + buildFile : string; + deploy : DeployEntry[]; +} + +export type DeployConfig = Map; + +function isStringArray(v : unknown) : v is string[] { + return Array.isArray(v) && v.every(x => typeof x === 'string'); +} + +export function loadDeployConfig(path : string) : DeployConfig { + let text : string; + try { + text = fs.readFileSync(path, 'utf8'); + } catch { + throw new BuildError(`missing ${path}; see README ("Deploying") for the schema`); + } + let raw : unknown; + try { + raw = JSON5.parse(text); + } catch (err) { + throw new BuildError(`${path}: ${(err as Error).message}`); + } + if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) { + throw new BuildError(`${path}: top level must be an object of deploy names`); + } + + const config : DeployConfig = new Map(); + for (const [name, t] of Object.entries(raw)) { + const target = t as Partial; + if (typeof t !== 'object' || t === null || typeof target.buildFile !== 'string' + || !Array.isArray(target.deploy)) { + throw new BuildError(`${path}: ${name}: expected {buildFile: string, deploy: [...]}`); + } + for (const e of target.deploy as Partial[]) { + if (typeof e !== 'object' || e === null + || !isStringArray(e.subset) || typeof e.cmd !== 'string') { + throw new BuildError( + `${path}: ${name}: each deploy entry needs {subset: string[], cmd: string}`); + } + } + config.set(name, target as DeployTarget); + } + return config; +} + +// Route finish outputs to deploy entries: per entry, the set of dsts its +// subset globs match. Every glob must match something and every dst must be +// covered by some entry; to unship an output, don't emit it in finish. +export function matchSubsets(dsts : readonly string[], + entries : readonly DeployEntry[]) : Set[] { + const covered = new Set(); + const perEntry = entries.map(entry => { + const matched = new Set(); + for (const glob of entry.subset) { + const hits = dsts.filter(d => nodePath.matchesGlob(d, glob)); + if (hits.length === 0) { + throw new BuildError(`subset glob matches no outputs: ${glob}`); + } + for (const hit of hits) { + matched.add(hit); + covered.add(hit); + } + } + return matched; + }); + const uncovered = dsts.filter(d => !covered.has(d)); + if (uncovered.length > 0) { + const shown = uncovered.slice(0, 10).join('\n '); + const more = uncovered.length > 10 ? `\n ... and ${uncovered.length - 10} more` : ''; + throw new BuildError(`outputs not covered by any deploy entry:\n ${shown}${more}`); + } + return perEntry; +} diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index ae205f12..7188525e 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -16,6 +16,7 @@ import {killAllProcessGroups} from '../build/exec.ts'; import {setConfig} from '../build/helpers.ts'; import {Store, acquireLock, dbVersion} from '../build/store.ts'; import {type DeploySpec, makeCtx} from './api.ts'; +import {loadDeployConfig, matchSubsets} from './config.ts'; import {ActionQueue} from './queue.ts'; const root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..'); @@ -173,35 +174,42 @@ common(program.command('build [files...]')) process.exitCode = await buildThen(getDecls(), opts, gc); }); -common(program.command('deploy ')) - .description('build, finish, and pipe the tar to DEPLOY_COMMAND from .env') - .action(async (file : string, opts : CommonOpts) => { - try { - process.loadEnvFile('.env'); - } catch { - console.error(`missing .env; set DEPLOY_COMMAND="ssh smogon smogonctl assets upload"`); - process.exitCode = 1; - return; - } - const command = process.env.DEPLOY_COMMAND; - if (command === undefined) { - console.error(`DEPLOY_COMMAND not set in .env`); - process.exitCode = 1; - return; +common(program.command('deploy [names...]')) + .description('build, finish, and pipe each subset tar to its command from deploy.json5') + .action(async (names : string[], opts : CommonOpts) => { + const config = loadDeployConfig('deploy.json5'); + const targets = names.length > 0 ? names : [...config.keys()]; + for (const name of targets) { + if (!config.has(name)) { + throw new BuildError(`deploy.json5: no deploy named ${name}`); + } } setConfig(loadConfig(opts.config)); - const specs = await importDeploys([file]); + const files = [...new Set(targets.map(n => config.get(n)!.buildFile))]; + const specs = await importDeploys(files); process.exitCode = await buildThen(getDecls(), opts, false, async () => { - const aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose)); - if (aq === null) { - return 1; + for (const name of targets) { + const target = config.get(name)!; + const aq = await runFinish(finishOf(specs, target.buildFile), Boolean(opts.verbose)); + if (aq === null) { + return 1; + } + const dsts = aq.log.filter(e => e.type === 'Op').map(e => e.dst); + const subsets = matchSubsets(dsts, target.deploy); + for (const [i, entry] of target.deploy.entries()) { + const matched = subsets[i]!; + console.log(`${name}: ${matched.size} files | ${entry.cmd}`); + const upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'inherit', 'inherit']}); + // If the command dies early we report its exit code; don't + // also crash on the resulting EPIPE. + upload.stdin!.on('error', () => {}); + aq.pack(dst => matched.has(dst)).pipe(upload.stdin!); + if (await waitExit(upload) !== 0) { + return 1; + } + } } - const upload = spawn(command, {shell: true, stdio: ['pipe', 'inherit', 'inherit']}); - // If the command dies early we report its exit code; don't also - // crash on the resulting EPIPE. - upload.stdin!.on('error', () => {}); - aq.pack().pipe(upload.stdin!); - return await waitExit(upload) !== 0 ? 1 : 0; + return 0; }); }); diff --git a/tools/deploy/package.json b/tools/deploy/package.json index d594f791..695f5f26 100644 --- a/tools/deploy/package.json +++ b/tools/deploy/package.json @@ -6,6 +6,7 @@ "base32-encode": "^2.0.0", "commander": "^5.1.0", "debug": "^4.1.1", + "json5": "^2.2.3", "tar-stream": "^3.0.0" }, "scripts": { diff --git a/tools/deploy/queue.ts b/tools/deploy/queue.ts index ff244fea..bba14691 100644 --- a/tools/deploy/queue.ts +++ b/tools/deploy/queue.ts @@ -160,12 +160,12 @@ export class ActionQueue { } } - pack() : NodeJS.ReadableStream { + pack(filter? : (dst : string) => boolean) : NodeJS.ReadableStream { if (!this.valid) throw new Error(`Invalid ActionQueue`); let t = tar.pack(); for (const entry of this.log) { - if (entry.type !== 'Op') + if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) continue; const op = entry.op; if (op.type === 'Copy'){ diff --git a/tools/deploy/test/api.test.ts b/tools/deploy/test/api.test.ts index 7360e2b8..4fe8b5c5 100644 --- a/tools/deploy/test/api.test.ts +++ b/tools/deploy/test/api.test.ts @@ -88,7 +88,8 @@ test('ctx.list sorts, parses extensions, skips dotfiles and directories', () => ]); }); -function packedEntries(aq : ActionQueue) : Promise<{name : string, data : string}[]> { +function packedEntries(aq : ActionQueue, filter? : (dst : string) => boolean) + : Promise<{name : string, data : string}[]> { return new Promise((resolve, reject) => { const extract = tar.extract(); const entries : {name : string, data : string}[] = []; @@ -102,22 +103,32 @@ function packedEntries(aq : ActionQueue) : Promise<{name : string, data : string }); extract.on('finish', () => resolve(entries)); extract.on('error', reject); - aq.pack().pipe(extract); + aq.pack(filter).pipe(extract); }); } -test('pack preserves op order with __key first', async () => { +test('pack preserves op order', async () => { const aq = new ActionQueue(); - aq.write('sprites', '__key'); aq.write('zzz', 'z.txt'); aq.write('aaa', 'a.txt'); assert.deepEqual(await packedEntries(aq), [ - {name: '__key', data: 'sprites'}, {name: 'z.txt', data: 'zzz'}, {name: 'a.txt', data: 'aaa'}, ]); }); +test('pack with a filter packs only matching entries in order', async () => { + const aq = new ActionQueue(); + aq.write('1', 'xy/a.png'); + aq.write('2', 'meta/m.json'); + aq.write('3', 'xy/b.png'); + const subset = await packedEntries(aq, dst => dst.startsWith('xy/')); + assert.deepEqual(subset, [ + {name: 'xy/a.png', data: '1'}, + {name: 'xy/b.png', data: '3'}, + ]); +}); + test('duplicate and absolute destinations invalidate the queue', () => { const dup = new ActionQueue(); dup.write('a', 'x.txt'); diff --git a/tools/deploy/test/config.test.ts b/tools/deploy/test/config.test.ts new file mode 100644 index 00000000..4a42b068 --- /dev/null +++ b/tools/deploy/test/config.test.ts @@ -0,0 +1,62 @@ + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import pathlib from 'node:path'; +import {test} from 'node:test'; + +import {loadDeployConfig, matchSubsets} from '../config.ts'; + +function configFile(text : string) : string { + const dir = fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-config-test-')); + const p = pathlib.join(dir, 'deploy.json5'); + fs.writeFileSync(p, text); + return p; +} + +test('loadDeployConfig parses json5 with comments and trailing commas', () => { + const config = loadDeployConfig(configFile(`{ + // dex assets + assets: { + buildFile: "assets.build.ts", + deploy: [ + {subset: ["**"], cmd: "cat > /dev/null"}, + ], + }, + }`)); + assert.deepEqual([...config.keys()], ['assets']); + assert.deepEqual(config.get('assets'), { + buildFile: 'assets.build.ts', + deploy: [{subset: ['**'], cmd: 'cat > /dev/null'}], + }); +}); + +test('loadDeployConfig rejects missing files and malformed shapes', () => { + assert.throws(() => loadDeployConfig('/nonexistent/deploy.json5'), /missing/); + assert.throws(() => loadDeployConfig(configFile('[1]')), /must be an object/); + assert.throws(() => loadDeployConfig(configFile('{a: {deploy: []}}')), /expected \{buildFile/); + assert.throws(() => loadDeployConfig(configFile( + '{a: {buildFile: "x.build.ts", deploy: [{subset: "**", cmd: "c"}]}}')), + /needs \{subset/); + assert.throws(() => loadDeployConfig(configFile('{a: {buildFile:')), /deploy.json5:/); +}); + +test('matchSubsets routes dsts to entries, overlap allowed', () => { + const dsts = ['xy/a.gif', 'xy/manifest.json', 'xyicons/b.png']; + const [xy, manifests] = matchSubsets(dsts, [ + {subset: ['xy/**', 'xyicons/**'], cmd: 'one'}, + {subset: ['**/manifest.json'], cmd: 'two'}, + ]); + assert.deepEqual([...xy!].sort(), dsts); + assert.deepEqual([...manifests!], ['xy/manifest.json']); +}); + +test('matchSubsets errors on a glob matching nothing', () => { + assert.throws(() => matchSubsets(['xy/a.gif'], [{subset: ['xy/**', 'zz/**'], cmd: 'c'}]), + /matches no outputs: zz\/\*\*/); +}); + +test('matchSubsets errors on uncovered outputs, listing them', () => { + assert.throws(() => matchSubsets(['xy/a.gif', 'stray.txt'], [{subset: ['xy/**'], cmd: 'c'}]), + /not covered by any deploy entry:\n {2}stray.txt/); +});