diff --git a/.gitignore b/.gitignore index e12159ad..5932f34d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ pnpm-debug.log .cache/ .build/ build.config -.env +deploy.json5 diff --git a/README.md b/README.md index 455c17cf..8c25e06b 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,9 @@ reuses the build's digests. All state lives in `.build/`. ``` $ pnpm build # build every deploy's rules, GC stale state -$ pnpm deploy # assets.build.ts -> tar -> DEPLOY_COMMAND (.env) -$ node tools/deploy/index.ts build ps.build.ts # build one deploy's rules +$ pnpm deploy # every deploy in deploy.json5 +$ node tools/deploy/index.ts deploy assets # one named deploy +$ node tools/deploy/index.ts build ps.build.ts # build one deploy's rules $ node tools/deploy/index.ts run smogon.build.ts -o deploy/smogon $ node tools/deploy/index.ts inspect src/minisprites/items/i1.png -o /tmp/out ``` @@ -76,6 +77,26 @@ copies the outputs out under readable names for eyeballing. Useful flags: `-j ` parallelism, `-n` dry run, `-v` verbose, `--fail-fast` stop after the first failure. +## Deploying + +`deploy` reads `deploy.json5` at the repo root (not tracked by git). It maps +deploy names to a buildFile and a list of (subset, cmd) entries: after +building and finishing the buildFile, each entry's globs select a subset of +the finish outputs, which are tarred and piped to the entry's command on +stdin. Every glob must match something, and every output must be covered by +some entry. + +```json5 +{ + assets: { + buildFile: "assets.build.ts", + deploy: [ + {subset: ["**"], cmd: "ssh smogon smogonctl assets upload sprites"}, + ], + }, +} +``` + ## Configuration Build settings are configurable in `build.config` (not tracked by git). diff --git a/assets.build.ts b/assets.build.ts index f069765d..5783574c 100644 --- a/assets.build.ts +++ b/assets.build.ts @@ -39,11 +39,6 @@ const forumG6 = gen6Padded(); export default defineDeploy({ finish(ctx) { - // The upload contract wants __key first in the tar, naming the asset - // set; the tar packer emits ops in queue order, so it has to be the - // first op. - ctx.write("__key", "sprites"); - // Dex spritesheet assets: hash-stamped css + webp. The css suffix // pointer rides in __meta/ for the dex to read. { diff --git a/package.json b/package.json index 1e5bdb86..83cf5255 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "type": "module", "scripts": { "build": "node tools/deploy/index.ts build", - "deploy": "node tools/deploy/index.ts deploy assets.build.ts", + "deploy": "node tools/deploy/index.ts deploy", "check": "tsc --build tsconfig-workspace.json" }, "dependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8f96ce04..53ad154e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,6 +78,9 @@ importers: debug: specifier: ^4.1.1 version: 4.4.3 + json5: + specifier: ^2.2.3 + version: 2.2.3 tar-stream: specifier: ^3.0.0 version: 3.2.0 @@ -547,6 +550,11 @@ packages: json-stringify-safe@5.0.1: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + jsprim@1.4.2: resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} engines: {node: '>=0.6.0'} @@ -1161,6 +1169,8 @@ snapshots: json-stringify-safe@5.0.1: {} + json5@2.2.3: {} + jsprim@1.4.2: dependencies: assert-plus: 1.0.0 diff --git a/tools/deploy/config.ts b/tools/deploy/config.ts new file mode 100644 index 00000000..24452bdc --- /dev/null +++ b/tools/deploy/config.ts @@ -0,0 +1,88 @@ + +import fs from 'fs'; +import nodePath from 'path'; + +import JSON5 from 'json5'; + +import {BuildError} from '../build/errors.ts'; + +export interface DeployEntry { + subset : string[]; + cmd : string; +} + +export interface DeployTarget { + buildFile : string; + deploy : DeployEntry[]; +} + +export type DeployConfig = Map; + +function isStringArray(v : unknown) : v is string[] { + return Array.isArray(v) && v.every(x => typeof x === 'string'); +} + +export function loadDeployConfig(path : string) : DeployConfig { + let text : string; + try { + text = fs.readFileSync(path, 'utf8'); + } catch { + throw new BuildError(`missing ${path}; see README ("Deploying") for the schema`); + } + let raw : unknown; + try { + raw = JSON5.parse(text); + } catch (err) { + throw new BuildError(`${path}: ${(err as Error).message}`); + } + if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) { + throw new BuildError(`${path}: top level must be an object of deploy names`); + } + + const config : DeployConfig = new Map(); + for (const [name, t] of Object.entries(raw)) { + const target = t as Partial; + if (typeof t !== 'object' || t === null || typeof target.buildFile !== 'string' + || !Array.isArray(target.deploy)) { + throw new BuildError(`${path}: ${name}: expected {buildFile: string, deploy: [...]}`); + } + for (const e of target.deploy as Partial[]) { + if (typeof e !== 'object' || e === null + || !isStringArray(e.subset) || typeof e.cmd !== 'string') { + throw new BuildError( + `${path}: ${name}: each deploy entry needs {subset: string[], cmd: string}`); + } + } + config.set(name, target as DeployTarget); + } + return config; +} + +// Route finish outputs to deploy entries: per entry, the set of dsts its +// subset globs match. Every glob must match something and every dst must be +// covered by some entry; to unship an output, don't emit it in finish. +export function matchSubsets(dsts : readonly string[], + entries : readonly DeployEntry[]) : Set[] { + const covered = new Set(); + const perEntry = entries.map(entry => { + const matched = new Set(); + for (const glob of entry.subset) { + const hits = dsts.filter(d => nodePath.matchesGlob(d, glob)); + if (hits.length === 0) { + throw new BuildError(`subset glob matches no outputs: ${glob}`); + } + for (const hit of hits) { + matched.add(hit); + covered.add(hit); + } + } + return matched; + }); + const uncovered = dsts.filter(d => !covered.has(d)); + if (uncovered.length > 0) { + const shown = uncovered.slice(0, 10).join('\n '); + const more = uncovered.length > 10 ? `\n ... and ${uncovered.length - 10} more` : ''; + throw new BuildError(`outputs not covered by any deploy entry:\n ${shown}${more}`); + } + return perEntry; +} diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index ae205f12..7188525e 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -16,6 +16,7 @@ import {killAllProcessGroups} from '../build/exec.ts'; import {setConfig} from '../build/helpers.ts'; import {Store, acquireLock, dbVersion} from '../build/store.ts'; import {type DeploySpec, makeCtx} from './api.ts'; +import {loadDeployConfig, matchSubsets} from './config.ts'; import {ActionQueue} from './queue.ts'; const root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..'); @@ -173,35 +174,42 @@ common(program.command('build [files...]')) process.exitCode = await buildThen(getDecls(), opts, gc); }); -common(program.command('deploy ')) - .description('build, finish, and pipe the tar to DEPLOY_COMMAND from .env') - .action(async (file : string, opts : CommonOpts) => { - try { - process.loadEnvFile('.env'); - } catch { - console.error(`missing .env; set DEPLOY_COMMAND="ssh smogon smogonctl assets upload"`); - process.exitCode = 1; - return; - } - const command = process.env.DEPLOY_COMMAND; - if (command === undefined) { - console.error(`DEPLOY_COMMAND not set in .env`); - process.exitCode = 1; - return; +common(program.command('deploy [names...]')) + .description('build, finish, and pipe each subset tar to its command from deploy.json5') + .action(async (names : string[], opts : CommonOpts) => { + const config = loadDeployConfig('deploy.json5'); + const targets = names.length > 0 ? names : [...config.keys()]; + for (const name of targets) { + if (!config.has(name)) { + throw new BuildError(`deploy.json5: no deploy named ${name}`); + } } setConfig(loadConfig(opts.config)); - const specs = await importDeploys([file]); + const files = [...new Set(targets.map(n => config.get(n)!.buildFile))]; + const specs = await importDeploys(files); process.exitCode = await buildThen(getDecls(), opts, false, async () => { - const aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose)); - if (aq === null) { - return 1; + for (const name of targets) { + const target = config.get(name)!; + const aq = await runFinish(finishOf(specs, target.buildFile), Boolean(opts.verbose)); + if (aq === null) { + return 1; + } + const dsts = aq.log.filter(e => e.type === 'Op').map(e => e.dst); + const subsets = matchSubsets(dsts, target.deploy); + for (const [i, entry] of target.deploy.entries()) { + const matched = subsets[i]!; + console.log(`${name}: ${matched.size} files | ${entry.cmd}`); + const upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'inherit', 'inherit']}); + // If the command dies early we report its exit code; don't + // also crash on the resulting EPIPE. + upload.stdin!.on('error', () => {}); + aq.pack(dst => matched.has(dst)).pipe(upload.stdin!); + if (await waitExit(upload) !== 0) { + return 1; + } + } } - const upload = spawn(command, {shell: true, stdio: ['pipe', 'inherit', 'inherit']}); - // If the command dies early we report its exit code; don't also - // crash on the resulting EPIPE. - upload.stdin!.on('error', () => {}); - aq.pack().pipe(upload.stdin!); - return await waitExit(upload) !== 0 ? 1 : 0; + return 0; }); }); diff --git a/tools/deploy/package.json b/tools/deploy/package.json index d594f791..695f5f26 100644 --- a/tools/deploy/package.json +++ b/tools/deploy/package.json @@ -6,6 +6,7 @@ "base32-encode": "^2.0.0", "commander": "^5.1.0", "debug": "^4.1.1", + "json5": "^2.2.3", "tar-stream": "^3.0.0" }, "scripts": { diff --git a/tools/deploy/queue.ts b/tools/deploy/queue.ts index ff244fea..bba14691 100644 --- a/tools/deploy/queue.ts +++ b/tools/deploy/queue.ts @@ -160,12 +160,12 @@ export class ActionQueue { } } - pack() : NodeJS.ReadableStream { + pack(filter? : (dst : string) => boolean) : NodeJS.ReadableStream { if (!this.valid) throw new Error(`Invalid ActionQueue`); let t = tar.pack(); for (const entry of this.log) { - if (entry.type !== 'Op') + if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) continue; const op = entry.op; if (op.type === 'Copy'){ diff --git a/tools/deploy/test/api.test.ts b/tools/deploy/test/api.test.ts index 7360e2b8..4fe8b5c5 100644 --- a/tools/deploy/test/api.test.ts +++ b/tools/deploy/test/api.test.ts @@ -88,7 +88,8 @@ test('ctx.list sorts, parses extensions, skips dotfiles and directories', () => ]); }); -function packedEntries(aq : ActionQueue) : Promise<{name : string, data : string}[]> { +function packedEntries(aq : ActionQueue, filter? : (dst : string) => boolean) + : Promise<{name : string, data : string}[]> { return new Promise((resolve, reject) => { const extract = tar.extract(); const entries : {name : string, data : string}[] = []; @@ -102,22 +103,32 @@ function packedEntries(aq : ActionQueue) : Promise<{name : string, data : string }); extract.on('finish', () => resolve(entries)); extract.on('error', reject); - aq.pack().pipe(extract); + aq.pack(filter).pipe(extract); }); } -test('pack preserves op order with __key first', async () => { +test('pack preserves op order', async () => { const aq = new ActionQueue(); - aq.write('sprites', '__key'); aq.write('zzz', 'z.txt'); aq.write('aaa', 'a.txt'); assert.deepEqual(await packedEntries(aq), [ - {name: '__key', data: 'sprites'}, {name: 'z.txt', data: 'zzz'}, {name: 'a.txt', data: 'aaa'}, ]); }); +test('pack with a filter packs only matching entries in order', async () => { + const aq = new ActionQueue(); + aq.write('1', 'xy/a.png'); + aq.write('2', 'meta/m.json'); + aq.write('3', 'xy/b.png'); + const subset = await packedEntries(aq, dst => dst.startsWith('xy/')); + assert.deepEqual(subset, [ + {name: 'xy/a.png', data: '1'}, + {name: 'xy/b.png', data: '3'}, + ]); +}); + test('duplicate and absolute destinations invalidate the queue', () => { const dup = new ActionQueue(); dup.write('a', 'x.txt'); diff --git a/tools/deploy/test/config.test.ts b/tools/deploy/test/config.test.ts new file mode 100644 index 00000000..4a42b068 --- /dev/null +++ b/tools/deploy/test/config.test.ts @@ -0,0 +1,62 @@ + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import pathlib from 'node:path'; +import {test} from 'node:test'; + +import {loadDeployConfig, matchSubsets} from '../config.ts'; + +function configFile(text : string) : string { + const dir = fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-config-test-')); + const p = pathlib.join(dir, 'deploy.json5'); + fs.writeFileSync(p, text); + return p; +} + +test('loadDeployConfig parses json5 with comments and trailing commas', () => { + const config = loadDeployConfig(configFile(`{ + // dex assets + assets: { + buildFile: "assets.build.ts", + deploy: [ + {subset: ["**"], cmd: "cat > /dev/null"}, + ], + }, + }`)); + assert.deepEqual([...config.keys()], ['assets']); + assert.deepEqual(config.get('assets'), { + buildFile: 'assets.build.ts', + deploy: [{subset: ['**'], cmd: 'cat > /dev/null'}], + }); +}); + +test('loadDeployConfig rejects missing files and malformed shapes', () => { + assert.throws(() => loadDeployConfig('/nonexistent/deploy.json5'), /missing/); + assert.throws(() => loadDeployConfig(configFile('[1]')), /must be an object/); + assert.throws(() => loadDeployConfig(configFile('{a: {deploy: []}}')), /expected \{buildFile/); + assert.throws(() => loadDeployConfig(configFile( + '{a: {buildFile: "x.build.ts", deploy: [{subset: "**", cmd: "c"}]}}')), + /needs \{subset/); + assert.throws(() => loadDeployConfig(configFile('{a: {buildFile:')), /deploy.json5:/); +}); + +test('matchSubsets routes dsts to entries, overlap allowed', () => { + const dsts = ['xy/a.gif', 'xy/manifest.json', 'xyicons/b.png']; + const [xy, manifests] = matchSubsets(dsts, [ + {subset: ['xy/**', 'xyicons/**'], cmd: 'one'}, + {subset: ['**/manifest.json'], cmd: 'two'}, + ]); + assert.deepEqual([...xy!].sort(), dsts); + assert.deepEqual([...manifests!], ['xy/manifest.json']); +}); + +test('matchSubsets errors on a glob matching nothing', () => { + assert.throws(() => matchSubsets(['xy/a.gif'], [{subset: ['xy/**', 'zz/**'], cmd: 'c'}]), + /matches no outputs: zz\/\*\*/); +}); + +test('matchSubsets errors on uncovered outputs, listing them', () => { + assert.throws(() => matchSubsets(['xy/a.gif', 'stray.txt'], [{subset: ['xy/**'], cmd: 'c'}]), + /not covered by any deploy entry:\n {2}stray.txt/); +});