Replace the .env deploy command with deploy.json5

An untracked json5 config maps deploy names to a buildFile plus
(subset globs, cmd) entries: the deploy verb builds the named
buildFiles, finishes each, routes the outputs through the subset
globs, and pipes one tar per entry to its command. Every glob must
match an output and every output must be covered by some entry.

The tar no longer carries __key; the upload command names the asset
set itself (smogonctl assets upload sprites).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.gitignore

@@ -4,4 +4,4 @@ pnpm-debug.log
 .cache/
 .build/
 build.config
-.env
+deploy.json5

README.md

@@ -62,8 +62,9 @@ reuses the build's digests. All state lives in `.build/`.
 
 ```
 $ pnpm build                                     # build every deploy's rules, GC stale state
-$ pnpm deploy                                    # assets.build.ts -> tar -> DEPLOY_COMMAND (.env)
-$ node tools/deploy/index.ts build ps.build.ts  # build one deploy's rules
+$ pnpm deploy                                    # every deploy in deploy.json5
+$ node tools/deploy/index.ts deploy assets       # one named deploy
+$ node tools/deploy/index.ts build ps.build.ts   # build one deploy's rules
 $ node tools/deploy/index.ts run smogon.build.ts -o deploy/smogon
 $ node tools/deploy/index.ts inspect src/minisprites/items/i1.png -o /tmp/out
 ```
@@ -76,6 +77,26 @@ copies the outputs out under readable names for eyeballing.
 Useful flags: `-j <n>` parallelism, `-n` dry run, `-v` verbose,
 `--fail-fast` stop after the first failure.
 
+## Deploying
+
+`deploy` reads `deploy.json5` at the repo root (not tracked by git). It maps
+deploy names to a buildFile and a list of (subset, cmd) entries: after
+building and finishing the buildFile, each entry's globs select a subset of
+the finish outputs, which are tarred and piped to the entry's command on
+stdin. Every glob must match something, and every output must be covered by
+some entry.
+
+```json5
+{
+    assets: {
+        buildFile: "assets.build.ts",
+        deploy: [
+            {subset: ["**"], cmd: "ssh smogon smogonctl assets upload sprites"},
+        ],
+    },
+}
+```
+
 ## Configuration
 
 Build settings are configurable in `build.config` (not tracked by git).

assets.build.ts

@@ -39,11 +39,6 @@ const forumG6 = gen6Padded();
 
 export default defineDeploy({
     finish(ctx) {
-        // The upload contract wants __key first in the tar, naming the asset
-        // set; the tar packer emits ops in queue order, so it has to be the
-        // first op.
-        ctx.write("__key", "sprites");
-
         // Dex spritesheet assets: hash-stamped css + webp. The css suffix
         // pointer rides in __meta/ for the dex to read.
         {

package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "scripts": {
         "build": "node tools/deploy/index.ts build",
-        "deploy": "node tools/deploy/index.ts deploy assets.build.ts",
+        "deploy": "node tools/deploy/index.ts deploy",
         "check": "tsc --build tsconfig-workspace.json"
     },
     "dependencies": {

pnpm-lock.yaml

@@ -78,6 +78,9 @@ importers:
       debug:
         specifier: ^4.1.1
         version: 4.4.3
+      json5:
+        specifier: ^2.2.3
+        version: 2.2.3
       tar-stream:
         specifier: ^3.0.0
         version: 3.2.0
@@ -547,6 +550,11 @@ packages:
   json-stringify-safe@5.0.1:
     resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
 
+  json5@2.2.3:
+    resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
+    engines: {node: '>=6'}
+    hasBin: true
+
   jsprim@1.4.2:
     resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==}
     engines: {node: '>=0.6.0'}
@@ -1161,6 +1169,8 @@ snapshots:
 
   json-stringify-safe@5.0.1: {}
 
+  json5@2.2.3: {}
+
   jsprim@1.4.2:
     dependencies:
       assert-plus: 1.0.0

tools/deploy/config.ts

@@ -0,0 +1,88 @@
+
+import fs from 'fs';
+import nodePath from 'path';
+
+import JSON5 from 'json5';
+
+import {BuildError} from '../build/errors.ts';
+
+export interface DeployEntry {
+    subset : string[];
+    cmd : string;
+}
+
+export interface DeployTarget {
+    buildFile : string;
+    deploy : DeployEntry[];
+}
+
+export type DeployConfig = Map<string, DeployTarget>;
+
+function isStringArray(v : unknown) : v is string[] {
+    return Array.isArray(v) && v.every(x => typeof x === 'string');
+}
+
+export function loadDeployConfig(path : string) : DeployConfig {
+    let text : string;
+    try {
+        text = fs.readFileSync(path, 'utf8');
+    } catch {
+        throw new BuildError(`missing ${path}; see README ("Deploying") for the schema`);
+    }
+    let raw : unknown;
+    try {
+        raw = JSON5.parse(text);
+    } catch (err) {
+        throw new BuildError(`${path}: ${(err as Error).message}`);
+    }
+    if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
+        throw new BuildError(`${path}: top level must be an object of deploy names`);
+    }
+
+    const config : DeployConfig = new Map();
+    for (const [name, t] of Object.entries(raw)) {
+        const target = t as Partial<DeployTarget>;
+        if (typeof t !== 'object' || t === null || typeof target.buildFile !== 'string'
+            || !Array.isArray(target.deploy)) {
+            throw new BuildError(`${path}: ${name}: expected {buildFile: string, deploy: [...]}`);
+        }
+        for (const e of target.deploy as Partial<DeployEntry>[]) {
+            if (typeof e !== 'object' || e === null
+                || !isStringArray(e.subset) || typeof e.cmd !== 'string') {
+                throw new BuildError(
+                    `${path}: ${name}: each deploy entry needs {subset: string[], cmd: string}`);
+            }
+        }
+        config.set(name, target as DeployTarget);
+    }
+    return config;
+}
+
+// Route finish outputs to deploy entries: per entry, the set of dsts its
+// subset globs match. Every glob must match something and every dst must be
+// covered by some entry; to unship an output, don't emit it in finish.
+export function matchSubsets(dsts : readonly string[],
+                             entries : readonly DeployEntry[]) : Set<string>[] {
+    const covered = new Set<string>();
+    const perEntry = entries.map(entry => {
+        const matched = new Set<string>();
+        for (const glob of entry.subset) {
+            const hits = dsts.filter(d => nodePath.matchesGlob(d, glob));
+            if (hits.length === 0) {
+                throw new BuildError(`subset glob matches no outputs: ${glob}`);
+            }
+            for (const hit of hits) {
+                matched.add(hit);
+                covered.add(hit);
+            }
+        }
+        return matched;
+    });
+    const uncovered = dsts.filter(d => !covered.has(d));
+    if (uncovered.length > 0) {
+        const shown = uncovered.slice(0, 10).join('\n  ');
+        const more = uncovered.length > 10 ? `\n  ... and ${uncovered.length - 10} more` : '';
+        throw new BuildError(`outputs not covered by any deploy entry:\n  ${shown}${more}`);
+    }
+    return perEntry;
+}

tools/deploy/index.ts

@@ -16,6 +16,7 @@ import {killAllProcessGroups} from '../build/exec.ts';
 import {setConfig} from '../build/helpers.ts';
 import {Store, acquireLock, dbVersion} from '../build/store.ts';
 import {type DeploySpec, makeCtx} from './api.ts';
+import {loadDeployConfig, matchSubsets} from './config.ts';
 import {ActionQueue} from './queue.ts';
 
 const root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..');
@@ -173,35 +174,42 @@ common(program.command('build [files...]'))
         process.exitCode = await buildThen(getDecls(), opts, gc);
     });
 
-common(program.command('deploy <file>'))
-    .description('build, finish, and pipe the tar to DEPLOY_COMMAND from .env')
-    .action(async (file : string, opts : CommonOpts) => {
-        try {
-            process.loadEnvFile('.env');
-        } catch {
-            console.error(`missing .env; set DEPLOY_COMMAND="ssh smogon smogonctl assets upload"`);
-            process.exitCode = 1;
-            return;
-        }
-        const command = process.env.DEPLOY_COMMAND;
-        if (command === undefined) {
-            console.error(`DEPLOY_COMMAND not set in .env`);
-            process.exitCode = 1;
-            return;
+common(program.command('deploy [names...]'))
+    .description('build, finish, and pipe each subset tar to its command from deploy.json5')
+    .action(async (names : string[], opts : CommonOpts) => {
+        const config = loadDeployConfig('deploy.json5');
+        const targets = names.length > 0 ? names : [...config.keys()];
+        for (const name of targets) {
+            if (!config.has(name)) {
+                throw new BuildError(`deploy.json5: no deploy named ${name}`);
+            }
         }
         setConfig(loadConfig(opts.config));
-        const specs = await importDeploys([file]);
+        const files = [...new Set(targets.map(n => config.get(n)!.buildFile))];
+        const specs = await importDeploys(files);
         process.exitCode = await buildThen(getDecls(), opts, false, async () => {
-            const aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose));
-            if (aq === null) {
-                return 1;
+            for (const name of targets) {
+                const target = config.get(name)!;
+                const aq = await runFinish(finishOf(specs, target.buildFile), Boolean(opts.verbose));
+                if (aq === null) {
+                    return 1;
+                }
+                const dsts = aq.log.filter(e => e.type === 'Op').map(e => e.dst);
+                const subsets = matchSubsets(dsts, target.deploy);
+                for (const [i, entry] of target.deploy.entries()) {
+                    const matched = subsets[i]!;
+                    console.log(`${name}: ${matched.size} files | ${entry.cmd}`);
+                    const upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'inherit', 'inherit']});
+                    // If the command dies early we report its exit code; don't
+                    // also crash on the resulting EPIPE.
+                    upload.stdin!.on('error', () => {});
+                    aq.pack(dst => matched.has(dst)).pipe(upload.stdin!);
+                    if (await waitExit(upload) !== 0) {
+                        return 1;
+                    }
+                }
             }
-            const upload = spawn(command, {shell: true, stdio: ['pipe', 'inherit', 'inherit']});
-            // If the command dies early we report its exit code; don't also
-            // crash on the resulting EPIPE.
-            upload.stdin!.on('error', () => {});
-            aq.pack().pipe(upload.stdin!);
-            return await waitExit(upload) !== 0 ? 1 : 0;
+            return 0;
         });
     });
 

tools/deploy/package.json

@@ -6,6 +6,7 @@
         "base32-encode": "^2.0.0",
         "commander": "^5.1.0",
         "debug": "^4.1.1",
+        "json5": "^2.2.3",
         "tar-stream": "^3.0.0"
     },
     "scripts": {

tools/deploy/queue.ts

@@ -160,12 +160,12 @@ export class ActionQueue {
         }
     }
 
-    pack() : NodeJS.ReadableStream {
+    pack(filter? : (dst : string) => boolean) : NodeJS.ReadableStream {
         if (!this.valid)
             throw new Error(`Invalid ActionQueue`);
         let t = tar.pack();
         for (const entry of this.log) {
-            if (entry.type !== 'Op')
+            if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst)))
                 continue;
             const op = entry.op;
             if (op.type === 'Copy'){

tools/deploy/test/api.test.ts

@@ -88,7 +88,8 @@ test('ctx.list sorts, parses extensions, skips dotfiles and directories', () =>
     ]);
 });
 
-function packedEntries(aq : ActionQueue) : Promise<{name : string, data : string}[]> {
+function packedEntries(aq : ActionQueue, filter? : (dst : string) => boolean)
+    : Promise<{name : string, data : string}[]> {
     return new Promise((resolve, reject) => {
         const extract = tar.extract();
         const entries : {name : string, data : string}[] = [];
@@ -102,22 +103,32 @@ function packedEntries(aq : ActionQueue) : Promise<{name : string, data : string
         });
         extract.on('finish', () => resolve(entries));
         extract.on('error', reject);
-        aq.pack().pipe(extract);
+        aq.pack(filter).pipe(extract);
     });
 }
 
-test('pack preserves op order with __key first', async () => {
+test('pack preserves op order', async () => {
     const aq = new ActionQueue();
-    aq.write('sprites', '__key');
     aq.write('zzz', 'z.txt');
     aq.write('aaa', 'a.txt');
     assert.deepEqual(await packedEntries(aq), [
-        {name: '__key', data: 'sprites'},
         {name: 'z.txt', data: 'zzz'},
         {name: 'a.txt', data: 'aaa'},
     ]);
 });
 
+test('pack with a filter packs only matching entries in order', async () => {
+    const aq = new ActionQueue();
+    aq.write('1', 'xy/a.png');
+    aq.write('2', 'meta/m.json');
+    aq.write('3', 'xy/b.png');
+    const subset = await packedEntries(aq, dst => dst.startsWith('xy/'));
+    assert.deepEqual(subset, [
+        {name: 'xy/a.png', data: '1'},
+        {name: 'xy/b.png', data: '3'},
+    ]);
+});
+
 test('duplicate and absolute destinations invalidate the queue', () => {
     const dup = new ActionQueue();
     dup.write('a', 'x.txt');

tools/deploy/test/config.test.ts

@@ -0,0 +1,62 @@
+
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import pathlib from 'node:path';
+import {test} from 'node:test';
+
+import {loadDeployConfig, matchSubsets} from '../config.ts';
+
+function configFile(text : string) : string {
+    const dir = fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-config-test-'));
+    const p = pathlib.join(dir, 'deploy.json5');
+    fs.writeFileSync(p, text);
+    return p;
+}
+
+test('loadDeployConfig parses json5 with comments and trailing commas', () => {
+    const config = loadDeployConfig(configFile(`{
+        // dex assets
+        assets: {
+            buildFile: "assets.build.ts",
+            deploy: [
+                {subset: ["**"], cmd: "cat > /dev/null"},
+            ],
+        },
+    }`));
+    assert.deepEqual([...config.keys()], ['assets']);
+    assert.deepEqual(config.get('assets'), {
+        buildFile: 'assets.build.ts',
+        deploy: [{subset: ['**'], cmd: 'cat > /dev/null'}],
+    });
+});
+
+test('loadDeployConfig rejects missing files and malformed shapes', () => {
+    assert.throws(() => loadDeployConfig('/nonexistent/deploy.json5'), /missing/);
+    assert.throws(() => loadDeployConfig(configFile('[1]')), /must be an object/);
+    assert.throws(() => loadDeployConfig(configFile('{a: {deploy: []}}')), /expected \{buildFile/);
+    assert.throws(() => loadDeployConfig(configFile(
+        '{a: {buildFile: "x.build.ts", deploy: [{subset: "**", cmd: "c"}]}}')),
+    /needs \{subset/);
+    assert.throws(() => loadDeployConfig(configFile('{a: {buildFile:')), /deploy.json5:/);
+});
+
+test('matchSubsets routes dsts to entries, overlap allowed', () => {
+    const dsts = ['xy/a.gif', 'xy/manifest.json', 'xyicons/b.png'];
+    const [xy, manifests] = matchSubsets(dsts, [
+        {subset: ['xy/**', 'xyicons/**'], cmd: 'one'},
+        {subset: ['**/manifest.json'], cmd: 'two'},
+    ]);
+    assert.deepEqual([...xy!].sort(), dsts);
+    assert.deepEqual([...manifests!], ['xy/manifest.json']);
+});
+
+test('matchSubsets errors on a glob matching nothing', () => {
+    assert.throws(() => matchSubsets(['xy/a.gif'], [{subset: ['xy/**', 'zz/**'], cmd: 'c'}]),
+        /matches no outputs: zz\/\*\*/);
+});
+
+test('matchSubsets errors on uncovered outputs, listing them', () => {
+    assert.throws(() => matchSubsets(['xy/a.gif', 'stray.txt'], [{subset: ['xy/**'], cmd: 'c'}]),
+        /not covered by any deploy entry:\n {2}stray.txt/);
+});