Add a dir mode to deploy entries

An entry with dir: true materializes its subset into a temp
directory whose path replaces %d in the cmd, instead of piping a
tar on stdin -- rsync-style transports (the PS ani sync) need real
files. The flag and %d must agree, so a forgotten one is caught at
config load.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

README.md

@@ -83,15 +83,23 @@ Useful flags: `-j <n>` parallelism, `-n` dry run, `-v` verbose,
 deploy names to a buildFile and a list of (subset, cmd) entries: after
 building and finishing the buildFile, each entry's globs select a subset of
 the finish outputs, which are tarred and piped to the entry's command on
-stdin. Every glob must match something, and every output must be covered by
-some entry.
+stdin. An entry with `dir: true` instead materializes the subset into a temp
+directory whose path replaces `%d` in the command (for rsync-style
+transports). Every glob must match something, and every output must be
+covered by some entry.
 
 ```json5
 {
     assets: {
         buildFile: "assets.build.ts",
         deploy: [
-            {subset: ["**"], cmd: "ssh smogon smogonctl assets upload sprites"},
+            {subset: ["**"], cmd: "smogonctl assets upload sprites1"},
+        ],
+    },
+    ps: {
+        buildFile: "ps.build.ts",
+        deploy: [
+            {subset: ["ani/**"], dir: true, cmd: "rsync -a %d/ani/ ps:sprites/ani/"},
         ],
     },
 }

tools/deploy/config.ts

@@ -9,6 +9,9 @@ import {BuildError} from '../build/errors.ts';
 export interface DeployEntry {
     subset : string[];
     cmd : string;
+    // dir entries get their subset materialized into a temp directory whose
+    // path replaces %d in cmd; tar entries get the subset tarred on stdin.
+    dir? : boolean;
 }
 
 export interface DeployTarget {
@@ -48,9 +51,15 @@ export function loadDeployConfig(path : string) : DeployConfig {
         }
         for (const e of target.deploy as Partial<DeployEntry>[]) {
             if (typeof e !== 'object' || e === null
-                || !isStringArray(e.subset) || typeof e.cmd !== 'string') {
+                || !isStringArray(e.subset) || typeof e.cmd !== 'string'
+                || (e.dir !== undefined && typeof e.dir !== 'boolean')) {
                 throw new BuildError(
-                    `${path}: ${name}: each deploy entry needs {subset: string[], cmd: string}`);
+                    `${path}: ${name}: each deploy entry needs {subset: string[], cmd: string, dir?: boolean}`);
+            }
+            if (Boolean(e.dir) !== e.cmd.includes('%d')) {
+                throw new BuildError(`${path}: ${name}: ${e.dir
+                    ? 'a dir entry\'s cmd must use %d'
+                    : 'a tar entry\'s cmd must not use %d (missing dir: true?)'}: ${e.cmd}`);
             }
         }
         config.set(name, target as DeployTarget);

tools/deploy/index.ts

@@ -208,6 +208,21 @@ common(program.command('deploy [names...]'))
                 for (const [i, entry] of target.deploy.entries()) {
                     const matched = subsets[i]!;
                     console.log(`${name}: ${matched.size} files | ${entry.cmd}`);
+                    if (entry.dir) {
+                        fs.mkdirSync(TMP_DIR, {recursive: true});
+                        const tmp = fs.mkdtempSync(nodePath.join(TMP_DIR, 'deploy-'));
+                        try {
+                            await aq.run(tmp, 'copy', dst => matched.has(dst));
+                            const cmd = spawn(entry.cmd.replaceAll('%d', tmp),
+                                {shell: true, stdio: ['ignore', 'inherit', 'inherit']});
+                            if (await waitExit(cmd) !== 0) {
+                                return 1;
+                            }
+                        } finally {
+                            fs.rmSync(tmp, {recursive: true, force: true});
+                        }
+                        continue;
+                    }
                     const upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'inherit', 'inherit']});
                     // If the command dies early we report its exit code; don't
                     // also crash on the resulting EPIPE, which reaches both

tools/deploy/queue.ts

@@ -125,12 +125,12 @@ export class ActionQueue {
         }
     }
 
-    async run(dir : string, mode : 'link' | 'copy' | 'tar') {
+    async run(dir : string, mode : 'link' | 'copy' | 'tar', filter? : (dst : string) => boolean) {
         if (!this.valid)
             throw new Error(`Invalid ActionQueue`);
         if (mode !== 'tar') {
             for (const entry of this.log) {
-                if (entry.type !== 'Op')
+                if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst)))
                     continue;
                 const op = entry.op;
                 const dst = nodePath.join(dir, entry.dst);
@@ -152,7 +152,7 @@ export class ActionQueue {
         } else {
             // In this case, I guess its a file rather than a dir.
             const out = fs.createWriteStream(dir);
-            this.pack().pipe(out);
+            this.pack(filter).pipe(out);
             return new Promise<void>((resolve, reject) => {
                 out.on('error', reject);
                 out.on('finish', () => resolve());

tools/deploy/test/api.test.ts

@@ -141,6 +141,17 @@ test('duplicate and absolute destinations invalidate the queue', () => {
     assert.ok(!abs.valid);
 });
 
+test('run with a filter materializes only matching entries', async () => {
+    const dir = tmpdir();
+    const aq = new ActionQueue();
+    aq.write('1', 'ani/a.gif');
+    aq.write('2', 'dex/b.png');
+    const out = pathlib.join(dir, 'deploy');
+    await aq.run(out, 'copy', dst => dst.startsWith('ani/'));
+    assert.equal(fs.readFileSync(pathlib.join(out, 'ani/a.gif'), 'utf8'), '1');
+    assert.ok(!fs.existsSync(pathlib.join(out, 'dex')));
+});
+
 test('copy-mode materialization restores 0644 on read-only sources', async () => {
     const dir = tmpdir();
     const src = pathlib.join(dir, 'obj');

tools/deploy/test/config.test.ts

@@ -31,6 +31,18 @@ test('loadDeployConfig parses json5 with comments and trailing commas', () => {
     });
 });
 
+test('loadDeployConfig ties the dir flag to %d in the cmd', () => {
+    const dir = loadDeployConfig(configFile(
+        '{ps: {buildFile: "ps.build.ts", deploy: [{subset: ["ani/**"], dir: true, cmd: "rsync -a %d/ani/ h:a/"}]}}'));
+    assert.equal(dir.get('ps')!.deploy[0]!.dir, true);
+    assert.throws(() => loadDeployConfig(configFile(
+        '{a: {buildFile: "x.build.ts", deploy: [{subset: ["**"], dir: true, cmd: "rsync -a h:a/"}]}}')),
+    /dir entry's cmd must use %d/);
+    assert.throws(() => loadDeployConfig(configFile(
+        '{a: {buildFile: "x.build.ts", deploy: [{subset: ["**"], cmd: "rsync -a %d/ h:a/"}]}}')),
+    /must not use %d \(missing dir: true\?\)/);
+});
+
 test('loadDeployConfig rejects missing files and malformed shapes', () => {
     assert.throws(() => loadDeployConfig('/nonexistent/deploy.json5'), /missing/);
     assert.throws(() => loadDeployConfig(configFile('[1]')), /must be an object/);