From 561b7ba0fd2febf88377e69dadd5638259486b58 Mon Sep 17 00:00:00 2001 From: Christopher Monsanto Date: Mon, 17 Aug 2026 16:16:42 -0400 Subject: [PATCH] Add a dir mode to deploy entries An entry with dir: true materializes its subset into a temp directory whose path replaces %d in the cmd, instead of piping a tar on stdin -- rsync-style transports (the PS ani sync) need real files. The flag and %d must agree, so a forgotten one is caught at config load. Co-Authored-By: Claude Fable 5 --- README.md | 14 +++++++++++--- tools/deploy/config.ts | 13 +++++++++++-- tools/deploy/index.ts | 15 +++++++++++++++ tools/deploy/queue.ts | 6 +++--- tools/deploy/test/api.test.ts | 11 +++++++++++ tools/deploy/test/config.test.ts | 12 ++++++++++++ 6 files changed, 63 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1348bae5..94ea30f0 100644 --- a/README.md +++ b/README.md @@ -83,15 +83,23 @@ Useful flags: `-j ` parallelism, `-n` dry run, `-v` verbose, deploy names to a buildFile and a list of (subset, cmd) entries: after building and finishing the buildFile, each entry's globs select a subset of the finish outputs, which are tarred and piped to the entry's command on -stdin. Every glob must match something, and every output must be covered by -some entry. +stdin. An entry with `dir: true` instead materializes the subset into a temp +directory whose path replaces `%d` in the command (for rsync-style +transports). Every glob must match something, and every output must be +covered by some entry. ```json5 { assets: { buildFile: "assets.build.ts", deploy: [ - {subset: ["**"], cmd: "ssh smogon smogonctl assets upload sprites"}, + {subset: ["**"], cmd: "smogonctl assets upload sprites1"}, + ], + }, + ps: { + buildFile: "ps.build.ts", + deploy: [ + {subset: ["ani/**"], dir: true, cmd: "rsync -a %d/ani/ ps:sprites/ani/"}, ], }, } diff --git a/tools/deploy/config.ts b/tools/deploy/config.ts index 24452bdc..ace6347a 100644 --- a/tools/deploy/config.ts +++ b/tools/deploy/config.ts @@ -9,6 +9,9 @@ import {BuildError} from '../build/errors.ts'; export interface DeployEntry { subset : string[]; cmd : string; + // dir entries get their subset materialized into a temp directory whose + // path replaces %d in cmd; tar entries get the subset tarred on stdin. + dir? : boolean; } export interface DeployTarget { @@ -48,9 +51,15 @@ export function loadDeployConfig(path : string) : DeployConfig { } for (const e of target.deploy as Partial[]) { if (typeof e !== 'object' || e === null - || !isStringArray(e.subset) || typeof e.cmd !== 'string') { + || !isStringArray(e.subset) || typeof e.cmd !== 'string' + || (e.dir !== undefined && typeof e.dir !== 'boolean')) { throw new BuildError( - `${path}: ${name}: each deploy entry needs {subset: string[], cmd: string}`); + `${path}: ${name}: each deploy entry needs {subset: string[], cmd: string, dir?: boolean}`); + } + if (Boolean(e.dir) !== e.cmd.includes('%d')) { + throw new BuildError(`${path}: ${name}: ${e.dir + ? 'a dir entry\'s cmd must use %d' + : 'a tar entry\'s cmd must not use %d (missing dir: true?)'}: ${e.cmd}`); } } config.set(name, target as DeployTarget); diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index ab02a72e..6788831b 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -208,6 +208,21 @@ common(program.command('deploy [names...]')) for (const [i, entry] of target.deploy.entries()) { const matched = subsets[i]!; console.log(`${name}: ${matched.size} files | ${entry.cmd}`); + if (entry.dir) { + fs.mkdirSync(TMP_DIR, {recursive: true}); + const tmp = fs.mkdtempSync(nodePath.join(TMP_DIR, 'deploy-')); + try { + await aq.run(tmp, 'copy', dst => matched.has(dst)); + const cmd = spawn(entry.cmd.replaceAll('%d', tmp), + {shell: true, stdio: ['ignore', 'inherit', 'inherit']}); + if (await waitExit(cmd) !== 0) { + return 1; + } + } finally { + fs.rmSync(tmp, {recursive: true, force: true}); + } + continue; + } const upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'inherit', 'inherit']}); // If the command dies early we report its exit code; don't // also crash on the resulting EPIPE, which reaches both diff --git a/tools/deploy/queue.ts b/tools/deploy/queue.ts index 5fb8734d..7aa511c4 100644 --- a/tools/deploy/queue.ts +++ b/tools/deploy/queue.ts @@ -125,12 +125,12 @@ export class ActionQueue { } } - async run(dir : string, mode : 'link' | 'copy' | 'tar') { + async run(dir : string, mode : 'link' | 'copy' | 'tar', filter? : (dst : string) => boolean) { if (!this.valid) throw new Error(`Invalid ActionQueue`); if (mode !== 'tar') { for (const entry of this.log) { - if (entry.type !== 'Op') + if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) continue; const op = entry.op; const dst = nodePath.join(dir, entry.dst); @@ -152,7 +152,7 @@ export class ActionQueue { } else { // In this case, I guess its a file rather than a dir. const out = fs.createWriteStream(dir); - this.pack().pipe(out); + this.pack(filter).pipe(out); return new Promise((resolve, reject) => { out.on('error', reject); out.on('finish', () => resolve()); diff --git a/tools/deploy/test/api.test.ts b/tools/deploy/test/api.test.ts index 74821456..174ec0e6 100644 --- a/tools/deploy/test/api.test.ts +++ b/tools/deploy/test/api.test.ts @@ -141,6 +141,17 @@ test('duplicate and absolute destinations invalidate the queue', () => { assert.ok(!abs.valid); }); +test('run with a filter materializes only matching entries', async () => { + const dir = tmpdir(); + const aq = new ActionQueue(); + aq.write('1', 'ani/a.gif'); + aq.write('2', 'dex/b.png'); + const out = pathlib.join(dir, 'deploy'); + await aq.run(out, 'copy', dst => dst.startsWith('ani/')); + assert.equal(fs.readFileSync(pathlib.join(out, 'ani/a.gif'), 'utf8'), '1'); + assert.ok(!fs.existsSync(pathlib.join(out, 'dex'))); +}); + test('copy-mode materialization restores 0644 on read-only sources', async () => { const dir = tmpdir(); const src = pathlib.join(dir, 'obj'); diff --git a/tools/deploy/test/config.test.ts b/tools/deploy/test/config.test.ts index 4a42b068..6b0de96e 100644 --- a/tools/deploy/test/config.test.ts +++ b/tools/deploy/test/config.test.ts @@ -31,6 +31,18 @@ test('loadDeployConfig parses json5 with comments and trailing commas', () => { }); }); +test('loadDeployConfig ties the dir flag to %d in the cmd', () => { + const dir = loadDeployConfig(configFile( + '{ps: {buildFile: "ps.build.ts", deploy: [{subset: ["ani/**"], dir: true, cmd: "rsync -a %d/ani/ h:a/"}]}}')); + assert.equal(dir.get('ps')!.deploy[0]!.dir, true); + assert.throws(() => loadDeployConfig(configFile( + '{a: {buildFile: "x.build.ts", deploy: [{subset: ["**"], dir: true, cmd: "rsync -a h:a/"}]}}')), + /dir entry's cmd must use %d/); + assert.throws(() => loadDeployConfig(configFile( + '{a: {buildFile: "x.build.ts", deploy: [{subset: ["**"], cmd: "rsync -a %d/ h:a/"}]}}')), + /must not use %d \(missing dir: true\?\)/); +}); + test('loadDeployConfig rejects missing files and malformed shapes', () => { assert.throws(() => loadDeployConfig('/nonexistent/deploy.json5'), /missing/); assert.throws(() => loadDeployConfig(configFile('[1]')), /must be an object/);