diff --git a/README.md b/README.md index 395ec656..93eb4f67 100644 --- a/README.md +++ b/README.md @@ -93,19 +93,32 @@ Useful flags: `-j ` parallelism, `-n` dry run, `-v` verbose, `deploy` reads `deploy.json5` at the repo root (not tracked by git). It maps deploy names to a buildFile and a list of (subset, cmd) entries: after building and finishing the buildFile, each entry's globs select a subset of -the finish outputs, which are tarred and piped to the entry's command on -stdin. An entry with `dir: true` instead materializes the subset into a temp -directory whose path replaces `%d` in the command (for rsync-style -transports). Every glob must match something, and every output must be -covered by some entry. `deploy -o ` materializes each entry's -subset under `///` instead of running its command, -for eyeballing what would ship. +the finish outputs and sends it to the entry's command, which is `smogonctl +assets upload` (possibly behind `ssh` and `sudo`). An entry with `dir: true` +instead materializes the subset into a temp directory whose path replaces +`%d` in the command (for rsync-style transports). Every glob must match +something, and every output must be covered by some entry. `deploy -o +` materializes each entry's subset under `///` +instead of running its command, for eyeballing what would ship. -Where stderr is a terminal, each entry draws a progress bar over its files -while they go out -- an entry counts once the command has taken it, not once -it has been read off disk -- and the line is cleared again afterwards. Piped -or under CI nothing is drawn, and the lines the deploy prints are the same -either way. +Sending is a conversation over the command's stdin and stdout rather than a +tar poured into it. Every served name carries a content hash and the served +tree is add-only, so a name the home already has is a file that needn't +travel: the deploy first writes a manifest -- every name with its size, and +every `__meta/` link -- and the command answers with the ones it hasn't got. +Only those, plus the `__meta/` files, are then tarred into the same stream. A +redeploy of an unchanged tree moves kilobytes. The command's stdout has to be +the upload's alone and reach the deploy unfiltered: the reply is the first +thing on it, and anything else there (a shell profile that prints over ssh, a +`| grep`) is taken as a refusal and fails the deploy rather than being +skipped past. `run --tar` still writes a bare tar, which the upload also +accepts. + +Where stderr is a terminal, each entry draws a progress bar over the files +that travel while they go out -- an entry counts once the command has taken +it, not once it has been read off disk -- and the line is cleared again +afterwards. Piped or under CI nothing is drawn, and the lines the deploy +prints are the same either way. This file is not committed, because it is where the hosts and paths this repo ships to are written down. @@ -127,7 +140,7 @@ each of them has to cover all of it. ### The asset upload's tar layout -`smogonctl assets upload` publishes a tar into a served tree under a prefix +`smogonctl assets upload` publishes a tree into a served tree under a prefix named in the receiving home's `services.toml`, which this side can't read. So `smogon.build.ts` writes that prefix itself -- everything served ships under `sprites/` -- and the upload rejects a tar whose tree disagrees. The two are diff --git a/tools/deploy/config.ts b/tools/deploy/config.ts index 323dd38b..6d56980f 100644 --- a/tools/deploy/config.ts +++ b/tools/deploy/config.ts @@ -10,7 +10,9 @@ export type DeployEntry = { subset: string[], cmd: string, // dir entries get their subset materialized into a temp directory whose - // path replaces %d in cmd; tar entries get the subset tarred on stdin. + // path replaces %d in cmd. Every other entry's cmd is `smogonctl assets + // upload` (or a wrapper around it): it gets a manifest of the subset on + // stdin, answers on stdout with what it lacks, and gets a tar of that. dir?: boolean, }; diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index fe2bcf82..7f3fb9c8 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -19,6 +19,7 @@ import * as db from '../build/store.ts'; import * as api from './api.ts'; import {loadDeployConfig, matchSubsets} from './config.ts'; import {withProgress} from './progress.ts'; +import {manifestOf, negotiate} from './protocol.ts'; import {ActionQueue} from './queue.ts'; let root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..'); @@ -50,8 +51,9 @@ let USAGE = `usage: node tools/deploy/index.ts [options] commands: build [files...] build the rules of the given deploys (default: all *.build.ts) - deploy [names...] build, finish, and pipe each subset tar (or %d dir) to its - command from deploy.json5 (no names: list the deploys; + deploy [names...] build, finish, and send each subset to its command from + deploy.json5: a manifest and then a tar of what it asks + for on stdin, or a %d dir (no names: list the deploys; -o : materialize each subset there instead of running its command) run -o build, finish, and materialize to a directory (or tar file) @@ -287,7 +289,10 @@ async function cmdDeploy(names: string[], opts: VerbOpts): Promise { } continue; } - let upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'inherit', 'inherit']}); + // The command's stdout is ours to read: the reply to the + // manifest comes back on it, and its report after that is + // relayed. stderr stays the terminal's, for the bar. + let upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'pipe', 'inherit']}); let stdin = upload.stdin; if (stdin === null) { throw new BuildError(`no stdin pipe for: ${entry.cmd}`); @@ -296,14 +301,35 @@ async function cmdDeploy(names: string[], opts: VerbOpts): Promise { // also crash on the resulting EPIPE, which reaches both // stdin and (via streamx's destroy propagation) the pack. stdin.on('error', () => {}); - // The bar tracks the upload itself: an entry counts once the - // command has taken it, not once it has been read off disk. - let code = await withProgress(`${name}: uploading`, matched.size, async tick => { - let pack = await aq.pack(dst => matched.has(dst), tick); - pack.on('error', () => {}); - pack.pipe(stdin); - return await waitExit(upload); - }); + let manifest = await manifestOf(aq, dst => matched.has(dst)); + let reply = await negotiate(upload, manifest); + if ('error' in reply) { + // What the command had to say is already on stdout; this + // is for the case where it said nothing usable. + let code = await waitExit(upload); + console.log(` refused before anything was sent` + + (reply.error === null ? '' : `: ${reply.error}`)); + return code === 0 || code === null ? 1 : code; + } + let send = new Set([...reply.wanted, ...manifest.metaFiles]); + console.log(` ${reply.wanted.size} of ${manifest.assets.size} files to send`); + let code; + if (send.size === 0) { + // Nothing follows an empty answer, not even an empty tar: + // the command takes the end of the stream as the end. + stdin.end(); + code = await waitExit(upload); + } else { + // The bar tracks the upload itself: an entry counts once + // the command has taken it, not once it has been read off + // disk. It counts what travels, not what was published. + code = await withProgress(`${name}: uploading`, send.size, async tick => { + let pack = await aq.pack(dst => send.has(dst), tick); + pack.on('error', () => {}); + pack.pipe(stdin); + return await waitExit(upload); + }); + } if (code !== 0) { return 1; } diff --git a/tools/deploy/protocol.ts b/tools/deploy/protocol.ts new file mode 100644 index 00000000..293dd4a2 --- /dev/null +++ b/tools/deploy/protocol.ts @@ -0,0 +1,242 @@ + +// What a deploy says to `smogonctl assets upload` before it sends anything, and +// how it reads the answer. +// +// Every published name carries a content hash and the served tree is add-only, so +// a name the home already has is a file that needn't travel. The deploy therefore +// opens the stream with a manifest -- every name with its size, and every link -- +// and the command answers with the positions of the ones it hasn't got. Only those, +// plus the __meta/ files, are then tarred into the same stdin. The same connection +// and the same command line as a bare tar; the tar is just shorter. +// +// The format is binary and every string is length prefixed, so a name is whatever +// bytes a tar would carry for it and nothing in the framing could be mistaken for +// one. The server side spells it out in smogonctl's assets command; this is the +// other half, and the two are tested against each other by shape rather than +// sharing code. +// +// "smogonctl assets" u32 version=1 u32 count +// count times: u8 kind +// kind 1 (asset) u64 size u32 len path +// kind 2 (metadata) u64 size u32 len path (always sent) +// kind 3 (link) u32 len path u32 len linkname +// +// The reply is the same header with the count of what's wanted, then that many u32 +// positions into the manifest. Whatever the command prints after that is its own +// report, relayed as is. + +import type {ChildProcess} from 'node:child_process'; +import * as fs from 'node:fs/promises'; + +import {type ActionQueue} from './queue.ts'; + +export let MAGIC = Buffer.from('smogonctl assets'); +export let VERSION = 1; +let HEADER_SIZE = MAGIC.length + 4 + 4; +let ASSET = 1; +let METAFILE = 2; +let LINK = 3; +let META = '__meta'; + +export type Manifest = { + bytes: Buffer, + // The published name at each position: what a position in the reply means. + entries: string[], + // Positions that are assets, the only ones a reply may name. + assets: Set, + // Names the tar carries whatever the reply says. + metaFiles: Set, +}; + +function isMeta(dst: string): boolean { + return dst === META || dst.startsWith(`${META}/`); +} + +function u32(n: number): Buffer { + let b = Buffer.alloc(4); + b.writeUInt32BE(n); + return b; +} + +function u64(n: number): Buffer { + let b = Buffer.alloc(8); + b.writeBigUInt64BE(BigInt(n)); + return b; +} + +function str(s: string): Buffer { + let raw = Buffer.from(s); + return Buffer.concat([u32(raw.length), raw]); +} + +// The manifest for what `filter` selects of the queue, in queue order: the order +// the tar's entries would come in, and the order the reply's positions index. +export async function manifestOf(aq: ActionQueue, filter?: (dst: string) => boolean): Promise { + let parts: Buffer[] = []; + let entries: string[] = []; + let assets = new Set(); + let metaFiles = new Set(); + for (let entry of aq.log) { + if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) { + continue; + } + let op = entry.op; + let index = entries.length; + entries.push(entry.dst); + if (op.type === 'Symlink') { + parts.push(Buffer.from([LINK]), str(entry.dst), str(op.target)); + continue; + } + // The size the tar header will carry: a Write's data goes in as utf-8, the + // same way tar-stream takes the string. + let size = op.type === 'Copy' ? (await fs.stat(op.src)).size : Buffer.byteLength(op.data); + if (isMeta(entry.dst)) { + metaFiles.add(entry.dst); + parts.push(Buffer.from([METAFILE])); + } else { + assets.add(index); + parts.push(Buffer.from([ASSET])); + } + parts.push(u64(size), str(entry.dst)); + } + let bytes = Buffer.concat([MAGIC, u32(VERSION), u32(entries.length), ...parts]); + return {bytes, entries, assets, metaFiles}; +} + +export type Outcome = 'pending' | 'ok' | 'refused'; + +// The reply, read as it arrives. Strict from the first byte: the command's stdout +// is either a reply followed by its report, or it is a refusal, and a refusal is +// relayed whole. Nothing that isn't the reply is skipped over to find one, since +// anything printing on that stream ahead of the command is something the deploy +// shouldn't be talking through -- a shell profile, a filter -- and finding out is +// better than working around it. +export class Reply { + outcome: Outcome = 'pending'; + // Why a refusal was ours rather than the command's, or null where the command + // said so itself and its words were relayed. + error: string | null = null; + wanted = new Set(); + #manifest: Manifest; + #sink: (chunk: Buffer) => void; + #buf = Buffer.alloc(0); + #count = -1; + #seen = new Set(); + + constructor(manifest: Manifest, sink: (chunk: Buffer) => void) { + this.#manifest = manifest; + this.#sink = sink; + } + + feed(chunk: Buffer): void { + if (this.outcome !== 'pending') { + this.#sink(chunk); + return; + } + this.#buf = Buffer.concat([this.#buf, chunk]); + // Refused as early as the bytes disagree, not once there are enough of + // them: a one-line message is shorter than a header. + let head = this.#buf.subarray(0, MAGIC.length); + if (!head.equals(MAGIC.subarray(0, head.length))) { + this.#refuse('the command didn\'t answer the manifest'); + return; + } + if (this.#buf.length < HEADER_SIZE) { + return; + } + if (this.#count < 0) { + let version = this.#buf.readUInt32BE(MAGIC.length); + if (version !== VERSION) { + this.#refuse(`the command answered with version ${version}, and this speaks ${VERSION}`); + return; + } + this.#count = this.#buf.readUInt32BE(MAGIC.length + 4); + } + let offset = HEADER_SIZE + 4 * this.#seen.size; + while (this.#seen.size < this.#count && this.#buf.length >= offset + 4) { + let index = this.#buf.readUInt32BE(offset); + offset += 4; + let dst = this.#manifest.entries[index]; + if (dst === undefined || !this.#manifest.assets.has(index)) { + this.#refuse(`the command asked for position ${index}, which isn't an asset in the manifest`); + return; + } + if (this.#seen.has(index)) { + this.#refuse(`the command asked for ${dst} twice`); + return; + } + this.#seen.add(index); + this.wanted.add(dst); + } + if (this.#seen.size === this.#count) { + this.outcome = 'ok'; + let rest = this.#buf.subarray(offset); + this.#buf = Buffer.alloc(0); + if (rest.length > 0) { + this.#sink(rest); + } + } + } + + // The stream closed. A reply still being read is a command that went away + // without finishing its answer. + end(): void { + if (this.outcome === 'pending') { + this.#refuse('the command hung up before answering the manifest'); + } + } + + #refuse(error: string): void { + this.outcome = 'refused'; + this.error = error; + let held = this.#buf; + this.#buf = Buffer.alloc(0); + if (held.length > 0) { + this.#sink(held); + } + } +} + +// Send the manifest and read the reply: the names to tar, or null where the +// command refused. On a refusal stdin is closed, so a command waiting for a tar +// sees the end of the stream rather than a deploy waiting for it; everything it +// printed is relayed to `sink` either way, and goes on being relayed after. +export function negotiate(child: ChildProcess, manifest: Manifest, + sink: (chunk: Buffer) => void = chunk => { process.stdout.write(chunk); }) + : Promise<{wanted: Set} | {error: string | null}> { + let stdin = child.stdin; + let stdout = child.stdout; + if (stdin === null || stdout === null) { + throw new Error('negotiate needs the command\'s stdin and stdout piped'); + } + return new Promise(resolve => { + let reply = new Reply(manifest, sink); + let settled = false; + let settle = () => { + if (settled || reply.outcome === 'pending') { + return; + } + settled = true; + if (reply.outcome === 'refused') { + stdin.end(); + resolve({error: reply.error}); + } else { + resolve({wanted: reply.wanted}); + } + }; + // Attached before the manifest goes out and kept for the whole run: the + // reply can be longer than a pipe holds, and the report after it has to + // land somewhere while the tar is going the other way. + stdout.on('data', (chunk: Buffer) => { + reply.feed(chunk); + settle(); + }); + stdout.on('end', () => { + reply.end(); + settle(); + }); + // A command that dies on the manifest closes the pipe under this write; + // its exit code is the report, not the EPIPE. + stdin.write(manifest.bytes, () => {}); + }); +} diff --git a/tools/deploy/test/protocol.test.ts b/tools/deploy/test/protocol.test.ts new file mode 100644 index 00000000..8dfcdf63 --- /dev/null +++ b/tools/deploy/test/protocol.test.ts @@ -0,0 +1,212 @@ +import assert from 'node:assert/strict'; +import {spawn} from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as pathlib from 'node:path'; +import {test} from 'node:test'; + +import {MAGIC, Reply, manifestOf, negotiate, type Manifest} from '../protocol.ts'; +import {ActionQueue} from '../queue.ts'; + +let here = pathlib.dirname(new URL(import.meta.url).pathname); +let TIMEOUT = 10_000; + +function tmpdir(): string { + return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-protocol-test-')); +} + +// A queue shaped like a small deploy: two stamped assets, one of them named +// awkwardly, a manifest under __meta/ and a link mirroring an asset. +async function sampleQueue(dir: string): Promise<{aq: ActionQueue, manifest: Manifest}> { + let aq = new ActionQueue(); + let src = pathlib.join(dir, 'a.gif'); + fs.writeFileSync(src, 'gif bytes'); + aq.copy(src, 'sprites/xy/a-HASH.gif'); + aq.write('héllo', 'sprites/xy/b c\td-HASH.txt'); + aq.write('{}', '__meta/xy/manifest.json'); + aq.symlink('../../sprites/xy/a-HASH.gif', '__meta/links/xy/a.gif'); + return {aq, manifest: await manifestOf(aq)}; +} + +function u32(n: number): Buffer { + let b = Buffer.alloc(4); + b.writeUInt32BE(n); + return b; +} + +function reply(indices: number[], version = 1): Buffer { + return Buffer.concat([MAGIC, u32(version), u32(indices.length), ...indices.map(u32)]); +} + +test('manifestOf spells the wire format, in queue order, sizes as the tar would carry them', async () => { + let {manifest} = await sampleQueue(tmpdir()); + assert.deepEqual(manifest.entries, + ['sprites/xy/a-HASH.gif', 'sprites/xy/b c\td-HASH.txt', '__meta/xy/manifest.json', '__meta/links/xy/a.gif']); + assert.deepEqual([...manifest.assets], [0, 1]); + assert.deepEqual([...manifest.metaFiles], ['__meta/xy/manifest.json']); + + let b = manifest.bytes; + assert.ok(b.subarray(0, 16).equals(MAGIC)); + assert.equal(b.readUInt32BE(16), 1, 'version'); + assert.equal(b.readUInt32BE(20), 4, 'count'); + let at = 24; + let expectFile = (kind: number, size: number, name: string) => { + assert.equal(b[at], kind, `kind of ${name}`); + assert.equal(Number(b.readBigUInt64BE(at + 1)), size, `size of ${name}`); + let raw = Buffer.from(name); + assert.equal(b.readUInt32BE(at + 9), raw.length, `name length of ${name}`); + assert.ok(b.subarray(at + 13, at + 13 + raw.length).equals(raw), `name of ${name}`); + at += 13 + raw.length; + }; + expectFile(1, 'gif bytes'.length, 'sprites/xy/a-HASH.gif'); + // Six characters, seven bytes: the size is the utf-8's, since that is what + // tar-stream writes, and the tab rides inside the length-prefixed name. + expectFile(1, 6, 'sprites/xy/b c\td-HASH.txt'); + expectFile(2, 2, '__meta/xy/manifest.json'); + assert.equal(b[at], 3, 'a link'); + let name = Buffer.from('__meta/links/xy/a.gif'); + let target = Buffer.from('../../sprites/xy/a-HASH.gif'); + assert.equal(b.readUInt32BE(at + 1), name.length); + assert.ok(b.subarray(at + 5, at + 5 + name.length).equals(name)); + at += 5 + name.length; + assert.equal(b.readUInt32BE(at), target.length); + assert.ok(b.subarray(at + 4, at + 4 + target.length).equals(target)); + assert.equal(at + 4 + target.length, b.length, 'nothing after the last entry'); +}); + +test('manifestOf takes only what the filter selects, positions renumbered', async () => { + let {aq} = await sampleQueue(tmpdir()); + let manifest = await manifestOf(aq, dst => dst.startsWith('__meta/')); + assert.deepEqual(manifest.entries, ['__meta/xy/manifest.json', '__meta/links/xy/a.gif']); + assert.equal(manifest.assets.size, 0); + assert.equal(manifest.bytes.readUInt32BE(20), 2); +}); + +test('Reply reads a reply however it is chunked and relays the report after it', async () => { + let {manifest} = await sampleQueue(tmpdir()); + let bytes = Buffer.concat([reply([1, 0]), Buffer.from('* 2 files\n')]); + for (let step of [bytes.length, 1, 7]) { + let relayed: Buffer[] = []; + let r = new Reply(manifest, chunk => relayed.push(chunk)); + for (let i = 0; i < bytes.length; i += step) { + r.feed(bytes.subarray(i, i + step)); + } + assert.equal(r.outcome, 'ok', `chunked by ${step}`); + assert.deepEqual([...r.wanted].sort(), ['sprites/xy/a-HASH.gif', 'sprites/xy/b c\td-HASH.txt']); + assert.equal(Buffer.concat(relayed).toString(), '* 2 files\n', `report chunked by ${step}`); + } +}); + +test('Reply refuses anything that is not a reply, and relays it whole', async () => { + let {manifest} = await sampleQueue(tmpdir()); + + // Text where the magic should be: a refusal, a shell profile, a filter + let relayed: Buffer[] = []; + let r = new Reply(manifest, chunk => relayed.push(chunk)); + r.feed(Buffer.from('No such asset set: sprites\n')); + assert.equal(r.outcome, 'refused'); + assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites\n'); + // And what comes after goes through too + r.feed(Buffer.from('more\n')); + assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites\nmore\n'); + + // Even where it starts out looking right + r = new Reply(manifest, () => {}); + r.feed(Buffer.from('smogonctl asset')); + assert.equal(r.outcome, 'pending', 'a prefix of the magic is still possible'); + r.feed(Buffer.from('!')); + assert.equal(r.outcome, 'refused'); + + r = new Reply(manifest, () => {}); + r.feed(reply([0], 2)); + assert.equal(r.outcome, 'refused'); + assert.match(r.error!, /version 2/); + + r = new Reply(manifest, () => {}); + r.feed(reply([7])); + assert.equal(r.outcome, 'refused'); + assert.match(r.error!, /position 7/); + + r = new Reply(manifest, () => {}); + r.feed(reply([2])); + assert.equal(r.outcome, 'refused', 'a __meta/ file is not something to ask for'); + + r = new Reply(manifest, () => {}); + r.feed(reply([0, 0])); + assert.equal(r.outcome, 'refused'); + assert.match(r.error!, /twice/); + + // Hung up partway through the header, and partway through the positions + r = new Reply(manifest, () => {}); + r.feed(reply([0, 1]).subarray(0, 10)); + r.end(); + assert.equal(r.outcome, 'refused'); + assert.match(r.error!, /hung up/); + r = new Reply(manifest, () => {}); + r.feed(reply([0, 1]).subarray(0, 26)); + r.end(); + assert.equal(r.outcome, 'refused'); + + // An empty answer is an answer + r = new Reply(manifest, () => {}); + r.feed(reply([])); + assert.equal(r.outcome, 'ok'); + assert.equal(r.wanted.size, 0); +}); + +function run(cmd: string, args: string[]) { + let child = spawn(cmd, args, {stdio: ['pipe', 'pipe', 'inherit']}); + child.stdin!.on('error', () => {}); + let exit = new Promise(resolve => child.on('close', resolve)); + return {child, exit}; +} + +test('negotiate against a command that answers: only what it asked for travels', {timeout: TIMEOUT}, async () => { + let dir = tmpdir(); + let {aq, manifest} = await sampleQueue(dir); + let out = pathlib.join(dir, 'seen.json'); + let {child, exit} = run(process.execPath, [pathlib.join(here, 'servers/answer.ts'), out]); + let relayed: Buffer[] = []; + let result = await negotiate(child, manifest, chunk => relayed.push(chunk)); + assert.ok('wanted' in result, `should be answered, got ${JSON.stringify(result)}`); + // Every other asset: position 0 and not 1 + assert.deepEqual([...result.wanted], ['sprites/xy/a-HASH.gif']); + let send = new Set([...result.wanted, ...manifest.metaFiles]); + (await aq.pack(dst => send.has(dst))).pipe(child.stdin!); + assert.equal(await exit, 0); + let seen = JSON.parse(fs.readFileSync(out, 'utf8')); + assert.deepEqual(seen.manifest.map((e: {name: string}) => e.name), manifest.entries); + assert.deepEqual(seen.received, [ + {name: 'sprites/xy/a-HASH.gif', size: 9}, + {name: '__meta/xy/manifest.json', size: 2}, + ]); + assert.equal(Buffer.concat(relayed).toString(), '* 2 received\n', 'the report after the reply is relayed'); +}); + +test('negotiate against a command that refuses: relayed, stdin closed, its exit code kept', {timeout: TIMEOUT}, async () => { + let {manifest} = await sampleQueue(tmpdir()); + let {child, exit} = run(process.execPath, [pathlib.join(here, 'servers/refuse.ts')]); + let relayed: Buffer[] = []; + let result = await negotiate(child, manifest, chunk => relayed.push(chunk)); + assert.ok('error' in result); + assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites.\n'); + assert.equal(await exit, 3); +}); + +test('negotiate against cat: an echo of the manifest is not a reply', {timeout: TIMEOUT}, async () => { + let {manifest} = await sampleQueue(tmpdir()); + let {child, exit} = run('cat', []); + let result = await negotiate(child, manifest, () => {}); + assert.ok('error' in result, 'the manifest coming back starts with the magic but is no reply'); + // stdin was closed for it, so cat finishes instead of waiting + assert.equal(await exit, 0); +}); + +test('negotiate against a command that says nothing: refused when it hangs up', {timeout: TIMEOUT}, async () => { + let {manifest} = await sampleQueue(tmpdir()); + let {child, exit} = run('true', []); + let result = await negotiate(child, manifest, () => {}); + assert.ok('error' in result); + assert.match(result.error!, /hung up|didn't answer/); + await exit; +}); diff --git a/tools/deploy/test/servers/answer.ts b/tools/deploy/test/servers/answer.ts new file mode 100644 index 00000000..541a7d6c --- /dev/null +++ b/tools/deploy/test/servers/answer.ts @@ -0,0 +1,94 @@ +// A stand-in for `smogonctl assets upload`: reads the manifest, asks for every +// other asset, takes the tar, and writes what it saw to the path in argv[2] as +// JSON. Prints one line of report after the reply, the way the real one does. + +import * as fs from 'node:fs'; +import tar from 'tar-stream'; + +let MAGIC = Buffer.from('smogonctl assets'); +let out = process.argv[2]!; + +let buf = Buffer.alloc(0); +let pending: {n: number, resolve: (b: Buffer) => void} | null = null; +let ended = false; + +process.stdin.on('data', (chunk: Buffer) => { + buf = Buffer.concat([buf, chunk]); + serve(); +}); +process.stdin.on('end', () => { + ended = true; + serve(); +}); + +function serve() { + if (pending !== null && buf.length >= pending.n) { + let {n, resolve} = pending; + pending = null; + let head = buf.subarray(0, n); + buf = buf.subarray(n); + resolve(head); + } else if (pending !== null && ended) { + throw new Error('manifest ended early'); + } +} + +function take(n: number): Promise { + return new Promise(resolve => { + pending = {n, resolve}; + serve(); + }); +} + +async function name(): Promise { + let len = (await take(4)).readUInt32BE(0); + return (await take(len)).toString(); +} + +let header = await take(24); +if (!header.subarray(0, 16).equals(MAGIC)) { + throw new Error('not a manifest'); +} +let count = header.readUInt32BE(20); +let manifest: {kind: number, name: string, size?: number, target?: string}[] = []; +for (let i = 0; i < count; i++) { + let kind = (await take(1))[0]!; + if (kind === 3) { + manifest.push({kind, name: await name(), target: await name()}); + } else { + let size = Number((await take(8)).readBigUInt64BE(0)); + manifest.push({kind, name: await name(), size}); + } +} +let asked = manifest.map((e, i) => i).filter(i => manifest[i]!.kind === 1 && i % 2 === 0); +let reply = Buffer.alloc(24 + 4 * asked.length); +MAGIC.copy(reply); +reply.writeUInt32BE(1, 16); +reply.writeUInt32BE(asked.length, 20); +asked.forEach((index, i) => reply.writeUInt32BE(index, 24 + 4 * i)); +process.stdout.write(reply); + +// Whatever follows is the tar, or nothing at all. +let received: {name: string, size: number}[] = []; +let extract = tar.extract(); +extract.on('entry', (header, stream, next) => { + received.push({name: header.name, size: header.size ?? 0}); + stream.on('end', next); + stream.resume(); +}); +let done = new Promise((resolve, reject) => { + extract.on('finish', resolve); + extract.on('error', reject); +}); +if (buf.length > 0) { + extract.write(buf); +} +buf = Buffer.alloc(0); +if (ended) { + extract.end(); +} else { + process.stdin.pipe(extract); +} +await done; +fs.writeFileSync(out, JSON.stringify({manifest, asked, received})); +process.stdout.write(`* ${received.length} received\n`); diff --git a/tools/deploy/test/servers/refuse.ts b/tools/deploy/test/servers/refuse.ts new file mode 100644 index 00000000..0442c221 --- /dev/null +++ b/tools/deploy/test/servers/refuse.ts @@ -0,0 +1,4 @@ +// A command that reads nothing and says no: what a refused manifest looks like +// from the deploy's side, with the exit code that goes with it. +process.stdout.write('No such asset set: sprites.\n'); +process.exit(3);