From 68cdcf989183af22762dbe1dae19850d7c15a779 Mon Sep 17 00:00:00 2001 From: Christopher Monsanto Date: Mon, 19 Oct 2026 11:39:14 -0400 Subject: [PATCH] Publish straight into a served tree, and drop the smogonctl upload path `publish --tree --meta ` places the outputs itself: the tree is add-only (a name already there is touched, another size is refused), CAS objects are hardlinked in, __meta/ goes to its own directory with the links pointed into the tree, and __files/__digest record the publish. The deploy verb, deploy.json5, the manifest/tar negotiation with `smogonctl assets upload`, the progress bar and `run --tar` go with it; smogon.com's sprites-sync service runs build and publish on the server now. The output root is the served tree, no sprites/ prefix. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TNa8LzaaMo3QbEVuFUtFGU --- .gitignore | 1 - README.md | 129 +++++---------- package.json | 1 - pnpm-lock.yaml | 158 ------------------ rules/publish.ts | 21 ++- smogon.build.ts | 25 ++- tools/deploy/config.ts | 99 ------------ tools/deploy/index.ts | 186 ++++++--------------- tools/deploy/package.json | 5 +- tools/deploy/progress.ts | 106 ------------ tools/deploy/protocol.ts | 242 ---------------------------- tools/deploy/queue.ts | 240 ++++++++++++++++++--------- tools/deploy/test/api.test.ts | 171 +++++++++----------- tools/deploy/test/config.test.ts | 75 --------- tools/deploy/test/progress.test.ts | 76 --------- tools/deploy/test/protocol.test.ts | 212 ------------------------ tools/deploy/test/servers/answer.ts | 94 ----------- 17 files changed, 358 insertions(+), 1483 deletions(-) delete mode 100644 tools/deploy/config.ts delete mode 100644 tools/deploy/progress.ts delete mode 100644 tools/deploy/protocol.ts delete mode 100644 tools/deploy/test/config.test.ts delete mode 100644 tools/deploy/test/progress.test.ts delete mode 100644 tools/deploy/test/protocol.test.ts delete mode 100644 tools/deploy/test/servers/answer.ts diff --git a/.gitignore b/.gitignore index 5932f34d..65571d87 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,3 @@ pnpm-debug.log .cache/ .build/ build.config -deploy.json5 diff --git a/README.md b/README.md index 93eb4f67..ea9278d9 100644 --- a/README.md +++ b/README.md @@ -46,13 +46,13 @@ Using [`brew`](https://brew.sh/) on a macOS: $ brew install imagemagick gifsicle advancecomp optipng pngquant webp ``` -## Building and deploying +## Building Install dependencies once with `pnpm install`. -Each deploy is a root `*.build.ts` module: it declares its build rules +Each output tree is a root `*.build.ts` module: it declares its build rules (shared sets are plain functions in `rules/`; declaring an identical rule -twice is a no-op returning the existing artifacts, so any number of deploys +twice is a no-op returning the existing artifacts, so any number of modules can call the same set) and, next to each set of rules, a `deploy(ctx => ...)` block that maps the built artifacts to their published names; the blocks run in registration order after the build, sharing one output tree per module. Build outputs are content-addressed: @@ -62,20 +62,19 @@ content, same-byte renames rebuild nothing, and hash-stamped publishing reuses the build's digests. All state lives in `.build/`. ``` -$ pnpm build # build every deploy's rules, GC stale state -$ pnpm deploy # list the deploys in deploy.json5 -$ pnpm deploy smogon # run a named deploy -$ node tools/deploy/index.ts build smogon.build.ts # build one deploy's rules -$ node tools/deploy/index.ts run smogon.build.ts -o deploy/smogon +$ pnpm build # build every module's rules, GC stale state +$ node tools/deploy/index.ts build smogon.build.ts # build one module's rules +$ node tools/deploy/index.ts run smogon.build.ts -o /tmp/out +$ node tools/deploy/index.ts publish smogon.build.ts --tree /srv/served --meta /srv/publishes/1 $ node tools/deploy/index.ts inspect src/minisprites/items/ileftovers.png -o /tmp/out -$ node tools/deploy/index.ts refactor --record # remember what the deploys publish +$ node tools/deploy/index.ts refactor --record # remember what the modules publish $ node tools/deploy/index.ts refactor # and what a change did to it ``` -`run` materializes a deploy to a directory (`--link` hardlinks, `--tar` -writes a tar file) without uploading anything. `inspect` builds every rule -that consumes the given source paths and copies the outputs out under -readable names for eyeballing. +`run` materializes a module's outputs to a directory (`--link` hardlinks) +without publishing anything. `inspect` builds every rule that consumes the +given source paths and copies the outputs out under readable names for +eyeballing. `refactor` answers "did that change anything we ship". It builds, runs the deploy blocks, and digests the bytes landing at every published name, then @@ -88,86 +87,40 @@ run it again. Useful flags: `-j ` parallelism, `-n` dry run, `-v` verbose, `--fail-fast` stop after the first failure. -## Deploying +A build can be interrupted at any point (SIGINT or SIGTERM: running rules are +stopped, a second signal kills them) and picks up where it left off next time. -`deploy` reads `deploy.json5` at the repo root (not tracked by git). It maps -deploy names to a buildFile and a list of (subset, cmd) entries: after -building and finishing the buildFile, each entry's globs select a subset of -the finish outputs and sends it to the entry's command, which is `smogonctl -assets upload` (possibly behind `ssh` and `sudo`). An entry with `dir: true` -instead materializes the subset into a temp directory whose path replaces -`%d` in the command (for rsync-style transports). Every glob must match -something, and every output must be covered by some entry. `deploy -o -` materializes each entry's subset under `///` -instead of running its command, for eyeballing what would ship. +## Publishing -Sending is a conversation over the command's stdin and stdout rather than a -tar poured into it. Every served name carries a content hash and the served -tree is add-only, so a name the home already has is a file that needn't -travel: the deploy first writes a manifest -- every name with its size, and -every `__meta/` link -- and the command answers with the ones it hasn't got. -Only those, plus the `__meta/` files, are then tarred into the same stream. A -redeploy of an unchanged tree moves kilobytes. The command's stdout has to be -the upload's alone and reach the deploy unfiltered: the reply is the first -thing on it, and anything else there (a shell profile that prints over ssh, a -`| grep`) is taken as a refusal and fails the deploy rather than being -skipped past. `run --tar` still writes a bare tar, which the upload also -accepts. +`publish --tree --meta ` builds, runs the module's deploy +blocks and places the outputs for serving. The output root is the served +tree, and `__meta/` at the root is everything that isn't served. -Where stderr is a terminal, each entry draws a progress bar over the files -that travel while they go out -- an entry counts once the command has taken -it, not once it has been read off disk -- and the line is cleared again -afterwards. Piped or under CI nothing is drawn, and the lines the deploy -prints are the same either way. +The tree is add-only. Every name in it carries a content hash and is served +as if it can never change, so a name already there is the same file and is +touched rather than rewritten, and one there at another size fails the +publish. New files are hardlinked from the CAS where they can be and written +beside their final name and renamed into place otherwise, so nothing ever +reads a half-written file. -This file is not committed, because it is where the hosts and paths this -repo ships to are written down. +`--meta` is this publish's own directory and is written whole. Three kinds of +thing ride there. A manifest, `name` -> the whole url of the stamped file, +for a reader that looks one up. A pointer file naming a single url, for the +one-file sets. And `__meta/links/`, the served tree recreated as symlinks +under the un-stamped names, which is the same mapping said in names instead +of a file; it lives out here rather than in the tree because the tree is +add-only and a link is repointed on every publish. A link names where in the +output its file is (`__meta/links/xy/charizard.gif` names +`xy/charizard-.gif`), and the publish points it at where that file sits +in the tree; a link to anything the publish doesn't carry is refused. The +publish also writes `__files`, every tree name it stands behind, and +`__digest`, a digest of every name and the bytes under it; names under +`__meta/` starting with `__` are reserved for these. -```json5 -{ - smogon: { - buildFile: "smogon.build.ts", - deploy: [ - {subset: ["**"], cmd: "smogonctl assets upload sprites"}, - ], - }, -} -``` - -Note that the coverage rule is per deploy name, over the whole buildFile's -outputs: two names on one buildFile can't split its tree between them, since -each of them has to cover all of it. - -### The asset upload's tar layout - -`smogonctl assets upload` publishes a tree into a served tree under a prefix -named in the receiving home's `services.toml`, which this side can't read. So -`smogon.build.ts` writes that prefix itself -- everything served ships under -`sprites/` -- and the upload rejects a tar whose tree disagrees. The two are -checked against each other instead of each guessing, which is what lets the -manifests and pointers in `__meta/` name whole urls (`/__assets/sprites/...`) -and their readers hold no configuration at all. - -`__meta/` is the exception and stays at the tar root: the upload diverts it to -`assets-meta/`, beside the served tree rather than in it, because a served -name carries a content hash and something un-stamped has to say which name to -ask for. - -Three kinds of thing ride there. A manifest, `name` -> the whole url of the -stamped file, for a reader that looks one up. A pointer file naming a single -url, for the one-file sets. And `__meta/links/`, the served tree recreated as -symlinks under the un-stamped names, which is the same mapping said in names -instead of a file. It lives out here rather than in the tree because the tree -is add-only -- a name in it is promised never to change -- and a link is -repointed on every upload. - -A link says where in the tar its file is, not what to write into the link: -`__meta/links/xy/charizard.gif` names `sprites/xy/charizard-.gif` and -what is packed is the path between the two. The two halves land in different -trees on the far side (`assets-meta//` and `assets/`), so the upload -retargets every link at where its asset actually went, and refuses one naming -anything that tar didn't carry. The mirror doesn't repeat the `sprites/` -prefix, since the directory it lands in is already this set's. +Production runs this: smogon.com's `sprites-sync` service fetches this +repository, runs `build` and then `publish`, swaps the meta directory in for +its readers and sweeps tree files no publish has named for a while. Each +checkout's `build.config` is written by whoever made the checkout. ## Configuration diff --git a/package.json b/package.json index 83cf5255..7cafc5b9 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,6 @@ "type": "module", "scripts": { "build": "node tools/deploy/index.ts build", - "deploy": "node tools/deploy/index.ts deploy", "check": "tsc --build tsconfig-workspace.json" }, "dependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5a205613..8c7c97e1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,18 +45,9 @@ importers: '@types/node': specifier: ^24.10.7 version: 24.13.3 - '@types/tar-stream': - specifier: ^2.2.2 - version: 2.2.3 base32-encode: specifier: ^2.0.0 version: 2.0.0 - json5: - specifier: ^2.2.3 - version: 2.2.3 - tar-stream: - specifier: ^3.0.0 - version: 3.2.0 tools/smogdexspritesheet: dependencies: @@ -90,9 +81,6 @@ packages: '@types/spritesmith@3.4.5': resolution: {integrity: sha512-LRyCIJ+7LW59eFnUVfF4Wwhs23MceEqFQQtV7l7hHYz/z0zeN24J5TJOE2qZjaUOsGcSZ0ISDTIhe2dvpWrnmw==} - '@types/tar-stream@2.2.3': - resolution: {integrity: sha512-if3mugZfjVkXOMZdFjIHySxY13r6GXPpyOlsDmLffvyI7tLz9wXE8BFjNivXsvUeyJ1KNlOpfLnag+ISmxgxPw==} - '@types/vinyl@2.0.12': resolution: {integrity: sha512-Sr2fYMBUVGYq8kj3UthXFAu5UN6ZW+rYr4NACjZQJvHvj+c8lYv0CahmZ2P/r7iUkN44gGUBwqxZkrKXYPb7cw==} @@ -238,51 +226,6 @@ packages: aws4@1.13.2: resolution: {integrity: sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==} - b4a@1.8.1: - resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} - peerDependencies: - react-native-b4a: '*' - peerDependenciesMeta: - react-native-b4a: - optional: true - - bare-events@2.9.1: - resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==} - peerDependencies: - bare-abort-controller: '*' - peerDependenciesMeta: - bare-abort-controller: - optional: true - - bare-fs@4.8.0: - resolution: {integrity: sha512-fM+MhCvdQhZ7NV6S95a07gPSqjIYKn6mFaXfx266wN3ajZGl/+1AzH+ubkXQ0fFZvOe2nk9VHkzdYkQE5zMV3Q==} - engines: {bare: '>=1.28.0'} - peerDependencies: - bare-buffer: '*' - peerDependenciesMeta: - bare-buffer: - optional: true - - bare-path@3.1.1: - resolution: {integrity: sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==} - - bare-stream@2.13.3: - resolution: {integrity: sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==} - peerDependencies: - bare-abort-controller: '*' - bare-buffer: '*' - bare-events: '*' - peerDependenciesMeta: - bare-abort-controller: - optional: true - bare-buffer: - optional: true - bare-events: - optional: true - - bare-url@2.5.2: - resolution: {integrity: sha512-L13PCJzKG8RGvx8V1/DdMi12ERhC3tprr7/8a94BxpmnRsFqxh5XZNdhtMxu5HPkRshYOOWRGY8lDP7ZhpG9Cg==} - base32-encode@2.0.0: resolution: {integrity: sha512-mlmkfc2WqdDtMl/id4qm3A7RjW6jxcbAoMjdRmsPiwQP0ufD4oXItYMnPgVHe80lnAIy+1xwzhHE1s4FoIceSw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -338,9 +281,6 @@ packages: ecc-jsbn@0.1.2: resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} - events-universal@1.0.1: - resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} - extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} @@ -351,9 +291,6 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} - fast-fifo@1.3.2: - resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} @@ -433,11 +370,6 @@ packages: json-stringify-safe@5.0.1: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} - json5@2.2.3: - resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} - engines: {node: '>=6'} - hasBin: true - jsprim@1.4.2: resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} engines: {node: '>=0.6.0'} @@ -557,9 +489,6 @@ packages: engines: {node: '>=0.10.0'} hasBin: true - streamx@2.28.0: - resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} - string_decoder@0.10.31: resolution: {integrity: sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==} @@ -574,15 +503,6 @@ packages: resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==} engines: {node: '>=0.10.0'} - tar-stream@3.2.0: - resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==} - - teex@1.0.1: - resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==} - - text-decoder@1.2.7: - resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} - through2@2.0.5: resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} @@ -656,10 +576,6 @@ snapshots: dependencies: '@types/vinyl': 2.0.12 - '@types/tar-stream@2.2.3': - dependencies: - '@types/node': 24.13.3 - '@types/vinyl@2.0.12': dependencies: '@types/expect': 1.20.4 @@ -746,37 +662,6 @@ snapshots: aws4@1.13.2: {} - b4a@1.8.1: {} - - bare-events@2.9.1: {} - - bare-fs@4.8.0: - dependencies: - bare-events: 2.9.1 - bare-path: 3.1.1 - bare-stream: 2.13.3(bare-events@2.9.1) - bare-url: 2.5.2 - fast-fifo: 1.3.2 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - - bare-path@3.1.1: {} - - bare-stream@2.13.3(bare-events@2.9.1): - dependencies: - b4a: 1.8.1 - streamx: 2.28.0 - teex: 1.0.1 - optionalDependencies: - bare-events: 2.9.1 - transitivePeerDependencies: - - react-native-b4a - - bare-url@2.5.2: - dependencies: - bare-path: 3.1.1 - base32-encode@2.0.0: dependencies: to-data-view: 2.0.0 @@ -828,20 +713,12 @@ snapshots: jsbn: 0.1.1 safer-buffer: 2.1.2 - events-universal@1.0.1: - dependencies: - bare-events: 2.9.1 - transitivePeerDependencies: - - bare-abort-controller - extend@3.0.2: {} extsprintf@1.3.0: {} fast-deep-equal@3.1.3: {} - fast-fifo@1.3.2: {} - fast-json-stable-stringify@2.1.0: {} first-chunk-stream@1.0.0: {} @@ -917,8 +794,6 @@ snapshots: json-stringify-safe@5.0.1: {} - json5@2.2.3: {} - jsprim@1.4.2: dependencies: assert-plus: 1.0.0 @@ -1087,15 +962,6 @@ snapshots: safer-buffer: 2.1.2 tweetnacl: 0.14.5 - streamx@2.28.0: - dependencies: - events-universal: 1.0.1 - fast-fifo: 1.3.2 - text-decoder: 1.2.7 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - string_decoder@0.10.31: {} string_decoder@1.1.1: @@ -1111,30 +977,6 @@ snapshots: dependencies: is-utf8: 0.2.1 - tar-stream@3.2.0: - dependencies: - b4a: 1.8.1 - bare-fs: 4.8.0 - fast-fifo: 1.3.2 - streamx: 2.28.0 - transitivePeerDependencies: - - bare-abort-controller - - bare-buffer - - react-native-b4a - - teex@1.0.1: - dependencies: - streamx: 2.28.0 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - - text-decoder@1.2.7: - dependencies: - b4a: 1.8.1 - transitivePeerDependencies: - - react-native-b4a - through2@2.0.5: dependencies: readable-stream: 2.3.8 diff --git a/rules/publish.ts b/rules/publish.ts index d5e27300..21338347 100644 --- a/rules/publish.ts +++ b/rules/publish.ts @@ -1,4 +1,6 @@ +import * as nodePath from 'node:path'; + import * as spritedata from '@smogon/sprite-data/index.ts'; import type {Artifact} from '../tools/build/artifact.ts'; @@ -8,10 +10,10 @@ import type {DeployCtx, SrcFile} from '../tools/deploy/api.ts'; // or a raw source file. export type Sprite = Artifact | SrcFile; -// The published tree a stamped set goes into: where it sits in the output, and -// the url that root is served at. Both live here rather than in a `dir` so -// that they are written once and everything a set publishes -- the copies, the -// urls, the mirror -- is spelled from the same place. +// The published tree a stamped set goes into: where it sits in the output ('' for +// the output root itself), and the url that root is served at. Both live here +// rather than in a `dir` so that they are written once and everything a set +// publishes -- the copies, the urls, the mirror -- is spelled from the same place. export type Tree = { root: string, served: string, @@ -52,7 +54,7 @@ export class Manifest { } let stamped = `${name}-${h}.${e}`; this.#entries.set(name, {dir, stamped, filename: `${name}.${e}`}); - this.#ctx.copy(f, `${this.#tree.root}/${dir}/${stamped}`); + this.#ctx.copy(f, nodePath.posix.join(this.#tree.root, dir, stamped)); } // The whole url the copy `name` answers to. Exposed on its own for a set @@ -74,15 +76,12 @@ export class Manifest { } // The same set again under its un-stamped names, as links to the stamped - // files. `dst` is a root of its own rather than the tree's, so what a link - // says is a path and not a name: the two ends land apart, and the reader - // that follows one composed its half from nothing but the sprite. The - // tree's own root is not repeated under it, since the mirror is already - // one set's worth of it. + // files. `dst` is a root of its own rather than the tree's: the reader + // that follows one composed its half from nothing but the sprite. links(dst: string): void { for (let [, e] of this.#sorted()) { this.#ctx.symlink(`${dst}/${e.dir}/${e.filename}`, - `${this.#tree.root}/${e.dir}/${e.stamped}`); + nodePath.posix.join(this.#tree.root, e.dir, e.stamped)); } } diff --git a/smogon.build.ts b/smogon.build.ts index cceb7630..b1c86c6d 100644 --- a/smogon.build.ts +++ b/smogon.build.ts @@ -9,16 +9,13 @@ import {type Artifact, type CmdSpec, forEachRule, rule} from './tools/build/arti import {base, compresspng, pad, spriteglob, trimimg} from './tools/build/helpers.ts'; import {type DeployCtx, deploy} from './tools/deploy/api.ts'; -// The tar root maps onto the served tree: sprites/x is served at -// /__assets/sprites/x. The upload rejects a tar whose tree disagrees with the -// prefix in services.toml, so the two are checked against each other rather -// than each guessing -- which is what lets the pointers below name whole urls -// and their readers hold no configuration. __meta/ is the exception: the -// upload diverts it to assets-meta/, beside the served tree and out of it. +// The output root is the served tree: x is served at /__assets/sprites/x, +// which is what lets the pointers below name whole urls and their readers +// hold no configuration. __meta/ is the exception: `publish` puts it beside +// the served tree and out of it. -let ASSETS = 'sprites'; -let SERVED = '/__assets'; -let TREE: Tree = {root: ASSETS, served: `${SERVED}/${ASSETS}`}; +let SERVED = '/__assets/sprites'; +let TREE: Tree = {root: '', served: SERVED}; // Where the un-stamped names go. A served name carries a content hash and so // can be cached forever, which is exactly why it can't be composed by a reader @@ -27,8 +24,8 @@ let TREE: Tree = {root: ASSETS, served: `${SERVED}/${ASSETS}`}; // second time, as links under the un-stamped names naming the stamped file. // They ride in __meta/ rather than the served tree because the tree is // add-only -- a name in it is promised never to change -- and a link is -// repointed on every upload; the upload retargets each one at where its asset -// landed on its side, under a directory it already names for this set. +// repointed on every publish, which points each one at where its file sits +// in the tree. let LINKS = '__meta/links'; let minispriteInputs = spriteglob(['src/minisprites/pokemon/gen6/*', 'src/minisprites/items/*'], {a: false}); @@ -216,7 +213,7 @@ let [, sheetCss, sheetWebp] = rule(minispriteInputs, { // Hash-stamped css + webp. The css url rides in __meta/ for the dex to read. deploy(async ctx => { let wh = await ctx.hash(sheetWebp); - ctx.copy(sheetWebp, `${ASSETS}/spritesheet-${wh}.webp`); + ctx.copy(sheetWebp, `spritesheet-${wh}.webp`); let src = await ctx.read(sheetCss); let css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`); if (css === src) { @@ -226,8 +223,8 @@ deploy(async ctx => { // of (css, webp), so this changes exactly when the served bytes // change. let ch = await ctx.hash(sheetCss, sheetWebp); - ctx.write(`${ASSETS}/spritesheet-${ch}.css`, css); - ctx.write('__meta/spritesheet-css-url.txt', `${SERVED}/${ASSETS}/spritesheet-${ch}.css\n`); + ctx.write(`spritesheet-${ch}.css`, css); + ctx.write('__meta/spritesheet-css-url.txt', `${SERVED}/spritesheet-${ch}.css\n`); }); // Forumsprites: uniform-size minisprites under stamped names, with the diff --git a/tools/deploy/config.ts b/tools/deploy/config.ts deleted file mode 100644 index 6d56980f..00000000 --- a/tools/deploy/config.ts +++ /dev/null @@ -1,99 +0,0 @@ - -import * as fs from 'node:fs/promises'; -import * as nodePath from 'node:path'; - -import JSON5 from 'json5'; - -import {BuildError} from '../build/errors.ts'; - -export type DeployEntry = { - subset: string[], - cmd: string, - // dir entries get their subset materialized into a temp directory whose - // path replaces %d in cmd. Every other entry's cmd is `smogonctl assets - // upload` (or a wrapper around it): it gets a manifest of the subset on - // stdin, answers on stdout with what it lacks, and gets a tar of that. - dir?: boolean, -}; - -export type DeployTarget = { - buildFile: string, - deploy: DeployEntry[], -}; - -export type DeployConfig = Map; - -function isStringArray(v: unknown): v is string[] { - return Array.isArray(v) && v.every(x => typeof x === 'string'); -} - -export async function loadDeployConfig(path: string): Promise { - let text: string; - try { - text = await fs.readFile(path, 'utf8'); - } catch { - throw new BuildError(`missing ${path}; see README ("Deploying") for the schema`); - } - let raw: unknown; - try { - raw = JSON5.parse(text); - } catch (err) { - throw new BuildError(`${path}: ${(err as Error).message}`); - } - if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) { - throw new BuildError(`${path}: top level must be an object of deploy names`); - } - - let config: DeployConfig = new Map(); - for (let [name, t] of Object.entries(raw)) { - let target = t as Partial; - if (typeof t !== 'object' || t === null || typeof target.buildFile !== 'string' - || !Array.isArray(target.deploy)) { - throw new BuildError(`${path}: ${name}: expected {buildFile: string, deploy: [...]}`); - } - for (let e of target.deploy as Partial[]) { - if (typeof e !== 'object' || e === null - || !isStringArray(e.subset) || typeof e.cmd !== 'string' - || (e.dir !== undefined && typeof e.dir !== 'boolean')) { - throw new BuildError( - `${path}: ${name}: each deploy entry needs {subset: string[], cmd: string, dir?: boolean}`); - } - if (Boolean(e.dir) !== e.cmd.includes('%d')) { - throw new BuildError(`${path}: ${name}: ${e.dir - ? 'a dir entry\'s cmd must use %d' - : 'a tar entry\'s cmd must not use %d (missing dir: true?)'}: ${e.cmd}`); - } - } - config.set(name, target as DeployTarget); - } - return config; -} - -// Route finish outputs to deploy entries: per entry, the set of dsts its -// subset globs match. Every glob must match something and every dst must be -// covered by some entry; to unship an output, don't emit it in finish. -export function matchSubsets(dsts: readonly string[], entries: readonly DeployEntry[]) - : {entry: DeployEntry, matched: Set}[] { - let covered = new Set(); - let perEntry = entries.map(entry => { - let matched = new Set(); - for (let glob of entry.subset) { - let hits = dsts.filter(d => nodePath.matchesGlob(d, glob)); - if (hits.length === 0) { - throw new BuildError(`subset glob matches no outputs: ${glob}`); - } - for (let hit of hits) { - matched.add(hit); - covered.add(hit); - } - } - return {entry, matched}; - }); - let uncovered = dsts.filter(d => !covered.has(d)); - if (uncovered.length > 0) { - let shown = uncovered.slice(0, 10).join('\n '); - let more = uncovered.length > 10 ? `\n ... and ${uncovered.length - 10} more` : ''; - throw new BuildError(`outputs not covered by any deploy entry:\n ${shown}${more}`); - } - return perEntry; -} diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index 7f3fb9c8..1c16c1c2 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -1,5 +1,4 @@ -import {spawn, type ChildProcess} from 'node:child_process'; import * as crypto from 'node:crypto'; import * as fs from 'node:fs/promises'; import * as os from 'node:os'; @@ -17,9 +16,6 @@ import {killAllProcessGroups} from '../build/exec.ts'; import {setConfig} from '../build/helpers.ts'; import * as db from '../build/store.ts'; import * as api from './api.ts'; -import {loadDeployConfig, matchSubsets} from './config.ts'; -import {withProgress} from './progress.ts'; -import {manifestOf, negotiate} from './protocol.ts'; import {ActionQueue} from './queue.ts'; let root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..'); @@ -43,20 +39,20 @@ type CommonOpts = { type VerbOpts = CommonOpts & { output?: string, link?: boolean, - tar?: boolean, record?: boolean, + tree?: string, + meta?: string, }; let USAGE = `usage: node tools/deploy/index.ts [options] commands: build [files...] build the rules of the given deploys (default: all *.build.ts) - deploy [names...] build, finish, and send each subset to its command from - deploy.json5: a manifest and then a tar of what it asks - for on stdin, or a %d dir (no names: list the deploys; - -o : materialize each subset there instead of - running its command) - run -o build, finish, and materialize to a directory (or tar file) + run -o build, finish, and materialize to a directory + publish --tree --meta + build, finish, and place the outputs: the served tree gets + what it hasn't got, __meta/ is written whole to the meta + directory with its links pointing into the tree inspect -o build every rule touching the given source paths and copy the outputs out refactor [files...] report how the published files differ from the last @@ -67,10 +63,11 @@ options: -n, --dry-run print what would run without changing anything --fail-fast stop scheduling new rules after the first failure --config config file (default: build.config) - -o, --output run/inspect/deploy: output directory (a file with --tar) + -o, --output run/inspect: output directory + --tree publish: the served tree (add-only) + --meta publish: where __meta/ goes --record refactor: overwrite the baseline instead of checking it --link run: hardlink instead of copying - --tar run: write a tar file -v, --verbose print more detail -h, --help show this help`; @@ -85,13 +82,13 @@ let COMMON_OPTIONS = { let VERB_OPTIONS = { build: {}, - deploy: { - output: {type: 'string', short: 'o'}, - }, run: { output: {type: 'string', short: 'o'}, link: {type: 'boolean'}, - tar: {type: 'boolean'}, + }, + publish: { + tree: {type: 'string'}, + meta: {type: 'string'}, }, inspect: { output: {type: 'string', short: 'o'}, @@ -218,13 +215,6 @@ async function runFinish(fns: readonly api.DeployFn[], verbose: boolean): Promis return aq; } -function waitExit(child: ChildProcess): Promise { - return new Promise((resolve, reject) => { - child.on('error', reject); - child.on('close', code => resolve(code)); - }); -} - async function cmdBuild(files: string[], opts: CommonOpts): Promise { setConfig(await loadConfig(opts.config)); // GC needs the full rule universe: only an unfiltered union build @@ -234,111 +224,6 @@ async function cmdBuild(files: string[], opts: CommonOpts): Promise { process.exitCode = await buildThen(artifact.getDecls(), opts, gc); } -async function cmdDeploy(names: string[], opts: VerbOpts): Promise { - let config = await loadDeployConfig('deploy.json5'); - if (names.length === 0) { - for (let [name, target] of config) { - console.log(`${name} (${target.buildFile})`); - for (let entry of target.deploy) { - console.log(` ${entry.subset.join(' ')} | ${entry.cmd}`); - } - } - return; - } - let targets = names.map(name => { - let target = config.get(name); - if (target === undefined) { - throw new BuildError(`deploy.json5: no deploy named ${name}`); - } - return {name, target}; - }); - setConfig(await loadConfig(opts.config)); - let files = [...new Set(targets.map(t => t.target.buildFile))]; - let specs = await importDeploys(files); - process.exitCode = await buildThen(artifact.getDecls(), opts, false, async () => { - for (let {name, target} of targets) { - let aq = await runFinish(finishOf(specs, target.buildFile), Boolean(opts.verbose)); - if (aq === null) { - return 1; - } - let dsts = aq.log.filter(e => e.type === 'Op').map(e => e.dst); - for (let [i, {entry, matched}] of matchSubsets(dsts, target.deploy).entries()) { - console.log(`${name}: ${matched.size} files | ${entry.cmd}`); - // Debug mode: materialize what each entry would ship (tar - // entries included) instead of running its command. - if (opts.output !== undefined) { - let dir = nodePath.join(opts.output, name, String(i)); - await withProgress(`${name}: copying`, matched.size, - tick => aq.run(dir, 'copy', dst => matched.has(dst), tick)); - console.log(` -> ${dir}`); - continue; - } - if (entry.dir) { - await fs.mkdir(TMP_DIR, {recursive: true}); - let tmp = await fs.mkdtemp(nodePath.join(TMP_DIR, 'deploy-')); - try { - await withProgress(`${name}: staging`, matched.size, - tick => aq.run(tmp, 'copy', dst => matched.has(dst), tick)); - let cmd = spawn(entry.cmd.replaceAll('%d', tmp), - {shell: true, stdio: ['ignore', 'inherit', 'inherit']}); - if (await waitExit(cmd) !== 0) { - return 1; - } - } finally { - await fs.rm(tmp, {recursive: true, force: true}); - } - continue; - } - // The command's stdout is ours to read: the reply to the - // manifest comes back on it, and its report after that is - // relayed. stderr stays the terminal's, for the bar. - let upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'pipe', 'inherit']}); - let stdin = upload.stdin; - if (stdin === null) { - throw new BuildError(`no stdin pipe for: ${entry.cmd}`); - } - // If the command dies early we report its exit code; don't - // also crash on the resulting EPIPE, which reaches both - // stdin and (via streamx's destroy propagation) the pack. - stdin.on('error', () => {}); - let manifest = await manifestOf(aq, dst => matched.has(dst)); - let reply = await negotiate(upload, manifest); - if ('error' in reply) { - // What the command had to say is already on stdout; this - // is for the case where it said nothing usable. - let code = await waitExit(upload); - console.log(` refused before anything was sent` - + (reply.error === null ? '' : `: ${reply.error}`)); - return code === 0 || code === null ? 1 : code; - } - let send = new Set([...reply.wanted, ...manifest.metaFiles]); - console.log(` ${reply.wanted.size} of ${manifest.assets.size} files to send`); - let code; - if (send.size === 0) { - // Nothing follows an empty answer, not even an empty tar: - // the command takes the end of the stream as the end. - stdin.end(); - code = await waitExit(upload); - } else { - // The bar tracks the upload itself: an entry counts once - // the command has taken it, not once it has been read off - // disk. It counts what travels, not what was published. - code = await withProgress(`${name}: uploading`, send.size, async tick => { - let pack = await aq.pack(dst => send.has(dst), tick); - pack.on('error', () => {}); - pack.pipe(stdin); - return await waitExit(upload); - }); - } - if (code !== 0) { - return 1; - } - } - } - return 0; - }); -} - async function cmdRun(file: string, opts: VerbOpts): Promise { let output = requireOutput(opts); setConfig(await loadConfig(opts.config)); @@ -348,7 +233,36 @@ async function cmdRun(file: string, opts: VerbOpts): Promise { if (aq === null) { return 1; } - await aq.run(output, opts.tar ? 'tar' : opts.link ? 'link' : 'copy'); + await aq.run(output, opts.link ? 'link' : 'copy'); + return 0; + }); +} + +// What the publish stands for as one string: the sorted name -> digest map, +// so two publishes of the same bytes under the same names compare equal +// whatever built them. +function digestOf(outputs: Map): string { + let h = crypto.createHash('sha256'); + for (let [dst, digest] of [...outputs].sort((a, b) => a[0] < b[0] ? -1 : 1)) { + h.update(`${dst}\0${digest}\n`); + } + return h.digest('hex'); +} + +async function cmdPublish(file: string, opts: VerbOpts): Promise { + let {tree, meta} = opts; + if (tree === undefined || meta === undefined) { + throw new BuildError('publish needs --tree and --meta'); + } + setConfig(await loadConfig(opts.config)); + let specs = await importDeploys([file]); + process.exitCode = await buildThen(artifact.getDecls(), opts, false, async () => { + let aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose)); + if (aq === null) { + return 1; + } + let {files, placed, kept} = await aq.publish(tree, meta, digestOf(await outputs(aq))); + console.log(`published ${files} files: ${placed} new, ${kept} already there`); return 0; }); } @@ -521,20 +435,20 @@ async function main(argv: string[]): Promise { verbose: Boolean(v.verbose), output: v.output as string | undefined, link: Boolean(v.link), - tar: Boolean(v.tar), record: Boolean(v.record), + tree: v.tree as string | undefined, + meta: v.meta as string | undefined, }; switch (verb) { case 'build': return cmdBuild(positionals, opts); - case 'deploy': - return cmdDeploy(positionals, opts); - case 'run': { + case 'run': + case 'publish': { let [file, ...extra] = positionals; if (file === undefined || extra.length > 0) { - throw new BuildError('run takes exactly one deploy file'); + throw new BuildError(`${verb} takes exactly one deploy file`); } - return cmdRun(file, opts); + return verb === 'run' ? cmdRun(file, opts) : cmdPublish(file, opts); } case 'inspect': if (positionals.length === 0) { diff --git a/tools/deploy/package.json b/tools/deploy/package.json index bfba8ed7..0bbcd83a 100644 --- a/tools/deploy/package.json +++ b/tools/deploy/package.json @@ -1,10 +1,7 @@ { "dependencies": { "@types/node": "^24.10.7", - "@types/tar-stream": "^2.2.2", - "base32-encode": "^2.0.0", - "json5": "^2.2.3", - "tar-stream": "^3.0.0" + "base32-encode": "^2.0.0" }, "scripts": { "test": "node --test test/*.test.ts" diff --git a/tools/deploy/progress.ts b/tools/deploy/progress.ts deleted file mode 100644 index 18b14f04..00000000 --- a/tools/deploy/progress.ts +++ /dev/null @@ -1,106 +0,0 @@ -// A one-line progress bar for the publishing step of a deploy, which is -// otherwise a long silent wait: thousands of files being packed into an -// upload or copied into a staging tree, with nothing on the terminal between -// the line naming the command and the line after it finishes. -// -// Only drawn when stderr is a terminal. Piped or under CI a redraw per file -// is noise in the log, and the deploy's own stdout lines are the record of -// what happened; those are printed either way, and never carry the bar. - -let REDRAW_MS = 100; -// Below this a bar is more misleading than a bare count. -let MIN_BAR = 10; - -// The parts of a terminal a bar uses, so a test can hand it one. -export type Term = { - isTTY?: boolean | undefined, - columns?: number | undefined, - write(s: string): unknown, -}; - -export class Progress { - #label: string; - #total: number; - #term: Term; - #done: number; - #drawn: boolean; - #lastDraw: number; - - constructor(label: string, total: number, term: Term = process.stderr) { - this.#label = label; - this.#total = total; - this.#term = term; - this.#done = 0; - this.#drawn = false; - this.#lastDraw = 0; - if (this.#interactive) { - this.#draw(); - } - } - - get #interactive(): boolean { - return this.#term.isTTY === true && this.#total > 0; - } - - tick(n = 1) { - this.#done = Math.min(this.#done + n, this.#total); - if (!this.#interactive) { - return; - } - // The line is the bar's only while there is progress left to show. - // What the command has to say about the rest of the wait -- it is - // still running, with the terminal to itself -- lands on a clean line - // instead of on the end of a finished bar. - if (this.#done === this.#total) { - this.finish(); - return; - } - // Redraw on a timer rather than per file: a deploy ticks tens of - // thousands of times. - if (Date.now() - this.#lastDraw < REDRAW_MS) { - return; - } - this.#draw(); - } - - // Leave the line as the deploy found it: the bar is a live thing, and - // what stays in the scrollback is the deploy's own output. - finish() { - if (this.#drawn) { - this.#term.write('\x1b[2K\r'); - this.#drawn = false; - } - } - - #draw() { - this.#lastDraw = Date.now(); - let columns = this.#term.columns || 80; - // Pad the count so the bar keeps its width as the numbers grow into - // each other's columns. - let counts = `${String(this.#done).padStart(String(this.#total).length)}/${this.#total}`; - // label, a space, '[', ']', a space, counts, and a column to spare. - let width = columns - this.#label.length - counts.length - 5; - let line; - if (width < MIN_BAR) { - line = `${this.#label} ${counts}`; - } else { - let filled = Math.round(width * this.#done / this.#total); - line = `${this.#label} [${'#'.repeat(filled)}${'-'.repeat(width - filled)}] ${counts}`; - } - // One column short of the width: writing the last one wraps to the - // next line on some terminals, which leaves the bar behind. - this.#term.write(`\x1b[2K\r${line.slice(0, columns - 1)}`); - this.#drawn = true; - } -} - -// Draw a bar over `body`'s work, and leave the line clean however it goes. -export async function withProgress(label: string, total: number, - body: (tick: () => void) => Promise): Promise { - let bar = new Progress(label, total); - try { - return await body(() => bar.tick()); - } finally { - bar.finish(); - } -} diff --git a/tools/deploy/protocol.ts b/tools/deploy/protocol.ts deleted file mode 100644 index 293dd4a2..00000000 --- a/tools/deploy/protocol.ts +++ /dev/null @@ -1,242 +0,0 @@ - -// What a deploy says to `smogonctl assets upload` before it sends anything, and -// how it reads the answer. -// -// Every published name carries a content hash and the served tree is add-only, so -// a name the home already has is a file that needn't travel. The deploy therefore -// opens the stream with a manifest -- every name with its size, and every link -- -// and the command answers with the positions of the ones it hasn't got. Only those, -// plus the __meta/ files, are then tarred into the same stdin. The same connection -// and the same command line as a bare tar; the tar is just shorter. -// -// The format is binary and every string is length prefixed, so a name is whatever -// bytes a tar would carry for it and nothing in the framing could be mistaken for -// one. The server side spells it out in smogonctl's assets command; this is the -// other half, and the two are tested against each other by shape rather than -// sharing code. -// -// "smogonctl assets" u32 version=1 u32 count -// count times: u8 kind -// kind 1 (asset) u64 size u32 len path -// kind 2 (metadata) u64 size u32 len path (always sent) -// kind 3 (link) u32 len path u32 len linkname -// -// The reply is the same header with the count of what's wanted, then that many u32 -// positions into the manifest. Whatever the command prints after that is its own -// report, relayed as is. - -import type {ChildProcess} from 'node:child_process'; -import * as fs from 'node:fs/promises'; - -import {type ActionQueue} from './queue.ts'; - -export let MAGIC = Buffer.from('smogonctl assets'); -export let VERSION = 1; -let HEADER_SIZE = MAGIC.length + 4 + 4; -let ASSET = 1; -let METAFILE = 2; -let LINK = 3; -let META = '__meta'; - -export type Manifest = { - bytes: Buffer, - // The published name at each position: what a position in the reply means. - entries: string[], - // Positions that are assets, the only ones a reply may name. - assets: Set, - // Names the tar carries whatever the reply says. - metaFiles: Set, -}; - -function isMeta(dst: string): boolean { - return dst === META || dst.startsWith(`${META}/`); -} - -function u32(n: number): Buffer { - let b = Buffer.alloc(4); - b.writeUInt32BE(n); - return b; -} - -function u64(n: number): Buffer { - let b = Buffer.alloc(8); - b.writeBigUInt64BE(BigInt(n)); - return b; -} - -function str(s: string): Buffer { - let raw = Buffer.from(s); - return Buffer.concat([u32(raw.length), raw]); -} - -// The manifest for what `filter` selects of the queue, in queue order: the order -// the tar's entries would come in, and the order the reply's positions index. -export async function manifestOf(aq: ActionQueue, filter?: (dst: string) => boolean): Promise { - let parts: Buffer[] = []; - let entries: string[] = []; - let assets = new Set(); - let metaFiles = new Set(); - for (let entry of aq.log) { - if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) { - continue; - } - let op = entry.op; - let index = entries.length; - entries.push(entry.dst); - if (op.type === 'Symlink') { - parts.push(Buffer.from([LINK]), str(entry.dst), str(op.target)); - continue; - } - // The size the tar header will carry: a Write's data goes in as utf-8, the - // same way tar-stream takes the string. - let size = op.type === 'Copy' ? (await fs.stat(op.src)).size : Buffer.byteLength(op.data); - if (isMeta(entry.dst)) { - metaFiles.add(entry.dst); - parts.push(Buffer.from([METAFILE])); - } else { - assets.add(index); - parts.push(Buffer.from([ASSET])); - } - parts.push(u64(size), str(entry.dst)); - } - let bytes = Buffer.concat([MAGIC, u32(VERSION), u32(entries.length), ...parts]); - return {bytes, entries, assets, metaFiles}; -} - -export type Outcome = 'pending' | 'ok' | 'refused'; - -// The reply, read as it arrives. Strict from the first byte: the command's stdout -// is either a reply followed by its report, or it is a refusal, and a refusal is -// relayed whole. Nothing that isn't the reply is skipped over to find one, since -// anything printing on that stream ahead of the command is something the deploy -// shouldn't be talking through -- a shell profile, a filter -- and finding out is -// better than working around it. -export class Reply { - outcome: Outcome = 'pending'; - // Why a refusal was ours rather than the command's, or null where the command - // said so itself and its words were relayed. - error: string | null = null; - wanted = new Set(); - #manifest: Manifest; - #sink: (chunk: Buffer) => void; - #buf = Buffer.alloc(0); - #count = -1; - #seen = new Set(); - - constructor(manifest: Manifest, sink: (chunk: Buffer) => void) { - this.#manifest = manifest; - this.#sink = sink; - } - - feed(chunk: Buffer): void { - if (this.outcome !== 'pending') { - this.#sink(chunk); - return; - } - this.#buf = Buffer.concat([this.#buf, chunk]); - // Refused as early as the bytes disagree, not once there are enough of - // them: a one-line message is shorter than a header. - let head = this.#buf.subarray(0, MAGIC.length); - if (!head.equals(MAGIC.subarray(0, head.length))) { - this.#refuse('the command didn\'t answer the manifest'); - return; - } - if (this.#buf.length < HEADER_SIZE) { - return; - } - if (this.#count < 0) { - let version = this.#buf.readUInt32BE(MAGIC.length); - if (version !== VERSION) { - this.#refuse(`the command answered with version ${version}, and this speaks ${VERSION}`); - return; - } - this.#count = this.#buf.readUInt32BE(MAGIC.length + 4); - } - let offset = HEADER_SIZE + 4 * this.#seen.size; - while (this.#seen.size < this.#count && this.#buf.length >= offset + 4) { - let index = this.#buf.readUInt32BE(offset); - offset += 4; - let dst = this.#manifest.entries[index]; - if (dst === undefined || !this.#manifest.assets.has(index)) { - this.#refuse(`the command asked for position ${index}, which isn't an asset in the manifest`); - return; - } - if (this.#seen.has(index)) { - this.#refuse(`the command asked for ${dst} twice`); - return; - } - this.#seen.add(index); - this.wanted.add(dst); - } - if (this.#seen.size === this.#count) { - this.outcome = 'ok'; - let rest = this.#buf.subarray(offset); - this.#buf = Buffer.alloc(0); - if (rest.length > 0) { - this.#sink(rest); - } - } - } - - // The stream closed. A reply still being read is a command that went away - // without finishing its answer. - end(): void { - if (this.outcome === 'pending') { - this.#refuse('the command hung up before answering the manifest'); - } - } - - #refuse(error: string): void { - this.outcome = 'refused'; - this.error = error; - let held = this.#buf; - this.#buf = Buffer.alloc(0); - if (held.length > 0) { - this.#sink(held); - } - } -} - -// Send the manifest and read the reply: the names to tar, or null where the -// command refused. On a refusal stdin is closed, so a command waiting for a tar -// sees the end of the stream rather than a deploy waiting for it; everything it -// printed is relayed to `sink` either way, and goes on being relayed after. -export function negotiate(child: ChildProcess, manifest: Manifest, - sink: (chunk: Buffer) => void = chunk => { process.stdout.write(chunk); }) - : Promise<{wanted: Set} | {error: string | null}> { - let stdin = child.stdin; - let stdout = child.stdout; - if (stdin === null || stdout === null) { - throw new Error('negotiate needs the command\'s stdin and stdout piped'); - } - return new Promise(resolve => { - let reply = new Reply(manifest, sink); - let settled = false; - let settle = () => { - if (settled || reply.outcome === 'pending') { - return; - } - settled = true; - if (reply.outcome === 'refused') { - stdin.end(); - resolve({error: reply.error}); - } else { - resolve({wanted: reply.wanted}); - } - }; - // Attached before the manifest goes out and kept for the whole run: the - // reply can be longer than a pipe holds, and the report after it has to - // land somewhere while the tar is going the other way. - stdout.on('data', (chunk: Buffer) => { - reply.feed(chunk); - settle(); - }); - stdout.on('end', () => { - reply.end(); - settle(); - }); - // A command that dies on the manifest closes the pipe under this write; - // its exit code is the report, not the EPIPE. - stdin.write(manifest.bytes, () => {}); - }); -} diff --git a/tools/deploy/queue.ts b/tools/deploy/queue.ts index 693d9c1c..d54d77d3 100644 --- a/tools/deploy/queue.ts +++ b/tools/deploy/queue.ts @@ -1,8 +1,8 @@ import * as fs from 'node:fs/promises'; -import {createWriteStream} from 'node:fs'; import * as nodePath from 'node:path'; -import tar from 'tar-stream'; + +import {BuildError} from '../build/errors.ts'; type Op = { type: 'Write', @@ -31,6 +31,15 @@ type DebugEntry = { export type LogEntry = OpEntry | DebugEntry; +// The output root a publish keeps out of the served tree. +export let META = '__meta'; + +export type Published = { + files: number, + placed: number, + kept: number, +}; + // What a link names, resolved lexically against the directory it sits in: the // tree it describes isn't on disk anywhere yet, so there is nothing else to // resolve against. Null where it walks out of the tree, or names the tree @@ -52,6 +61,29 @@ function resolveLink(dst: string, target: string): string | null { return parts.length === 0 ? null : parts.join('/'); } +// The path under META, or null for a dst outside it. +function metaRest(dst: string): string | null { + if (dst === META) { + return ''; + } + return dst.startsWith(`${META}/`) ? dst.slice(META.length + 1) : null; +} + +// Write-then-rename: a served name is cached forever, so nothing may ever +// read it half written. +async function place(dst: string, write: (tmp: string) => Promise): Promise { + await fs.mkdir(nodePath.dirname(dst), {recursive: true}); + let tmp = nodePath.join(nodePath.dirname(dst), + `.${nodePath.basename(dst)}.${process.pid}.${Math.random().toString(36).slice(2)}`); + try { + await write(tmp); + await fs.rename(tmp, dst); + } catch (err) { + await fs.rm(tmp, {force: true}); + throw err; + } +} + export class ActionQueue { #seen: Map; // Have an accessor for this in the future? idk @@ -93,6 +125,9 @@ export class ActionQueue { if (nodePath.isAbsolute(dst)) { this.valid = false; entry.valid = 'Absolute'; + } else if (dst === '..' || dst.startsWith('../')) { + this.valid = false; + entry.valid = 'Escapes'; } else { let lastEntry = this.#seen.get(dst); if (lastEntry === undefined) { @@ -116,8 +151,7 @@ export class ActionQueue { this.#pushOp({type: 'Write', data}, dst); } - // A link the receiving side extracts as root, so its target may only name - // something else in this tree. + // A link's target may only name something else in this tree. symlink(target: string, dst: string) { let entry = this.#pushOp({type: 'Symlink', target}, dst); if (nodePath.isAbsolute(target) || resolveLink(entry.dst, target) === null) { @@ -163,86 +197,144 @@ export class ActionQueue { } } - // `onFile` is called once per published name as it lands, for progress. - async run(dir: string, mode: 'link' | 'copy' | 'tar', filter?: (dst: string) => boolean, - onFile?: () => void) { + #ops(): OpEntry[] { if (!this.valid) throw new Error(`Invalid ActionQueue`); - if (mode !== 'tar') { - for (let entry of this.log) { - if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) - continue; - let op = entry.op; - let dst = nodePath.join(dir, entry.dst); - await fs.mkdir(nodePath.dirname(dst), {recursive: true}); - if (op.type === 'Copy'){ - // Read-only sources are CAS objects; their mode must not - // leak into deploy trees (rsync -a would ship it), and a - // hardlink cannot carry its own mode, so copy those. - if (mode === 'link' && ((await fs.stat(op.src)).mode & 0o200) !== 0) { - await fs.link(op.src, dst); - } else { - await fs.copyFile(op.src, dst); - await fs.chmod(dst, 0o644); - } - } else if (op.type === 'Write') { - await fs.writeFile(dst, op.data); - } else if (op.type === 'Symlink') { - // A link and not the file it names, in every mode: what - // the tar carries is what a directory has to hold too. - // symlink() has no truncating open behind it, so the name - // is cleared first to keep a rerun over an existing tree - // working the way copy and write already do. - await fs.rm(dst, {force: true}); - await fs.symlink(op.target, dst); + return this.log.filter(entry => entry.type === 'Op'); + } + + async run(dir: string, mode: 'link' | 'copy', filter?: (dst: string) => boolean) { + for (let entry of this.#ops()) { + if (filter !== undefined && !filter(entry.dst)) + continue; + let op = entry.op; + let dst = nodePath.join(dir, entry.dst); + await fs.mkdir(nodePath.dirname(dst), {recursive: true}); + if (op.type === 'Copy'){ + // Read-only sources are CAS objects; their mode must not + // leak into deploy trees (rsync -a would ship it), and a + // hardlink cannot carry its own mode, so copy those. + if (mode === 'link' && ((await fs.stat(op.src)).mode & 0o200) !== 0) { + await fs.link(op.src, dst); + } else { + await fs.copyFile(op.src, dst); + await fs.chmod(dst, 0o644); } - onFile?.(); + } else if (op.type === 'Write') { + await fs.writeFile(dst, op.data); + } else if (op.type === 'Symlink') { + // symlink() has no truncating open behind it, so the name + // is cleared first to keep a rerun over an existing tree + // working the way copy and write already do. + await fs.rm(dst, {force: true}); + await fs.symlink(op.target, dst); } - } else { - // In this case, I guess its a file rather than a dir. - let out = createWriteStream(dir); - (await this.pack(filter, onFile)).pipe(out); - return new Promise((resolve, reject) => { - out.on('error', reject); - out.on('finish', () => resolve()); - }) } } - // Entries are handed to the pack as fast as they can be read, but each - // one's callback waits on the pack draining into whoever is reading it, - // so `onFile` tracks what the consumer has taken rather than what has - // been queued for it. - async pack(filter?: (dst: string) => boolean, onFile?: () => void) - : Promise { - if (!this.valid) - throw new Error(`Invalid ActionQueue`); - let t = tar.pack(); - // An entry's callback also carries the error the consumer is already - // reporting; count the entry only where it made it through. - let took = onFile === undefined ? undefined : (err?: Error | null) => { - if (err === null || err === undefined) { - onFile(); - } + // Into a served tree and the directory beside it holding what isn't + // served. The tree is add-only: every name in it carries a content hash + // and is cached forever, so a name already there is the same file and is + // touched, and one there at another size is refused. `meta` is this + // publish's own and is written whole; it gets `__files`, the tree names + // this publish stands behind, and `__digest`, the caller's digest of + // everything published. + async publish(tree: string, meta: string, digest: string): Promise { + let ops = this.#ops(); + let dsts = new Set(ops.map(entry => entry.dst)); + let files: string[] = []; + let placed = 0; + let kept = 0; + + let landed = (dst: string): string => { + let rest = metaRest(dst); + return rest === null ? nodePath.join(tree, dst) : nodePath.join(meta, rest); }; - for (let entry of this.log) { - if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) - continue; - let op = entry.op; - if (op.type === 'Symlink') { - t.entry({name: entry.dst, type: 'symlink', linkname: op.target}, took) - .on('error', () => {}); + + for (let entry of ops) { + let rest = metaRest(entry.dst); + if (rest === '' || rest?.split('/').some(part => part.startsWith('__'))) { + throw new BuildError(`${META} names starting with __ are the publish's own: ${entry.dst}`); + } + if (entry.op.type === 'Symlink') { + if (rest === null) { + throw new BuildError(`A served tree holds files, not links: ${entry.dst}`); + } + // The validity check already settled that it resolves. + let target = resolveLink(entry.dst, entry.op.target)!; + if (!dsts.has(target)) { + throw new BuildError(`A link to something this publish doesn't carry: ${entry.dst} -> ${target}`); + } + let dst = landed(entry.dst); + await fs.mkdir(nodePath.dirname(dst), {recursive: true}); + await fs.rm(dst, {force: true}); + await fs.symlink(nodePath.relative(nodePath.dirname(dst), landed(target)), dst); continue; } - let data = op.type === 'Copy' ? await fs.readFile(op.src) : op.data; - // A dying consumer destroys the pack and every pending entry - // sink, and each sink emits the error; the consumer is the one - // reporting the failure, so keep the sinks quiet. - t.entry({name: entry.dst}, data, took).on('error', () => {}); + let op = entry.op; + let dst = landed(entry.dst); + if (rest !== null) { + await fs.mkdir(nodePath.dirname(dst), {recursive: true}); + if (op.type === 'Copy') { + await fs.copyFile(op.src, dst); + await fs.chmod(dst, 0o644); + } else { + await fs.writeFile(dst, op.data); + } + continue; + } + files.push(entry.dst); + let size = op.type === 'Copy' ? (await fs.stat(op.src)).size : Buffer.byteLength(op.data); + let have = await fs.stat(dst).catch((err: NodeJS.ErrnoException) => { + if (err.code === 'ENOENT') { + return null; + } + throw err; + }); + if (have !== null) { + if (have.size !== size) { + throw new BuildError(`Different file size: ${entry.dst} is already in the tree holding ` + + `something else, and a name in there is served as if it can never change.`); + } + let now = new Date(); + await fs.utimes(dst, now, now); + kept++; + continue; + } + if (op.type === 'Copy') { + // A CAS object is read-only and never changes, so the tree can + // share it; a source file is copied, since the checkout it + // sits in moves on. + let src = op.src; + let linked = false; + if (((await fs.stat(src)).mode & 0o200) === 0) { + await fs.mkdir(nodePath.dirname(dst), {recursive: true}); + try { + await fs.link(src, dst); + linked = true; + } catch (err) { + // Another filesystem: a copy, like a source + if ((err as NodeJS.ErrnoException).code !== 'EXDEV') { + throw err; + } + } + } + if (!linked) { + await place(dst, async tmp => { + await fs.copyFile(src, tmp); + await fs.chmod(tmp, 0o644); + }); + } + } else { + let data = op.data; + await place(dst, tmp => fs.writeFile(tmp, data, {mode: 0o644})); + } + placed++; } - // Without this the archive has no end-of-archive marker, and strict - // readers (Python tarfile in stream mode) die on the truncation. - t.finalize(); - return t; + + await fs.mkdir(meta, {recursive: true}); + await fs.writeFile(nodePath.join(meta, '__files'), files.sort().map(f => `${f}\n`).join('')); + await fs.writeFile(nodePath.join(meta, '__digest'), `${digest}\n`); + return {files: files.length, placed, kept}; } } diff --git a/tools/deploy/test/api.test.ts b/tools/deploy/test/api.test.ts index a39338ef..db9a867f 100644 --- a/tools/deploy/test/api.test.ts +++ b/tools/deploy/test/api.test.ts @@ -4,16 +4,15 @@ import {createHash} from 'node:crypto'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as pathlib from 'node:path'; -import * as stream from 'node:stream'; import {beforeEach, test} from 'node:test'; import b32encode from 'base32-encode'; -import tar from 'tar-stream'; import {resetDecls, rule} from '../../build/artifact.ts'; import {casPath} from '../../build/cas.ts'; import {makeCtx} from '../api.ts'; import {ActionQueue} from '../queue.ts'; +import {BuildError} from '../../build/errors.ts'; beforeEach(resetDecls); @@ -33,6 +32,7 @@ function makeArtifact(casDir: string, content: string, ext: string) { let obj = casPath(casDir, digest, ext); fs.mkdirSync(pathlib.dirname(obj), {recursive: true}); fs.writeFileSync(obj, content); + fs.chmodSync(obj, 0o444); return artifact; } @@ -89,91 +89,16 @@ test('ctx.list sorts, parses extensions, skips dotfiles and directories', async ]); }); -type PackedEntry = {name: string, data: string, type?: string, linkname?: string}; - -async function packedEntries(aq: ActionQueue, filter?: (dst: string) => boolean) - : Promise { - let packed = await aq.pack(filter); - return new Promise((resolve, reject) => { - let extract = tar.extract(); - let entries: PackedEntry[] = []; - extract.on('entry', (header, stream, next) => { - let chunks: Buffer[] = []; - stream.on('data', c => chunks.push(c)); - stream.on('end', () => { - let entry: PackedEntry = {name: header.name, data: Buffer.concat(chunks).toString()}; - if (header.type === 'symlink') { - entry.type = header.type; - entry.linkname = header.linkname ?? undefined; - } - entries.push(entry); - next(); - }); - }); - extract.on('finish', () => resolve(entries)); - extract.on('error', reject); - packed.pipe(extract); - }); -} - -test('pack preserves op order', async () => { - let aq = new ActionQueue(); - aq.write('zzz', 'z.txt'); - aq.write('aaa', 'a.txt'); - assert.deepEqual(await packedEntries(aq), [ - {name: 'z.txt', data: 'zzz'}, - {name: 'a.txt', data: 'aaa'}, - ]); -}); - -test('pack with a filter packs only matching entries in order', async () => { - let aq = new ActionQueue(); - aq.write('1', 'xy/a.png'); - aq.write('2', 'meta/m.json'); - aq.write('3', 'xy/b.png'); - let subset = await packedEntries(aq, dst => dst.startsWith('xy/')); - assert.deepEqual(subset, [ - {name: 'xy/a.png', data: '1'}, - {name: 'xy/b.png', data: '3'}, - ]); -}); - -test('pack counts an entry for the progress bar as the consumer takes it', async () => { - let aq = new ActionQueue(); - for (let i = 0; i < 4; i++) { - aq.write('x'.repeat(50_000), `f${i}.txt`); - } - let counted = 0; - let packed = await aq.pack(undefined, () => counted++); - // Nothing has read the pack yet: an entry queued is not an entry shipped. - assert.equal(counted, 0); - let sink = new stream.Writable({highWaterMark: 1, write: (_c, _e, cb) => setImmediate(cb)}); - await new Promise((resolve, reject) => { - sink.on('finish', resolve); - sink.on('error', reject); - packed.pipe(sink); - }); - assert.equal(counted, 4); -}); - -test('run counts each materialized entry for the progress bar, filter and all', async () => { - let dir = tmpdir(); - let aq = new ActionQueue(); - aq.write('1', 'ani/a.gif'); - aq.symlink('a.gif', 'ani/b.gif'); - aq.write('2', 'dex/c.png'); - let counted: number[] = []; - await aq.run(pathlib.join(dir, 'deploy'), 'copy', dst => dst.startsWith('ani/'), - () => counted.push(counted.length + 1)); - assert.deepEqual(counted, [1, 2]); -}); - test('duplicate and absolute destinations invalidate the queue', async () => { let dup = new ActionQueue(); dup.write('a', 'x.txt'); dup.write('b', 'x.txt'); assert.ok(!dup.valid); - await assert.rejects(() => dup.pack(), /Invalid ActionQueue/); + await assert.rejects(() => dup.run(tmpdir(), 'copy'), /Invalid ActionQueue/); + + let up = new ActionQueue(); + up.write('a', 'x/../../y.txt'); + assert.ok(!up.valid); let abs = new ActionQueue(); abs.write('a', '/etc/passwd'); @@ -203,16 +128,6 @@ test('copy-mode materialization restores 0644 on read-only sources', async () => assert.equal(fs.statSync(pathlib.join(out, 'out/x.png')).mode & 0o777, 0o644); }); -test('pack carries a symlink as a link, not the bytes it names', async () => { - let aq = new ActionQueue(); - aq.write('1', 'links/xy/a-XXXX.gif'); - aq.symlink('a-XXXX.gif', 'links/xy/a.gif'); - assert.deepEqual(await packedEntries(aq), [ - {name: 'links/xy/a-XXXX.gif', data: '1'}, - {name: 'links/xy/a.gif', data: '', type: 'symlink', linkname: 'a-XXXX.gif'}, - ]); -}); - test('run materializes a symlink resolving beside its target', async () => { let dir = tmpdir(); let aq = new ActionQueue(); @@ -262,3 +177,75 @@ test('ctx.symlink writes the path from the link to what it names', async () => { '../sprites/a-XXXX.gif', ]); }); + +// A tree and a meta directory under one scratch root, with a CAS beside them. +function publishDirs() { + let dir = tmpdir(); + return {dir, tree: pathlib.join(dir, 'tree'), meta: pathlib.join(dir, 'meta'), cas: pathlib.join(dir, 'cas')}; +} + +test('publish places the tree, retargets links into it and records what it stands behind', async () => { + let {dir, tree, meta, cas} = publishDirs(); + let art = makeArtifact(cas, 'gif bytes', 'gif'); + let src = pathlib.join(dir, 'src.png'); + fs.writeFileSync(src, 'png bytes'); + let aq = new ActionQueue(); + let ctx = makeCtx(cas, aq); + ctx.copy(art, 'xy/a-HASH.gif'); + ctx.copy(src, 'xy/b-HASH.png'); + ctx.write('spritesheet-HASH.css', 'css'); + ctx.write('__meta/xy/manifest.json', '{}'); + ctx.symlink('__meta/links/xy/a.gif', 'xy/a-HASH.gif'); + ctx.symlink('__meta/other.json', '__meta/xy/manifest.json'); + assert.deepEqual(await aq.publish(tree, meta, 'd1g3st'), {files: 3, placed: 3, kept: 0}); + + // The CAS object is shared, the source is copied, the write is a file + assert.equal(fs.statSync(pathlib.join(tree, 'xy/a-HASH.gif')).ino, fs.statSync(casPath(cas, art.hash, 'gif')).ino); + assert.notEqual(fs.statSync(pathlib.join(tree, 'xy/b-HASH.png')).ino, fs.statSync(src).ino); + assert.equal(fs.statSync(pathlib.join(tree, 'xy/b-HASH.png')).mode & 0o777, 0o644); + assert.equal(fs.readFileSync(pathlib.join(tree, 'spritesheet-HASH.css'), 'utf8'), 'css'); + assert.equal(fs.readFileSync(pathlib.join(meta, 'xy/manifest.json'), 'utf8'), '{}'); + // Both ends landed apart; the link follows the file into the tree + let link = pathlib.join(meta, 'links/xy/a.gif'); + assert.equal(fs.readlinkSync(link), '../../../tree/xy/a-HASH.gif'); + assert.equal(fs.readFileSync(link, 'utf8'), 'gif bytes'); + assert.equal(fs.readFileSync(pathlib.join(meta, 'other.json'), 'utf8'), '{}'); + assert.equal(fs.readFileSync(pathlib.join(meta, '__files'), 'utf8'), + 'spritesheet-HASH.css\nxy/a-HASH.gif\nxy/b-HASH.png\n'); + assert.equal(fs.readFileSync(pathlib.join(meta, '__digest'), 'utf8'), 'd1g3st\n'); + assert.ok(!fs.readdirSync(pathlib.join(tree, 'xy')).some(name => name.startsWith('.')), 'no temp files left'); +}); + +test('publish keeps what the tree has and refuses a name holding something else', async () => { + let {tree, meta, cas} = publishDirs(); + fs.mkdirSync(pathlib.join(tree, 'xy'), {recursive: true}); + fs.writeFileSync(pathlib.join(tree, 'xy/a-HASH.gif'), 'same size'); + let old = new Date(2020, 0, 1); + fs.utimesSync(pathlib.join(tree, 'xy/a-HASH.gif'), old, old); + let aq = new ActionQueue(); + aq.write('SAME SIZE', 'xy/a-HASH.gif'); + assert.deepEqual(await aq.publish(tree, meta, 'd'), {files: 1, placed: 0, kept: 1}); + assert.equal(fs.readFileSync(pathlib.join(tree, 'xy/a-HASH.gif'), 'utf8'), 'same size', 'not rewritten'); + assert.ok(fs.statSync(pathlib.join(tree, 'xy/a-HASH.gif')).mtime > old, 'touched'); + + let other = new ActionQueue(); + other.write('a different length', 'xy/a-HASH.gif'); + await assert.rejects(() => other.publish(tree, pathlib.join(meta, '2'), 'd'), /Different file size/); + void cas; +}); + +test('publish refuses links in the tree, links to what it does not carry, and __ meta names', async () => { + let {tree, meta} = publishDirs(); + let inTree = new ActionQueue(); + inTree.write('a', 'xy/a-HASH.gif'); + inTree.symlink('a-HASH.gif', 'xy/a.gif'); + await assert.rejects(() => inTree.publish(tree, meta, 'd'), BuildError); + + let dangling = new ActionQueue(); + dangling.symlink('../../xy/a-HASH.gif', '__meta/links/xy/a.gif'); + await assert.rejects(() => dangling.publish(tree, meta, 'd'), /doesn't carry/); + + let reserved = new ActionQueue(); + reserved.write('a', '__meta/__files'); + await assert.rejects(() => reserved.publish(tree, meta, 'd'), /publish's own/); +}); diff --git a/tools/deploy/test/config.test.ts b/tools/deploy/test/config.test.ts deleted file mode 100644 index fe882a0f..00000000 --- a/tools/deploy/test/config.test.ts +++ /dev/null @@ -1,75 +0,0 @@ - -import assert from 'node:assert/strict'; -import * as fs from 'node:fs'; -import * as os from 'node:os'; -import * as pathlib from 'node:path'; -import {test} from 'node:test'; - -import {loadDeployConfig, matchSubsets} from '../config.ts'; - -function configFile(text: string): string { - let dir = fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-config-test-')); - let p = pathlib.join(dir, 'deploy.json5'); - fs.writeFileSync(p, text); - return p; -} - -test('loadDeployConfig parses json5 with comments and trailing commas', async () => { - let config = await loadDeployConfig(configFile(`{ - // dex assets - assets: { - buildFile: "assets.build.ts", - deploy: [ - {subset: ["**"], cmd: "cat > /dev/null"}, - ], - }, - }`)); - assert.deepEqual([...config.keys()], ['assets']); - assert.deepEqual(config.get('assets'), { - buildFile: 'assets.build.ts', - deploy: [{subset: ['**'], cmd: 'cat > /dev/null'}], - }); -}); - -test('loadDeployConfig ties the dir flag to %d in the cmd', async () => { - let dir = await loadDeployConfig(configFile( - '{ps: {buildFile: "ps.build.ts", deploy: [{subset: ["ani/**"], dir: true, cmd: "rsync -a %d/ani/ h:a/"}]}}')); - assert.equal(dir.get('ps')!.deploy[0]!.dir, true); - await assert.rejects(() => loadDeployConfig(configFile( - '{a: {buildFile: "x.build.ts", deploy: [{subset: ["**"], dir: true, cmd: "rsync -a h:a/"}]}}')), - /dir entry's cmd must use %d/); - await assert.rejects(() => loadDeployConfig(configFile( - '{a: {buildFile: "x.build.ts", deploy: [{subset: ["**"], cmd: "rsync -a %d/ h:a/"}]}}')), - /must not use %d \(missing dir: true\?\)/); -}); - -test('loadDeployConfig rejects missing files and malformed shapes', async () => { - await assert.rejects(() => loadDeployConfig('/nonexistent/deploy.json5'), /missing/); - await assert.rejects(() => loadDeployConfig(configFile('[1]')), /must be an object/); - await assert.rejects(() => loadDeployConfig(configFile('{a: {deploy: []}}')), /expected \{buildFile/); - await assert.rejects(() => loadDeployConfig(configFile( - '{a: {buildFile: "x.build.ts", deploy: [{subset: "**", cmd: "c"}]}}')), - /needs \{subset/); - await assert.rejects(() => loadDeployConfig(configFile('{a: {buildFile:')), /deploy.json5:/); -}); - -test('matchSubsets routes dsts to entries, overlap allowed', () => { - let dsts = ['xy/a.gif', 'xy/manifest.json', 'xyicons/b.png']; - let [xy, manifests] = matchSubsets(dsts, [ - {subset: ['xy/**', 'xyicons/**'], cmd: 'one'}, - {subset: ['**/manifest.json'], cmd: 'two'}, - ]); - assert.deepEqual([...xy!.matched].sort(), dsts); - assert.equal(xy!.entry.cmd, 'one'); - assert.deepEqual([...manifests!.matched], ['xy/manifest.json']); -}); - -test('matchSubsets errors on a glob matching nothing', () => { - assert.throws(() => matchSubsets(['xy/a.gif'], [{subset: ['xy/**', 'zz/**'], cmd: 'c'}]), - /matches no outputs: zz\/\*\*/); -}); - -test('matchSubsets errors on uncovered outputs, listing them', () => { - assert.throws(() => matchSubsets(['xy/a.gif', 'stray.txt'], [{subset: ['xy/**'], cmd: 'c'}]), - /not covered by any deploy entry:\n {2}stray.txt/); -}); diff --git a/tools/deploy/test/progress.test.ts b/tools/deploy/test/progress.test.ts deleted file mode 100644 index 55bc884c..00000000 --- a/tools/deploy/test/progress.test.ts +++ /dev/null @@ -1,76 +0,0 @@ - -import assert from 'node:assert/strict'; -import {test} from 'node:test'; - -import {Progress, type Term} from '../progress.ts'; - -function term(overrides: Partial = {}): Term & {written: string[]} { - let written: string[] = []; - return {isTTY: true, columns: 80, write: (s: string) => written.push(s), written, ...overrides}; -} - -// Every write the bar makes, with the clear-and-return prefix stripped. -function lines(t: {written: string[]}): string[] { - return t.written.map(s => s.replace('\x1b[2K\r', '')); -} - -test('a bar draws nothing when stderr is not a terminal', () => { - let t = term({isTTY: false}); - let bar = new Progress('deploy: uploading', 4, t); - for (let i = 0; i < 4; i++) { - bar.tick(); - } - bar.finish(); - assert.deepEqual(t.written, []); -}); - -test('a bar draws its label, a filled share of the width, and the counts', () => { - let t = term(); - let bar = new Progress('up', 100, t); - // The timer holds the redraws back, so drive it to the end and read the - // first draw, which the constructor always makes. - let [first] = lines(t); - assert.equal(first, `up [${'-'.repeat(66)}] 0/100`); - assert.equal(first!.length, 79); - for (let i = 0; i < 50; i++) { - bar.tick(); - } - assert.equal(lines(t).length, 1); -}); - -test('a bar never writes past the terminal width', () => { - for (let columns of [20, 40, 80, 200]) { - let t = term({columns}); - new Progress('deploying something', 1000, t); - for (let line of lines(t)) { - assert.ok(line.length < columns, `${line.length} >= ${columns} at ${columns} columns`); - } - } -}); - -test('a narrow terminal gets the counts without a bar', () => { - let t = term({columns: 24}); - new Progress('up', 1000, t); - assert.deepEqual(lines(t), ['up 0/1000']); -}); - -test('a bar clears its line once it is full, leaving the rest of the wait alone', () => { - let t = term(); - let bar = new Progress('up', 2, t); - bar.tick(); - bar.tick(); - assert.equal(t.written.at(-1), '\x1b[2K\r'); - assert.equal(lines(t).at(-1), ''); - // Already cleared: finishing again is not a second clear. - let count = t.written.length; - bar.finish(); - assert.equal(t.written.length, count); -}); - -test('finishing a bar mid-run clears the line it drew', () => { - let t = term(); - let bar = new Progress('up', 100, t); - bar.tick(); - bar.finish(); - assert.equal(t.written.at(-1), '\x1b[2K\r'); -}); diff --git a/tools/deploy/test/protocol.test.ts b/tools/deploy/test/protocol.test.ts deleted file mode 100644 index 8dfcdf63..00000000 --- a/tools/deploy/test/protocol.test.ts +++ /dev/null @@ -1,212 +0,0 @@ -import assert from 'node:assert/strict'; -import {spawn} from 'node:child_process'; -import * as fs from 'node:fs'; -import * as os from 'node:os'; -import * as pathlib from 'node:path'; -import {test} from 'node:test'; - -import {MAGIC, Reply, manifestOf, negotiate, type Manifest} from '../protocol.ts'; -import {ActionQueue} from '../queue.ts'; - -let here = pathlib.dirname(new URL(import.meta.url).pathname); -let TIMEOUT = 10_000; - -function tmpdir(): string { - return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-protocol-test-')); -} - -// A queue shaped like a small deploy: two stamped assets, one of them named -// awkwardly, a manifest under __meta/ and a link mirroring an asset. -async function sampleQueue(dir: string): Promise<{aq: ActionQueue, manifest: Manifest}> { - let aq = new ActionQueue(); - let src = pathlib.join(dir, 'a.gif'); - fs.writeFileSync(src, 'gif bytes'); - aq.copy(src, 'sprites/xy/a-HASH.gif'); - aq.write('héllo', 'sprites/xy/b c\td-HASH.txt'); - aq.write('{}', '__meta/xy/manifest.json'); - aq.symlink('../../sprites/xy/a-HASH.gif', '__meta/links/xy/a.gif'); - return {aq, manifest: await manifestOf(aq)}; -} - -function u32(n: number): Buffer { - let b = Buffer.alloc(4); - b.writeUInt32BE(n); - return b; -} - -function reply(indices: number[], version = 1): Buffer { - return Buffer.concat([MAGIC, u32(version), u32(indices.length), ...indices.map(u32)]); -} - -test('manifestOf spells the wire format, in queue order, sizes as the tar would carry them', async () => { - let {manifest} = await sampleQueue(tmpdir()); - assert.deepEqual(manifest.entries, - ['sprites/xy/a-HASH.gif', 'sprites/xy/b c\td-HASH.txt', '__meta/xy/manifest.json', '__meta/links/xy/a.gif']); - assert.deepEqual([...manifest.assets], [0, 1]); - assert.deepEqual([...manifest.metaFiles], ['__meta/xy/manifest.json']); - - let b = manifest.bytes; - assert.ok(b.subarray(0, 16).equals(MAGIC)); - assert.equal(b.readUInt32BE(16), 1, 'version'); - assert.equal(b.readUInt32BE(20), 4, 'count'); - let at = 24; - let expectFile = (kind: number, size: number, name: string) => { - assert.equal(b[at], kind, `kind of ${name}`); - assert.equal(Number(b.readBigUInt64BE(at + 1)), size, `size of ${name}`); - let raw = Buffer.from(name); - assert.equal(b.readUInt32BE(at + 9), raw.length, `name length of ${name}`); - assert.ok(b.subarray(at + 13, at + 13 + raw.length).equals(raw), `name of ${name}`); - at += 13 + raw.length; - }; - expectFile(1, 'gif bytes'.length, 'sprites/xy/a-HASH.gif'); - // Six characters, seven bytes: the size is the utf-8's, since that is what - // tar-stream writes, and the tab rides inside the length-prefixed name. - expectFile(1, 6, 'sprites/xy/b c\td-HASH.txt'); - expectFile(2, 2, '__meta/xy/manifest.json'); - assert.equal(b[at], 3, 'a link'); - let name = Buffer.from('__meta/links/xy/a.gif'); - let target = Buffer.from('../../sprites/xy/a-HASH.gif'); - assert.equal(b.readUInt32BE(at + 1), name.length); - assert.ok(b.subarray(at + 5, at + 5 + name.length).equals(name)); - at += 5 + name.length; - assert.equal(b.readUInt32BE(at), target.length); - assert.ok(b.subarray(at + 4, at + 4 + target.length).equals(target)); - assert.equal(at + 4 + target.length, b.length, 'nothing after the last entry'); -}); - -test('manifestOf takes only what the filter selects, positions renumbered', async () => { - let {aq} = await sampleQueue(tmpdir()); - let manifest = await manifestOf(aq, dst => dst.startsWith('__meta/')); - assert.deepEqual(manifest.entries, ['__meta/xy/manifest.json', '__meta/links/xy/a.gif']); - assert.equal(manifest.assets.size, 0); - assert.equal(manifest.bytes.readUInt32BE(20), 2); -}); - -test('Reply reads a reply however it is chunked and relays the report after it', async () => { - let {manifest} = await sampleQueue(tmpdir()); - let bytes = Buffer.concat([reply([1, 0]), Buffer.from('* 2 files\n')]); - for (let step of [bytes.length, 1, 7]) { - let relayed: Buffer[] = []; - let r = new Reply(manifest, chunk => relayed.push(chunk)); - for (let i = 0; i < bytes.length; i += step) { - r.feed(bytes.subarray(i, i + step)); - } - assert.equal(r.outcome, 'ok', `chunked by ${step}`); - assert.deepEqual([...r.wanted].sort(), ['sprites/xy/a-HASH.gif', 'sprites/xy/b c\td-HASH.txt']); - assert.equal(Buffer.concat(relayed).toString(), '* 2 files\n', `report chunked by ${step}`); - } -}); - -test('Reply refuses anything that is not a reply, and relays it whole', async () => { - let {manifest} = await sampleQueue(tmpdir()); - - // Text where the magic should be: a refusal, a shell profile, a filter - let relayed: Buffer[] = []; - let r = new Reply(manifest, chunk => relayed.push(chunk)); - r.feed(Buffer.from('No such asset set: sprites\n')); - assert.equal(r.outcome, 'refused'); - assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites\n'); - // And what comes after goes through too - r.feed(Buffer.from('more\n')); - assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites\nmore\n'); - - // Even where it starts out looking right - r = new Reply(manifest, () => {}); - r.feed(Buffer.from('smogonctl asset')); - assert.equal(r.outcome, 'pending', 'a prefix of the magic is still possible'); - r.feed(Buffer.from('!')); - assert.equal(r.outcome, 'refused'); - - r = new Reply(manifest, () => {}); - r.feed(reply([0], 2)); - assert.equal(r.outcome, 'refused'); - assert.match(r.error!, /version 2/); - - r = new Reply(manifest, () => {}); - r.feed(reply([7])); - assert.equal(r.outcome, 'refused'); - assert.match(r.error!, /position 7/); - - r = new Reply(manifest, () => {}); - r.feed(reply([2])); - assert.equal(r.outcome, 'refused', 'a __meta/ file is not something to ask for'); - - r = new Reply(manifest, () => {}); - r.feed(reply([0, 0])); - assert.equal(r.outcome, 'refused'); - assert.match(r.error!, /twice/); - - // Hung up partway through the header, and partway through the positions - r = new Reply(manifest, () => {}); - r.feed(reply([0, 1]).subarray(0, 10)); - r.end(); - assert.equal(r.outcome, 'refused'); - assert.match(r.error!, /hung up/); - r = new Reply(manifest, () => {}); - r.feed(reply([0, 1]).subarray(0, 26)); - r.end(); - assert.equal(r.outcome, 'refused'); - - // An empty answer is an answer - r = new Reply(manifest, () => {}); - r.feed(reply([])); - assert.equal(r.outcome, 'ok'); - assert.equal(r.wanted.size, 0); -}); - -function run(cmd: string, args: string[]) { - let child = spawn(cmd, args, {stdio: ['pipe', 'pipe', 'inherit']}); - child.stdin!.on('error', () => {}); - let exit = new Promise(resolve => child.on('close', resolve)); - return {child, exit}; -} - -test('negotiate against a command that answers: only what it asked for travels', {timeout: TIMEOUT}, async () => { - let dir = tmpdir(); - let {aq, manifest} = await sampleQueue(dir); - let out = pathlib.join(dir, 'seen.json'); - let {child, exit} = run(process.execPath, [pathlib.join(here, 'servers/answer.ts'), out]); - let relayed: Buffer[] = []; - let result = await negotiate(child, manifest, chunk => relayed.push(chunk)); - assert.ok('wanted' in result, `should be answered, got ${JSON.stringify(result)}`); - // Every other asset: position 0 and not 1 - assert.deepEqual([...result.wanted], ['sprites/xy/a-HASH.gif']); - let send = new Set([...result.wanted, ...manifest.metaFiles]); - (await aq.pack(dst => send.has(dst))).pipe(child.stdin!); - assert.equal(await exit, 0); - let seen = JSON.parse(fs.readFileSync(out, 'utf8')); - assert.deepEqual(seen.manifest.map((e: {name: string}) => e.name), manifest.entries); - assert.deepEqual(seen.received, [ - {name: 'sprites/xy/a-HASH.gif', size: 9}, - {name: '__meta/xy/manifest.json', size: 2}, - ]); - assert.equal(Buffer.concat(relayed).toString(), '* 2 received\n', 'the report after the reply is relayed'); -}); - -test('negotiate against a command that refuses: relayed, stdin closed, its exit code kept', {timeout: TIMEOUT}, async () => { - let {manifest} = await sampleQueue(tmpdir()); - let {child, exit} = run(process.execPath, [pathlib.join(here, 'servers/refuse.ts')]); - let relayed: Buffer[] = []; - let result = await negotiate(child, manifest, chunk => relayed.push(chunk)); - assert.ok('error' in result); - assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites.\n'); - assert.equal(await exit, 3); -}); - -test('negotiate against cat: an echo of the manifest is not a reply', {timeout: TIMEOUT}, async () => { - let {manifest} = await sampleQueue(tmpdir()); - let {child, exit} = run('cat', []); - let result = await negotiate(child, manifest, () => {}); - assert.ok('error' in result, 'the manifest coming back starts with the magic but is no reply'); - // stdin was closed for it, so cat finishes instead of waiting - assert.equal(await exit, 0); -}); - -test('negotiate against a command that says nothing: refused when it hangs up', {timeout: TIMEOUT}, async () => { - let {manifest} = await sampleQueue(tmpdir()); - let {child, exit} = run('true', []); - let result = await negotiate(child, manifest, () => {}); - assert.ok('error' in result); - assert.match(result.error!, /hung up|didn't answer/); - await exit; -}); diff --git a/tools/deploy/test/servers/answer.ts b/tools/deploy/test/servers/answer.ts deleted file mode 100644 index 541a7d6c..00000000 --- a/tools/deploy/test/servers/answer.ts +++ /dev/null @@ -1,94 +0,0 @@ -// A stand-in for `smogonctl assets upload`: reads the manifest, asks for every -// other asset, takes the tar, and writes what it saw to the path in argv[2] as -// JSON. Prints one line of report after the reply, the way the real one does. - -import * as fs from 'node:fs'; -import tar from 'tar-stream'; - -let MAGIC = Buffer.from('smogonctl assets'); -let out = process.argv[2]!; - -let buf = Buffer.alloc(0); -let pending: {n: number, resolve: (b: Buffer) => void} | null = null; -let ended = false; - -process.stdin.on('data', (chunk: Buffer) => { - buf = Buffer.concat([buf, chunk]); - serve(); -}); -process.stdin.on('end', () => { - ended = true; - serve(); -}); - -function serve() { - if (pending !== null && buf.length >= pending.n) { - let {n, resolve} = pending; - pending = null; - let head = buf.subarray(0, n); - buf = buf.subarray(n); - resolve(head); - } else if (pending !== null && ended) { - throw new Error('manifest ended early'); - } -} - -function take(n: number): Promise { - return new Promise(resolve => { - pending = {n, resolve}; - serve(); - }); -} - -async function name(): Promise { - let len = (await take(4)).readUInt32BE(0); - return (await take(len)).toString(); -} - -let header = await take(24); -if (!header.subarray(0, 16).equals(MAGIC)) { - throw new Error('not a manifest'); -} -let count = header.readUInt32BE(20); -let manifest: {kind: number, name: string, size?: number, target?: string}[] = []; -for (let i = 0; i < count; i++) { - let kind = (await take(1))[0]!; - if (kind === 3) { - manifest.push({kind, name: await name(), target: await name()}); - } else { - let size = Number((await take(8)).readBigUInt64BE(0)); - manifest.push({kind, name: await name(), size}); - } -} -let asked = manifest.map((e, i) => i).filter(i => manifest[i]!.kind === 1 && i % 2 === 0); -let reply = Buffer.alloc(24 + 4 * asked.length); -MAGIC.copy(reply); -reply.writeUInt32BE(1, 16); -reply.writeUInt32BE(asked.length, 20); -asked.forEach((index, i) => reply.writeUInt32BE(index, 24 + 4 * i)); -process.stdout.write(reply); - -// Whatever follows is the tar, or nothing at all. -let received: {name: string, size: number}[] = []; -let extract = tar.extract(); -extract.on('entry', (header, stream, next) => { - received.push({name: header.name, size: header.size ?? 0}); - stream.on('end', next); - stream.resume(); -}); -let done = new Promise((resolve, reject) => { - extract.on('finish', resolve); - extract.on('error', reject); -}); -if (buf.length > 0) { - extract.write(buf); -} -buf = Buffer.alloc(0); -if (ended) { - extract.end(); -} else { - process.stdin.pipe(extract); -} -await done; -fs.writeFileSync(out, JSON.stringify({manifest, asked, received})); -process.stdout.write(`* ${received.length} received\n`);