mirror of
https://github.com/smogon/sprites.git
synced 2026-10-24 09:09:23 -05:00
Publish straight into a served tree, and drop the smogonctl upload path
`publish <file> --tree <dir> --meta <dir>` places the outputs itself: the tree is add-only (a name already there is touched, another size is refused), CAS objects are hardlinked in, __meta/ goes to its own directory with the links pointed into the tree, and __files/__digest record the publish. The deploy verb, deploy.json5, the manifest/tar negotiation with `smogonctl assets upload`, the progress bar and `run --tar` go with it; smogon.com's sprites-sync service runs build and publish on the server now. The output root is the served tree, no sprites/ prefix. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TNa8LzaaMo3QbEVuFUtFGU
This commit is contained in:
Christopher Monsanto
1
.gitignore
vendored
1
.gitignore
vendored
@@ -4,4 +4,3 @@ pnpm-debug.log
|
||||
.cache/
|
||||
.build/
|
||||
build.config
|
||||
deploy.json5
|
||||
|
||||
129
README.md
129
README.md
@@ -46,13 +46,13 @@ Using [`brew`](https://brew.sh/) on a macOS:
|
||||
$ brew install imagemagick gifsicle advancecomp optipng pngquant webp
|
||||
```
|
||||
|
||||
## Building and deploying
|
||||
## Building
|
||||
|
||||
Install dependencies once with `pnpm install`.
|
||||
|
||||
Each deploy is a root `*.build.ts` module: it declares its build rules
|
||||
Each output tree is a root `*.build.ts` module: it declares its build rules
|
||||
(shared sets are plain functions in `rules/`; declaring an identical rule
|
||||
twice is a no-op returning the existing artifacts, so any number of deploys
|
||||
twice is a no-op returning the existing artifacts, so any number of modules
|
||||
can call the same set) and, next to each set of rules, a `deploy(ctx => ...)`
|
||||
block that maps the built artifacts to their published names; the blocks run
|
||||
in registration order after the build, sharing one output tree per module. Build outputs are content-addressed:
|
||||
@@ -62,20 +62,19 @@ content, same-byte renames rebuild nothing, and hash-stamped publishing
|
||||
reuses the build's digests. All state lives in `.build/`.
|
||||
|
||||
```
|
||||
$ pnpm build # build every deploy's rules, GC stale state
|
||||
$ pnpm deploy # list the deploys in deploy.json5
|
||||
$ pnpm deploy smogon # run a named deploy
|
||||
$ node tools/deploy/index.ts build smogon.build.ts # build one deploy's rules
|
||||
$ node tools/deploy/index.ts run smogon.build.ts -o deploy/smogon
|
||||
$ pnpm build # build every module's rules, GC stale state
|
||||
$ node tools/deploy/index.ts build smogon.build.ts # build one module's rules
|
||||
$ node tools/deploy/index.ts run smogon.build.ts -o /tmp/out
|
||||
$ node tools/deploy/index.ts publish smogon.build.ts --tree /srv/served --meta /srv/publishes/1
|
||||
$ node tools/deploy/index.ts inspect src/minisprites/items/ileftovers.png -o /tmp/out
|
||||
$ node tools/deploy/index.ts refactor --record # remember what the deploys publish
|
||||
$ node tools/deploy/index.ts refactor --record # remember what the modules publish
|
||||
$ node tools/deploy/index.ts refactor # and what a change did to it
|
||||
```
|
||||
|
||||
`run` materializes a deploy to a directory (`--link` hardlinks, `--tar`
|
||||
writes a tar file) without uploading anything. `inspect` builds every rule
|
||||
that consumes the given source paths and copies the outputs out under
|
||||
readable names for eyeballing.
|
||||
`run` materializes a module's outputs to a directory (`--link` hardlinks)
|
||||
without publishing anything. `inspect` builds every rule that consumes the
|
||||
given source paths and copies the outputs out under readable names for
|
||||
eyeballing.
|
||||
|
||||
`refactor` answers "did that change anything we ship". It builds, runs the
|
||||
deploy blocks, and digests the bytes landing at every published name, then
|
||||
@@ -88,86 +87,40 @@ run it again.
|
||||
Useful flags: `-j <n>` parallelism, `-n` dry run, `-v` verbose,
|
||||
`--fail-fast` stop after the first failure.
|
||||
|
||||
## Deploying
|
||||
A build can be interrupted at any point (SIGINT or SIGTERM: running rules are
|
||||
stopped, a second signal kills them) and picks up where it left off next time.
|
||||
|
||||
`deploy` reads `deploy.json5` at the repo root (not tracked by git). It maps
|
||||
deploy names to a buildFile and a list of (subset, cmd) entries: after
|
||||
building and finishing the buildFile, each entry's globs select a subset of
|
||||
the finish outputs and sends it to the entry's command, which is `smogonctl
|
||||
assets upload` (possibly behind `ssh` and `sudo`). An entry with `dir: true`
|
||||
instead materializes the subset into a temp directory whose path replaces
|
||||
`%d` in the command (for rsync-style transports). Every glob must match
|
||||
something, and every output must be covered by some entry. `deploy <name> -o
|
||||
<dir>` materializes each entry's subset under `<dir>/<name>/<entry index>/`
|
||||
instead of running its command, for eyeballing what would ship.
|
||||
## Publishing
|
||||
|
||||
Sending is a conversation over the command's stdin and stdout rather than a
|
||||
tar poured into it. Every served name carries a content hash and the served
|
||||
tree is add-only, so a name the home already has is a file that needn't
|
||||
travel: the deploy first writes a manifest -- every name with its size, and
|
||||
every `__meta/` link -- and the command answers with the ones it hasn't got.
|
||||
Only those, plus the `__meta/` files, are then tarred into the same stream. A
|
||||
redeploy of an unchanged tree moves kilobytes. The command's stdout has to be
|
||||
the upload's alone and reach the deploy unfiltered: the reply is the first
|
||||
thing on it, and anything else there (a shell profile that prints over ssh, a
|
||||
`| grep`) is taken as a refusal and fails the deploy rather than being
|
||||
skipped past. `run --tar` still writes a bare tar, which the upload also
|
||||
accepts.
|
||||
`publish <file> --tree <dir> --meta <dir>` builds, runs the module's deploy
|
||||
blocks and places the outputs for serving. The output root is the served
|
||||
tree, and `__meta/` at the root is everything that isn't served.
|
||||
|
||||
Where stderr is a terminal, each entry draws a progress bar over the files
|
||||
that travel while they go out -- an entry counts once the command has taken
|
||||
it, not once it has been read off disk -- and the line is cleared again
|
||||
afterwards. Piped or under CI nothing is drawn, and the lines the deploy
|
||||
prints are the same either way.
|
||||
The tree is add-only. Every name in it carries a content hash and is served
|
||||
as if it can never change, so a name already there is the same file and is
|
||||
touched rather than rewritten, and one there at another size fails the
|
||||
publish. New files are hardlinked from the CAS where they can be and written
|
||||
beside their final name and renamed into place otherwise, so nothing ever
|
||||
reads a half-written file.
|
||||
|
||||
This file is not committed, because it is where the hosts and paths this
|
||||
repo ships to are written down.
|
||||
`--meta` is this publish's own directory and is written whole. Three kinds of
|
||||
thing ride there. A manifest, `name` -> the whole url of the stamped file,
|
||||
for a reader that looks one up. A pointer file naming a single url, for the
|
||||
one-file sets. And `__meta/links/`, the served tree recreated as symlinks
|
||||
under the un-stamped names, which is the same mapping said in names instead
|
||||
of a file; it lives out here rather than in the tree because the tree is
|
||||
add-only and a link is repointed on every publish. A link names where in the
|
||||
output its file is (`__meta/links/xy/charizard.gif` names
|
||||
`xy/charizard-<hash>.gif`), and the publish points it at where that file sits
|
||||
in the tree; a link to anything the publish doesn't carry is refused. The
|
||||
publish also writes `__files`, every tree name it stands behind, and
|
||||
`__digest`, a digest of every name and the bytes under it; names under
|
||||
`__meta/` starting with `__` are reserved for these.
|
||||
|
||||
```json5
|
||||
{
|
||||
smogon: {
|
||||
buildFile: "smogon.build.ts",
|
||||
deploy: [
|
||||
{subset: ["**"], cmd: "smogonctl assets upload sprites"},
|
||||
],
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
Note that the coverage rule is per deploy name, over the whole buildFile's
|
||||
outputs: two names on one buildFile can't split its tree between them, since
|
||||
each of them has to cover all of it.
|
||||
|
||||
### The asset upload's tar layout
|
||||
|
||||
`smogonctl assets upload` publishes a tree into a served tree under a prefix
|
||||
named in the receiving home's `services.toml`, which this side can't read. So
|
||||
`smogon.build.ts` writes that prefix itself -- everything served ships under
|
||||
`sprites/` -- and the upload rejects a tar whose tree disagrees. The two are
|
||||
checked against each other instead of each guessing, which is what lets the
|
||||
manifests and pointers in `__meta/` name whole urls (`/__assets/sprites/...`)
|
||||
and their readers hold no configuration at all.
|
||||
|
||||
`__meta/` is the exception and stays at the tar root: the upload diverts it to
|
||||
`assets-meta/`, beside the served tree rather than in it, because a served
|
||||
name carries a content hash and something un-stamped has to say which name to
|
||||
ask for.
|
||||
|
||||
Three kinds of thing ride there. A manifest, `name` -> the whole url of the
|
||||
stamped file, for a reader that looks one up. A pointer file naming a single
|
||||
url, for the one-file sets. And `__meta/links/`, the served tree recreated as
|
||||
symlinks under the un-stamped names, which is the same mapping said in names
|
||||
instead of a file. It lives out here rather than in the tree because the tree
|
||||
is add-only -- a name in it is promised never to change -- and a link is
|
||||
repointed on every upload.
|
||||
|
||||
A link says where in the tar its file is, not what to write into the link:
|
||||
`__meta/links/xy/charizard.gif` names `sprites/xy/charizard-<hash>.gif` and
|
||||
what is packed is the path between the two. The two halves land in different
|
||||
trees on the far side (`assets-meta/<key>/` and `assets/`), so the upload
|
||||
retargets every link at where its asset actually went, and refuses one naming
|
||||
anything that tar didn't carry. The mirror doesn't repeat the `sprites/`
|
||||
prefix, since the directory it lands in is already this set's.
|
||||
Production runs this: smogon.com's `sprites-sync` service fetches this
|
||||
repository, runs `build` and then `publish`, swaps the meta directory in for
|
||||
its readers and sweeps tree files no publish has named for a while. Each
|
||||
checkout's `build.config` is written by whoever made the checkout.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "node tools/deploy/index.ts build",
|
||||
"deploy": "node tools/deploy/index.ts deploy",
|
||||
"check": "tsc --build tsconfig-workspace.json"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
158
pnpm-lock.yaml
generated
158
pnpm-lock.yaml
generated
@@ -45,18 +45,9 @@ importers:
|
||||
'@types/node':
|
||||
specifier: ^24.10.7
|
||||
version: 24.13.3
|
||||
'@types/tar-stream':
|
||||
specifier: ^2.2.2
|
||||
version: 2.2.3
|
||||
base32-encode:
|
||||
specifier: ^2.0.0
|
||||
version: 2.0.0
|
||||
json5:
|
||||
specifier: ^2.2.3
|
||||
version: 2.2.3
|
||||
tar-stream:
|
||||
specifier: ^3.0.0
|
||||
version: 3.2.0
|
||||
|
||||
tools/smogdexspritesheet:
|
||||
dependencies:
|
||||
@@ -90,9 +81,6 @@ packages:
|
||||
'@types/spritesmith@3.4.5':
|
||||
resolution: {integrity: sha512-LRyCIJ+7LW59eFnUVfF4Wwhs23MceEqFQQtV7l7hHYz/z0zeN24J5TJOE2qZjaUOsGcSZ0ISDTIhe2dvpWrnmw==}
|
||||
|
||||
'@types/tar-stream@2.2.3':
|
||||
resolution: {integrity: sha512-if3mugZfjVkXOMZdFjIHySxY13r6GXPpyOlsDmLffvyI7tLz9wXE8BFjNivXsvUeyJ1KNlOpfLnag+ISmxgxPw==}
|
||||
|
||||
'@types/vinyl@2.0.12':
|
||||
resolution: {integrity: sha512-Sr2fYMBUVGYq8kj3UthXFAu5UN6ZW+rYr4NACjZQJvHvj+c8lYv0CahmZ2P/r7iUkN44gGUBwqxZkrKXYPb7cw==}
|
||||
|
||||
@@ -238,51 +226,6 @@ packages:
|
||||
aws4@1.13.2:
|
||||
resolution: {integrity: sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==}
|
||||
|
||||
b4a@1.8.1:
|
||||
resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==}
|
||||
peerDependencies:
|
||||
react-native-b4a: '*'
|
||||
peerDependenciesMeta:
|
||||
react-native-b4a:
|
||||
optional: true
|
||||
|
||||
bare-events@2.9.1:
|
||||
resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==}
|
||||
peerDependencies:
|
||||
bare-abort-controller: '*'
|
||||
peerDependenciesMeta:
|
||||
bare-abort-controller:
|
||||
optional: true
|
||||
|
||||
bare-fs@4.8.0:
|
||||
resolution: {integrity: sha512-fM+MhCvdQhZ7NV6S95a07gPSqjIYKn6mFaXfx266wN3ajZGl/+1AzH+ubkXQ0fFZvOe2nk9VHkzdYkQE5zMV3Q==}
|
||||
engines: {bare: '>=1.28.0'}
|
||||
peerDependencies:
|
||||
bare-buffer: '*'
|
||||
peerDependenciesMeta:
|
||||
bare-buffer:
|
||||
optional: true
|
||||
|
||||
bare-path@3.1.1:
|
||||
resolution: {integrity: sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==}
|
||||
|
||||
bare-stream@2.13.3:
|
||||
resolution: {integrity: sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==}
|
||||
peerDependencies:
|
||||
bare-abort-controller: '*'
|
||||
bare-buffer: '*'
|
||||
bare-events: '*'
|
||||
peerDependenciesMeta:
|
||||
bare-abort-controller:
|
||||
optional: true
|
||||
bare-buffer:
|
||||
optional: true
|
||||
bare-events:
|
||||
optional: true
|
||||
|
||||
bare-url@2.5.2:
|
||||
resolution: {integrity: sha512-L13PCJzKG8RGvx8V1/DdMi12ERhC3tprr7/8a94BxpmnRsFqxh5XZNdhtMxu5HPkRshYOOWRGY8lDP7ZhpG9Cg==}
|
||||
|
||||
base32-encode@2.0.0:
|
||||
resolution: {integrity: sha512-mlmkfc2WqdDtMl/id4qm3A7RjW6jxcbAoMjdRmsPiwQP0ufD4oXItYMnPgVHe80lnAIy+1xwzhHE1s4FoIceSw==}
|
||||
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
|
||||
@@ -338,9 +281,6 @@ packages:
|
||||
ecc-jsbn@0.1.2:
|
||||
resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==}
|
||||
|
||||
events-universal@1.0.1:
|
||||
resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==}
|
||||
|
||||
extend@3.0.2:
|
||||
resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
|
||||
|
||||
@@ -351,9 +291,6 @@ packages:
|
||||
fast-deep-equal@3.1.3:
|
||||
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
|
||||
|
||||
fast-fifo@1.3.2:
|
||||
resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==}
|
||||
|
||||
fast-json-stable-stringify@2.1.0:
|
||||
resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==}
|
||||
|
||||
@@ -433,11 +370,6 @@ packages:
|
||||
json-stringify-safe@5.0.1:
|
||||
resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
|
||||
|
||||
json5@2.2.3:
|
||||
resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
|
||||
engines: {node: '>=6'}
|
||||
hasBin: true
|
||||
|
||||
jsprim@1.4.2:
|
||||
resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==}
|
||||
engines: {node: '>=0.6.0'}
|
||||
@@ -557,9 +489,6 @@ packages:
|
||||
engines: {node: '>=0.10.0'}
|
||||
hasBin: true
|
||||
|
||||
streamx@2.28.0:
|
||||
resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==}
|
||||
|
||||
string_decoder@0.10.31:
|
||||
resolution: {integrity: sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==}
|
||||
|
||||
@@ -574,15 +503,6 @@ packages:
|
||||
resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
tar-stream@3.2.0:
|
||||
resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==}
|
||||
|
||||
teex@1.0.1:
|
||||
resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==}
|
||||
|
||||
text-decoder@1.2.7:
|
||||
resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==}
|
||||
|
||||
through2@2.0.5:
|
||||
resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==}
|
||||
|
||||
@@ -656,10 +576,6 @@ snapshots:
|
||||
dependencies:
|
||||
'@types/vinyl': 2.0.12
|
||||
|
||||
'@types/tar-stream@2.2.3':
|
||||
dependencies:
|
||||
'@types/node': 24.13.3
|
||||
|
||||
'@types/vinyl@2.0.12':
|
||||
dependencies:
|
||||
'@types/expect': 1.20.4
|
||||
@@ -746,37 +662,6 @@ snapshots:
|
||||
|
||||
aws4@1.13.2: {}
|
||||
|
||||
b4a@1.8.1: {}
|
||||
|
||||
bare-events@2.9.1: {}
|
||||
|
||||
bare-fs@4.8.0:
|
||||
dependencies:
|
||||
bare-events: 2.9.1
|
||||
bare-path: 3.1.1
|
||||
bare-stream: 2.13.3(bare-events@2.9.1)
|
||||
bare-url: 2.5.2
|
||||
fast-fifo: 1.3.2
|
||||
transitivePeerDependencies:
|
||||
- bare-abort-controller
|
||||
- react-native-b4a
|
||||
|
||||
bare-path@3.1.1: {}
|
||||
|
||||
bare-stream@2.13.3(bare-events@2.9.1):
|
||||
dependencies:
|
||||
b4a: 1.8.1
|
||||
streamx: 2.28.0
|
||||
teex: 1.0.1
|
||||
optionalDependencies:
|
||||
bare-events: 2.9.1
|
||||
transitivePeerDependencies:
|
||||
- react-native-b4a
|
||||
|
||||
bare-url@2.5.2:
|
||||
dependencies:
|
||||
bare-path: 3.1.1
|
||||
|
||||
base32-encode@2.0.0:
|
||||
dependencies:
|
||||
to-data-view: 2.0.0
|
||||
@@ -828,20 +713,12 @@ snapshots:
|
||||
jsbn: 0.1.1
|
||||
safer-buffer: 2.1.2
|
||||
|
||||
events-universal@1.0.1:
|
||||
dependencies:
|
||||
bare-events: 2.9.1
|
||||
transitivePeerDependencies:
|
||||
- bare-abort-controller
|
||||
|
||||
extend@3.0.2: {}
|
||||
|
||||
extsprintf@1.3.0: {}
|
||||
|
||||
fast-deep-equal@3.1.3: {}
|
||||
|
||||
fast-fifo@1.3.2: {}
|
||||
|
||||
fast-json-stable-stringify@2.1.0: {}
|
||||
|
||||
first-chunk-stream@1.0.0: {}
|
||||
@@ -917,8 +794,6 @@ snapshots:
|
||||
|
||||
json-stringify-safe@5.0.1: {}
|
||||
|
||||
json5@2.2.3: {}
|
||||
|
||||
jsprim@1.4.2:
|
||||
dependencies:
|
||||
assert-plus: 1.0.0
|
||||
@@ -1087,15 +962,6 @@ snapshots:
|
||||
safer-buffer: 2.1.2
|
||||
tweetnacl: 0.14.5
|
||||
|
||||
streamx@2.28.0:
|
||||
dependencies:
|
||||
events-universal: 1.0.1
|
||||
fast-fifo: 1.3.2
|
||||
text-decoder: 1.2.7
|
||||
transitivePeerDependencies:
|
||||
- bare-abort-controller
|
||||
- react-native-b4a
|
||||
|
||||
string_decoder@0.10.31: {}
|
||||
|
||||
string_decoder@1.1.1:
|
||||
@@ -1111,30 +977,6 @@ snapshots:
|
||||
dependencies:
|
||||
is-utf8: 0.2.1
|
||||
|
||||
tar-stream@3.2.0:
|
||||
dependencies:
|
||||
b4a: 1.8.1
|
||||
bare-fs: 4.8.0
|
||||
fast-fifo: 1.3.2
|
||||
streamx: 2.28.0
|
||||
transitivePeerDependencies:
|
||||
- bare-abort-controller
|
||||
- bare-buffer
|
||||
- react-native-b4a
|
||||
|
||||
teex@1.0.1:
|
||||
dependencies:
|
||||
streamx: 2.28.0
|
||||
transitivePeerDependencies:
|
||||
- bare-abort-controller
|
||||
- react-native-b4a
|
||||
|
||||
text-decoder@1.2.7:
|
||||
dependencies:
|
||||
b4a: 1.8.1
|
||||
transitivePeerDependencies:
|
||||
- react-native-b4a
|
||||
|
||||
through2@2.0.5:
|
||||
dependencies:
|
||||
readable-stream: 2.3.8
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
|
||||
import * as nodePath from 'node:path';
|
||||
|
||||
import * as spritedata from '@smogon/sprite-data/index.ts';
|
||||
|
||||
import type {Artifact} from '../tools/build/artifact.ts';
|
||||
@@ -8,10 +10,10 @@ import type {DeployCtx, SrcFile} from '../tools/deploy/api.ts';
|
||||
// or a raw source file.
|
||||
export type Sprite = Artifact | SrcFile;
|
||||
|
||||
// The published tree a stamped set goes into: where it sits in the output, and
|
||||
// the url that root is served at. Both live here rather than in a `dir` so
|
||||
// that they are written once and everything a set publishes -- the copies, the
|
||||
// urls, the mirror -- is spelled from the same place.
|
||||
// The published tree a stamped set goes into: where it sits in the output ('' for
|
||||
// the output root itself), and the url that root is served at. Both live here
|
||||
// rather than in a `dir` so that they are written once and everything a set
|
||||
// publishes -- the copies, the urls, the mirror -- is spelled from the same place.
|
||||
export type Tree = {
|
||||
root: string,
|
||||
served: string,
|
||||
@@ -52,7 +54,7 @@ export class Manifest {
|
||||
}
|
||||
let stamped = `${name}-${h}.${e}`;
|
||||
this.#entries.set(name, {dir, stamped, filename: `${name}.${e}`});
|
||||
this.#ctx.copy(f, `${this.#tree.root}/${dir}/${stamped}`);
|
||||
this.#ctx.copy(f, nodePath.posix.join(this.#tree.root, dir, stamped));
|
||||
}
|
||||
|
||||
// The whole url the copy `name` answers to. Exposed on its own for a set
|
||||
@@ -74,15 +76,12 @@ export class Manifest {
|
||||
}
|
||||
|
||||
// The same set again under its un-stamped names, as links to the stamped
|
||||
// files. `dst` is a root of its own rather than the tree's, so what a link
|
||||
// says is a path and not a name: the two ends land apart, and the reader
|
||||
// that follows one composed its half from nothing but the sprite. The
|
||||
// tree's own root is not repeated under it, since the mirror is already
|
||||
// one set's worth of it.
|
||||
// files. `dst` is a root of its own rather than the tree's: the reader
|
||||
// that follows one composed its half from nothing but the sprite.
|
||||
links(dst: string): void {
|
||||
for (let [, e] of this.#sorted()) {
|
||||
this.#ctx.symlink(`${dst}/${e.dir}/${e.filename}`,
|
||||
`${this.#tree.root}/${e.dir}/${e.stamped}`);
|
||||
nodePath.posix.join(this.#tree.root, e.dir, e.stamped));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,16 +9,13 @@ import {type Artifact, type CmdSpec, forEachRule, rule} from './tools/build/arti
|
||||
import {base, compresspng, pad, spriteglob, trimimg} from './tools/build/helpers.ts';
|
||||
import {type DeployCtx, deploy} from './tools/deploy/api.ts';
|
||||
|
||||
// The tar root maps onto the served tree: sprites/x is served at
|
||||
// /__assets/sprites/x. The upload rejects a tar whose tree disagrees with the
|
||||
// prefix in services.toml, so the two are checked against each other rather
|
||||
// than each guessing -- which is what lets the pointers below name whole urls
|
||||
// and their readers hold no configuration. __meta/ is the exception: the
|
||||
// upload diverts it to assets-meta/, beside the served tree and out of it.
|
||||
// The output root is the served tree: x is served at /__assets/sprites/x,
|
||||
// which is what lets the pointers below name whole urls and their readers
|
||||
// hold no configuration. __meta/ is the exception: `publish` puts it beside
|
||||
// the served tree and out of it.
|
||||
|
||||
let ASSETS = 'sprites';
|
||||
let SERVED = '/__assets';
|
||||
let TREE: Tree = {root: ASSETS, served: `${SERVED}/${ASSETS}`};
|
||||
let SERVED = '/__assets/sprites';
|
||||
let TREE: Tree = {root: '', served: SERVED};
|
||||
|
||||
// Where the un-stamped names go. A served name carries a content hash and so
|
||||
// can be cached forever, which is exactly why it can't be composed by a reader
|
||||
@@ -27,8 +24,8 @@ let TREE: Tree = {root: ASSETS, served: `${SERVED}/${ASSETS}`};
|
||||
// second time, as links under the un-stamped names naming the stamped file.
|
||||
// They ride in __meta/ rather than the served tree because the tree is
|
||||
// add-only -- a name in it is promised never to change -- and a link is
|
||||
// repointed on every upload; the upload retargets each one at where its asset
|
||||
// landed on its side, under a directory it already names for this set.
|
||||
// repointed on every publish, which points each one at where its file sits
|
||||
// in the tree.
|
||||
let LINKS = '__meta/links';
|
||||
|
||||
let minispriteInputs = spriteglob(['src/minisprites/pokemon/gen6/*', 'src/minisprites/items/*'], {a: false});
|
||||
@@ -216,7 +213,7 @@ let [, sheetCss, sheetWebp] = rule(minispriteInputs, {
|
||||
// Hash-stamped css + webp. The css url rides in __meta/ for the dex to read.
|
||||
deploy(async ctx => {
|
||||
let wh = await ctx.hash(sheetWebp);
|
||||
ctx.copy(sheetWebp, `${ASSETS}/spritesheet-${wh}.webp`);
|
||||
ctx.copy(sheetWebp, `spritesheet-${wh}.webp`);
|
||||
let src = await ctx.read(sheetCss);
|
||||
let css = src.replaceAll('url("./spritesheet.webp")', `url("./spritesheet-${wh}.webp")`);
|
||||
if (css === src) {
|
||||
@@ -226,8 +223,8 @@ deploy(async ctx => {
|
||||
// of (css, webp), so this changes exactly when the served bytes
|
||||
// change.
|
||||
let ch = await ctx.hash(sheetCss, sheetWebp);
|
||||
ctx.write(`${ASSETS}/spritesheet-${ch}.css`, css);
|
||||
ctx.write('__meta/spritesheet-css-url.txt', `${SERVED}/${ASSETS}/spritesheet-${ch}.css\n`);
|
||||
ctx.write(`spritesheet-${ch}.css`, css);
|
||||
ctx.write('__meta/spritesheet-css-url.txt', `${SERVED}/spritesheet-${ch}.css\n`);
|
||||
});
|
||||
|
||||
// Forumsprites: uniform-size minisprites under stamped names, with the
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
|
||||
import * as fs from 'node:fs/promises';
|
||||
import * as nodePath from 'node:path';
|
||||
|
||||
import JSON5 from 'json5';
|
||||
|
||||
import {BuildError} from '../build/errors.ts';
|
||||
|
||||
export type DeployEntry = {
|
||||
subset: string[],
|
||||
cmd: string,
|
||||
// dir entries get their subset materialized into a temp directory whose
|
||||
// path replaces %d in cmd. Every other entry's cmd is `smogonctl assets
|
||||
// upload` (or a wrapper around it): it gets a manifest of the subset on
|
||||
// stdin, answers on stdout with what it lacks, and gets a tar of that.
|
||||
dir?: boolean,
|
||||
};
|
||||
|
||||
export type DeployTarget = {
|
||||
buildFile: string,
|
||||
deploy: DeployEntry[],
|
||||
};
|
||||
|
||||
export type DeployConfig = Map<string, DeployTarget>;
|
||||
|
||||
function isStringArray(v: unknown): v is string[] {
|
||||
return Array.isArray(v) && v.every(x => typeof x === 'string');
|
||||
}
|
||||
|
||||
export async function loadDeployConfig(path: string): Promise<DeployConfig> {
|
||||
let text: string;
|
||||
try {
|
||||
text = await fs.readFile(path, 'utf8');
|
||||
} catch {
|
||||
throw new BuildError(`missing ${path}; see README ("Deploying") for the schema`);
|
||||
}
|
||||
let raw: unknown;
|
||||
try {
|
||||
raw = JSON5.parse(text);
|
||||
} catch (err) {
|
||||
throw new BuildError(`${path}: ${(err as Error).message}`);
|
||||
}
|
||||
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
|
||||
throw new BuildError(`${path}: top level must be an object of deploy names`);
|
||||
}
|
||||
|
||||
let config: DeployConfig = new Map();
|
||||
for (let [name, t] of Object.entries(raw)) {
|
||||
let target = t as Partial<DeployTarget>;
|
||||
if (typeof t !== 'object' || t === null || typeof target.buildFile !== 'string'
|
||||
|| !Array.isArray(target.deploy)) {
|
||||
throw new BuildError(`${path}: ${name}: expected {buildFile: string, deploy: [...]}`);
|
||||
}
|
||||
for (let e of target.deploy as Partial<DeployEntry>[]) {
|
||||
if (typeof e !== 'object' || e === null
|
||||
|| !isStringArray(e.subset) || typeof e.cmd !== 'string'
|
||||
|| (e.dir !== undefined && typeof e.dir !== 'boolean')) {
|
||||
throw new BuildError(
|
||||
`${path}: ${name}: each deploy entry needs {subset: string[], cmd: string, dir?: boolean}`);
|
||||
}
|
||||
if (Boolean(e.dir) !== e.cmd.includes('%d')) {
|
||||
throw new BuildError(`${path}: ${name}: ${e.dir
|
||||
? 'a dir entry\'s cmd must use %d'
|
||||
: 'a tar entry\'s cmd must not use %d (missing dir: true?)'}: ${e.cmd}`);
|
||||
}
|
||||
}
|
||||
config.set(name, target as DeployTarget);
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
// Route finish outputs to deploy entries: per entry, the set of dsts its
|
||||
// subset globs match. Every glob must match something and every dst must be
|
||||
// covered by some entry; to unship an output, don't emit it in finish.
|
||||
export function matchSubsets(dsts: readonly string[], entries: readonly DeployEntry[])
|
||||
: {entry: DeployEntry, matched: Set<string>}[] {
|
||||
let covered = new Set<string>();
|
||||
let perEntry = entries.map(entry => {
|
||||
let matched = new Set<string>();
|
||||
for (let glob of entry.subset) {
|
||||
let hits = dsts.filter(d => nodePath.matchesGlob(d, glob));
|
||||
if (hits.length === 0) {
|
||||
throw new BuildError(`subset glob matches no outputs: ${glob}`);
|
||||
}
|
||||
for (let hit of hits) {
|
||||
matched.add(hit);
|
||||
covered.add(hit);
|
||||
}
|
||||
}
|
||||
return {entry, matched};
|
||||
});
|
||||
let uncovered = dsts.filter(d => !covered.has(d));
|
||||
if (uncovered.length > 0) {
|
||||
let shown = uncovered.slice(0, 10).join('\n ');
|
||||
let more = uncovered.length > 10 ? `\n ... and ${uncovered.length - 10} more` : '';
|
||||
throw new BuildError(`outputs not covered by any deploy entry:\n ${shown}${more}`);
|
||||
}
|
||||
return perEntry;
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
|
||||
import {spawn, type ChildProcess} from 'node:child_process';
|
||||
import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import * as os from 'node:os';
|
||||
@@ -17,9 +16,6 @@ import {killAllProcessGroups} from '../build/exec.ts';
|
||||
import {setConfig} from '../build/helpers.ts';
|
||||
import * as db from '../build/store.ts';
|
||||
import * as api from './api.ts';
|
||||
import {loadDeployConfig, matchSubsets} from './config.ts';
|
||||
import {withProgress} from './progress.ts';
|
||||
import {manifestOf, negotiate} from './protocol.ts';
|
||||
import {ActionQueue} from './queue.ts';
|
||||
|
||||
let root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..');
|
||||
@@ -43,20 +39,20 @@ type CommonOpts = {
|
||||
type VerbOpts = CommonOpts & {
|
||||
output?: string,
|
||||
link?: boolean,
|
||||
tar?: boolean,
|
||||
record?: boolean,
|
||||
tree?: string,
|
||||
meta?: string,
|
||||
};
|
||||
|
||||
let USAGE = `usage: node tools/deploy/index.ts <command> [options]
|
||||
|
||||
commands:
|
||||
build [files...] build the rules of the given deploys (default: all *.build.ts)
|
||||
deploy [names...] build, finish, and send each subset to its command from
|
||||
deploy.json5: a manifest and then a tar of what it asks
|
||||
for on stdin, or a %d dir (no names: list the deploys;
|
||||
-o <dir>: materialize each subset there instead of
|
||||
running its command)
|
||||
run <file> -o <dir> build, finish, and materialize to a directory (or tar file)
|
||||
run <file> -o <dir> build, finish, and materialize to a directory
|
||||
publish <file> --tree <dir> --meta <dir>
|
||||
build, finish, and place the outputs: the served tree gets
|
||||
what it hasn't got, __meta/ is written whole to the meta
|
||||
directory with its links pointing into the tree
|
||||
inspect <paths...> -o <dir> build every rule touching the given source paths and copy
|
||||
the outputs out
|
||||
refactor [files...] report how the published files differ from the last
|
||||
@@ -67,10 +63,11 @@ options:
|
||||
-n, --dry-run print what would run without changing anything
|
||||
--fail-fast stop scheduling new rules after the first failure
|
||||
--config <file> config file (default: build.config)
|
||||
-o, --output <dir> run/inspect/deploy: output directory (a file with --tar)
|
||||
-o, --output <dir> run/inspect: output directory
|
||||
--tree <dir> publish: the served tree (add-only)
|
||||
--meta <dir> publish: where __meta/ goes
|
||||
--record refactor: overwrite the baseline instead of checking it
|
||||
--link run: hardlink instead of copying
|
||||
--tar run: write a tar file
|
||||
-v, --verbose print more detail
|
||||
-h, --help show this help`;
|
||||
|
||||
@@ -85,13 +82,13 @@ let COMMON_OPTIONS = {
|
||||
|
||||
let VERB_OPTIONS = {
|
||||
build: {},
|
||||
deploy: {
|
||||
output: {type: 'string', short: 'o'},
|
||||
},
|
||||
run: {
|
||||
output: {type: 'string', short: 'o'},
|
||||
link: {type: 'boolean'},
|
||||
tar: {type: 'boolean'},
|
||||
},
|
||||
publish: {
|
||||
tree: {type: 'string'},
|
||||
meta: {type: 'string'},
|
||||
},
|
||||
inspect: {
|
||||
output: {type: 'string', short: 'o'},
|
||||
@@ -218,13 +215,6 @@ async function runFinish(fns: readonly api.DeployFn[], verbose: boolean): Promis
|
||||
return aq;
|
||||
}
|
||||
|
||||
function waitExit(child: ChildProcess): Promise<number | null> {
|
||||
return new Promise((resolve, reject) => {
|
||||
child.on('error', reject);
|
||||
child.on('close', code => resolve(code));
|
||||
});
|
||||
}
|
||||
|
||||
async function cmdBuild(files: string[], opts: CommonOpts): Promise<void> {
|
||||
setConfig(await loadConfig(opts.config));
|
||||
// GC needs the full rule universe: only an unfiltered union build
|
||||
@@ -234,111 +224,6 @@ async function cmdBuild(files: string[], opts: CommonOpts): Promise<void> {
|
||||
process.exitCode = await buildThen(artifact.getDecls(), opts, gc);
|
||||
}
|
||||
|
||||
async function cmdDeploy(names: string[], opts: VerbOpts): Promise<void> {
|
||||
let config = await loadDeployConfig('deploy.json5');
|
||||
if (names.length === 0) {
|
||||
for (let [name, target] of config) {
|
||||
console.log(`${name} (${target.buildFile})`);
|
||||
for (let entry of target.deploy) {
|
||||
console.log(` ${entry.subset.join(' ')} | ${entry.cmd}`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
let targets = names.map(name => {
|
||||
let target = config.get(name);
|
||||
if (target === undefined) {
|
||||
throw new BuildError(`deploy.json5: no deploy named ${name}`);
|
||||
}
|
||||
return {name, target};
|
||||
});
|
||||
setConfig(await loadConfig(opts.config));
|
||||
let files = [...new Set(targets.map(t => t.target.buildFile))];
|
||||
let specs = await importDeploys(files);
|
||||
process.exitCode = await buildThen(artifact.getDecls(), opts, false, async () => {
|
||||
for (let {name, target} of targets) {
|
||||
let aq = await runFinish(finishOf(specs, target.buildFile), Boolean(opts.verbose));
|
||||
if (aq === null) {
|
||||
return 1;
|
||||
}
|
||||
let dsts = aq.log.filter(e => e.type === 'Op').map(e => e.dst);
|
||||
for (let [i, {entry, matched}] of matchSubsets(dsts, target.deploy).entries()) {
|
||||
console.log(`${name}: ${matched.size} files | ${entry.cmd}`);
|
||||
// Debug mode: materialize what each entry would ship (tar
|
||||
// entries included) instead of running its command.
|
||||
if (opts.output !== undefined) {
|
||||
let dir = nodePath.join(opts.output, name, String(i));
|
||||
await withProgress(`${name}: copying`, matched.size,
|
||||
tick => aq.run(dir, 'copy', dst => matched.has(dst), tick));
|
||||
console.log(` -> ${dir}`);
|
||||
continue;
|
||||
}
|
||||
if (entry.dir) {
|
||||
await fs.mkdir(TMP_DIR, {recursive: true});
|
||||
let tmp = await fs.mkdtemp(nodePath.join(TMP_DIR, 'deploy-'));
|
||||
try {
|
||||
await withProgress(`${name}: staging`, matched.size,
|
||||
tick => aq.run(tmp, 'copy', dst => matched.has(dst), tick));
|
||||
let cmd = spawn(entry.cmd.replaceAll('%d', tmp),
|
||||
{shell: true, stdio: ['ignore', 'inherit', 'inherit']});
|
||||
if (await waitExit(cmd) !== 0) {
|
||||
return 1;
|
||||
}
|
||||
} finally {
|
||||
await fs.rm(tmp, {recursive: true, force: true});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// The command's stdout is ours to read: the reply to the
|
||||
// manifest comes back on it, and its report after that is
|
||||
// relayed. stderr stays the terminal's, for the bar.
|
||||
let upload = spawn(entry.cmd, {shell: true, stdio: ['pipe', 'pipe', 'inherit']});
|
||||
let stdin = upload.stdin;
|
||||
if (stdin === null) {
|
||||
throw new BuildError(`no stdin pipe for: ${entry.cmd}`);
|
||||
}
|
||||
// If the command dies early we report its exit code; don't
|
||||
// also crash on the resulting EPIPE, which reaches both
|
||||
// stdin and (via streamx's destroy propagation) the pack.
|
||||
stdin.on('error', () => {});
|
||||
let manifest = await manifestOf(aq, dst => matched.has(dst));
|
||||
let reply = await negotiate(upload, manifest);
|
||||
if ('error' in reply) {
|
||||
// What the command had to say is already on stdout; this
|
||||
// is for the case where it said nothing usable.
|
||||
let code = await waitExit(upload);
|
||||
console.log(` refused before anything was sent`
|
||||
+ (reply.error === null ? '' : `: ${reply.error}`));
|
||||
return code === 0 || code === null ? 1 : code;
|
||||
}
|
||||
let send = new Set([...reply.wanted, ...manifest.metaFiles]);
|
||||
console.log(` ${reply.wanted.size} of ${manifest.assets.size} files to send`);
|
||||
let code;
|
||||
if (send.size === 0) {
|
||||
// Nothing follows an empty answer, not even an empty tar:
|
||||
// the command takes the end of the stream as the end.
|
||||
stdin.end();
|
||||
code = await waitExit(upload);
|
||||
} else {
|
||||
// The bar tracks the upload itself: an entry counts once
|
||||
// the command has taken it, not once it has been read off
|
||||
// disk. It counts what travels, not what was published.
|
||||
code = await withProgress(`${name}: uploading`, send.size, async tick => {
|
||||
let pack = await aq.pack(dst => send.has(dst), tick);
|
||||
pack.on('error', () => {});
|
||||
pack.pipe(stdin);
|
||||
return await waitExit(upload);
|
||||
});
|
||||
}
|
||||
if (code !== 0) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
async function cmdRun(file: string, opts: VerbOpts): Promise<void> {
|
||||
let output = requireOutput(opts);
|
||||
setConfig(await loadConfig(opts.config));
|
||||
@@ -348,7 +233,36 @@ async function cmdRun(file: string, opts: VerbOpts): Promise<void> {
|
||||
if (aq === null) {
|
||||
return 1;
|
||||
}
|
||||
await aq.run(output, opts.tar ? 'tar' : opts.link ? 'link' : 'copy');
|
||||
await aq.run(output, opts.link ? 'link' : 'copy');
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
// What the publish stands for as one string: the sorted name -> digest map,
|
||||
// so two publishes of the same bytes under the same names compare equal
|
||||
// whatever built them.
|
||||
function digestOf(outputs: Map<string, string>): string {
|
||||
let h = crypto.createHash('sha256');
|
||||
for (let [dst, digest] of [...outputs].sort((a, b) => a[0] < b[0] ? -1 : 1)) {
|
||||
h.update(`${dst}\0${digest}\n`);
|
||||
}
|
||||
return h.digest('hex');
|
||||
}
|
||||
|
||||
async function cmdPublish(file: string, opts: VerbOpts): Promise<void> {
|
||||
let {tree, meta} = opts;
|
||||
if (tree === undefined || meta === undefined) {
|
||||
throw new BuildError('publish needs --tree and --meta');
|
||||
}
|
||||
setConfig(await loadConfig(opts.config));
|
||||
let specs = await importDeploys([file]);
|
||||
process.exitCode = await buildThen(artifact.getDecls(), opts, false, async () => {
|
||||
let aq = await runFinish(finishOf(specs, file), Boolean(opts.verbose));
|
||||
if (aq === null) {
|
||||
return 1;
|
||||
}
|
||||
let {files, placed, kept} = await aq.publish(tree, meta, digestOf(await outputs(aq)));
|
||||
console.log(`published ${files} files: ${placed} new, ${kept} already there`);
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
@@ -521,20 +435,20 @@ async function main(argv: string[]): Promise<void> {
|
||||
verbose: Boolean(v.verbose),
|
||||
output: v.output as string | undefined,
|
||||
link: Boolean(v.link),
|
||||
tar: Boolean(v.tar),
|
||||
record: Boolean(v.record),
|
||||
tree: v.tree as string | undefined,
|
||||
meta: v.meta as string | undefined,
|
||||
};
|
||||
switch (verb) {
|
||||
case 'build':
|
||||
return cmdBuild(positionals, opts);
|
||||
case 'deploy':
|
||||
return cmdDeploy(positionals, opts);
|
||||
case 'run': {
|
||||
case 'run':
|
||||
case 'publish': {
|
||||
let [file, ...extra] = positionals;
|
||||
if (file === undefined || extra.length > 0) {
|
||||
throw new BuildError('run takes exactly one deploy file');
|
||||
throw new BuildError(`${verb} takes exactly one deploy file`);
|
||||
}
|
||||
return cmdRun(file, opts);
|
||||
return verb === 'run' ? cmdRun(file, opts) : cmdPublish(file, opts);
|
||||
}
|
||||
case 'inspect':
|
||||
if (positionals.length === 0) {
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"@types/node": "^24.10.7",
|
||||
"@types/tar-stream": "^2.2.2",
|
||||
"base32-encode": "^2.0.0",
|
||||
"json5": "^2.2.3",
|
||||
"tar-stream": "^3.0.0"
|
||||
"base32-encode": "^2.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "node --test test/*.test.ts"
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
// A one-line progress bar for the publishing step of a deploy, which is
|
||||
// otherwise a long silent wait: thousands of files being packed into an
|
||||
// upload or copied into a staging tree, with nothing on the terminal between
|
||||
// the line naming the command and the line after it finishes.
|
||||
//
|
||||
// Only drawn when stderr is a terminal. Piped or under CI a redraw per file
|
||||
// is noise in the log, and the deploy's own stdout lines are the record of
|
||||
// what happened; those are printed either way, and never carry the bar.
|
||||
|
||||
let REDRAW_MS = 100;
|
||||
// Below this a bar is more misleading than a bare count.
|
||||
let MIN_BAR = 10;
|
||||
|
||||
// The parts of a terminal a bar uses, so a test can hand it one.
|
||||
export type Term = {
|
||||
isTTY?: boolean | undefined,
|
||||
columns?: number | undefined,
|
||||
write(s: string): unknown,
|
||||
};
|
||||
|
||||
export class Progress {
|
||||
#label: string;
|
||||
#total: number;
|
||||
#term: Term;
|
||||
#done: number;
|
||||
#drawn: boolean;
|
||||
#lastDraw: number;
|
||||
|
||||
constructor(label: string, total: number, term: Term = process.stderr) {
|
||||
this.#label = label;
|
||||
this.#total = total;
|
||||
this.#term = term;
|
||||
this.#done = 0;
|
||||
this.#drawn = false;
|
||||
this.#lastDraw = 0;
|
||||
if (this.#interactive) {
|
||||
this.#draw();
|
||||
}
|
||||
}
|
||||
|
||||
get #interactive(): boolean {
|
||||
return this.#term.isTTY === true && this.#total > 0;
|
||||
}
|
||||
|
||||
tick(n = 1) {
|
||||
this.#done = Math.min(this.#done + n, this.#total);
|
||||
if (!this.#interactive) {
|
||||
return;
|
||||
}
|
||||
// The line is the bar's only while there is progress left to show.
|
||||
// What the command has to say about the rest of the wait -- it is
|
||||
// still running, with the terminal to itself -- lands on a clean line
|
||||
// instead of on the end of a finished bar.
|
||||
if (this.#done === this.#total) {
|
||||
this.finish();
|
||||
return;
|
||||
}
|
||||
// Redraw on a timer rather than per file: a deploy ticks tens of
|
||||
// thousands of times.
|
||||
if (Date.now() - this.#lastDraw < REDRAW_MS) {
|
||||
return;
|
||||
}
|
||||
this.#draw();
|
||||
}
|
||||
|
||||
// Leave the line as the deploy found it: the bar is a live thing, and
|
||||
// what stays in the scrollback is the deploy's own output.
|
||||
finish() {
|
||||
if (this.#drawn) {
|
||||
this.#term.write('\x1b[2K\r');
|
||||
this.#drawn = false;
|
||||
}
|
||||
}
|
||||
|
||||
#draw() {
|
||||
this.#lastDraw = Date.now();
|
||||
let columns = this.#term.columns || 80;
|
||||
// Pad the count so the bar keeps its width as the numbers grow into
|
||||
// each other's columns.
|
||||
let counts = `${String(this.#done).padStart(String(this.#total).length)}/${this.#total}`;
|
||||
// label, a space, '[', ']', a space, counts, and a column to spare.
|
||||
let width = columns - this.#label.length - counts.length - 5;
|
||||
let line;
|
||||
if (width < MIN_BAR) {
|
||||
line = `${this.#label} ${counts}`;
|
||||
} else {
|
||||
let filled = Math.round(width * this.#done / this.#total);
|
||||
line = `${this.#label} [${'#'.repeat(filled)}${'-'.repeat(width - filled)}] ${counts}`;
|
||||
}
|
||||
// One column short of the width: writing the last one wraps to the
|
||||
// next line on some terminals, which leaves the bar behind.
|
||||
this.#term.write(`\x1b[2K\r${line.slice(0, columns - 1)}`);
|
||||
this.#drawn = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Draw a bar over `body`'s work, and leave the line clean however it goes.
|
||||
export async function withProgress<T>(label: string, total: number,
|
||||
body: (tick: () => void) => Promise<T>): Promise<T> {
|
||||
let bar = new Progress(label, total);
|
||||
try {
|
||||
return await body(() => bar.tick());
|
||||
} finally {
|
||||
bar.finish();
|
||||
}
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
|
||||
// What a deploy says to `smogonctl assets upload` before it sends anything, and
|
||||
// how it reads the answer.
|
||||
//
|
||||
// Every published name carries a content hash and the served tree is add-only, so
|
||||
// a name the home already has is a file that needn't travel. The deploy therefore
|
||||
// opens the stream with a manifest -- every name with its size, and every link --
|
||||
// and the command answers with the positions of the ones it hasn't got. Only those,
|
||||
// plus the __meta/ files, are then tarred into the same stdin. The same connection
|
||||
// and the same command line as a bare tar; the tar is just shorter.
|
||||
//
|
||||
// The format is binary and every string is length prefixed, so a name is whatever
|
||||
// bytes a tar would carry for it and nothing in the framing could be mistaken for
|
||||
// one. The server side spells it out in smogonctl's assets command; this is the
|
||||
// other half, and the two are tested against each other by shape rather than
|
||||
// sharing code.
|
||||
//
|
||||
// "smogonctl assets" u32 version=1 u32 count
|
||||
// count times: u8 kind
|
||||
// kind 1 (asset) u64 size u32 len path
|
||||
// kind 2 (metadata) u64 size u32 len path (always sent)
|
||||
// kind 3 (link) u32 len path u32 len linkname
|
||||
//
|
||||
// The reply is the same header with the count of what's wanted, then that many u32
|
||||
// positions into the manifest. Whatever the command prints after that is its own
|
||||
// report, relayed as is.
|
||||
|
||||
import type {ChildProcess} from 'node:child_process';
|
||||
import * as fs from 'node:fs/promises';
|
||||
|
||||
import {type ActionQueue} from './queue.ts';
|
||||
|
||||
export let MAGIC = Buffer.from('smogonctl assets');
|
||||
export let VERSION = 1;
|
||||
let HEADER_SIZE = MAGIC.length + 4 + 4;
|
||||
let ASSET = 1;
|
||||
let METAFILE = 2;
|
||||
let LINK = 3;
|
||||
let META = '__meta';
|
||||
|
||||
export type Manifest = {
|
||||
bytes: Buffer,
|
||||
// The published name at each position: what a position in the reply means.
|
||||
entries: string[],
|
||||
// Positions that are assets, the only ones a reply may name.
|
||||
assets: Set<number>,
|
||||
// Names the tar carries whatever the reply says.
|
||||
metaFiles: Set<string>,
|
||||
};
|
||||
|
||||
function isMeta(dst: string): boolean {
|
||||
return dst === META || dst.startsWith(`${META}/`);
|
||||
}
|
||||
|
||||
function u32(n: number): Buffer {
|
||||
let b = Buffer.alloc(4);
|
||||
b.writeUInt32BE(n);
|
||||
return b;
|
||||
}
|
||||
|
||||
function u64(n: number): Buffer {
|
||||
let b = Buffer.alloc(8);
|
||||
b.writeBigUInt64BE(BigInt(n));
|
||||
return b;
|
||||
}
|
||||
|
||||
function str(s: string): Buffer {
|
||||
let raw = Buffer.from(s);
|
||||
return Buffer.concat([u32(raw.length), raw]);
|
||||
}
|
||||
|
||||
// The manifest for what `filter` selects of the queue, in queue order: the order
|
||||
// the tar's entries would come in, and the order the reply's positions index.
|
||||
export async function manifestOf(aq: ActionQueue, filter?: (dst: string) => boolean): Promise<Manifest> {
|
||||
let parts: Buffer[] = [];
|
||||
let entries: string[] = [];
|
||||
let assets = new Set<number>();
|
||||
let metaFiles = new Set<string>();
|
||||
for (let entry of aq.log) {
|
||||
if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) {
|
||||
continue;
|
||||
}
|
||||
let op = entry.op;
|
||||
let index = entries.length;
|
||||
entries.push(entry.dst);
|
||||
if (op.type === 'Symlink') {
|
||||
parts.push(Buffer.from([LINK]), str(entry.dst), str(op.target));
|
||||
continue;
|
||||
}
|
||||
// The size the tar header will carry: a Write's data goes in as utf-8, the
|
||||
// same way tar-stream takes the string.
|
||||
let size = op.type === 'Copy' ? (await fs.stat(op.src)).size : Buffer.byteLength(op.data);
|
||||
if (isMeta(entry.dst)) {
|
||||
metaFiles.add(entry.dst);
|
||||
parts.push(Buffer.from([METAFILE]));
|
||||
} else {
|
||||
assets.add(index);
|
||||
parts.push(Buffer.from([ASSET]));
|
||||
}
|
||||
parts.push(u64(size), str(entry.dst));
|
||||
}
|
||||
let bytes = Buffer.concat([MAGIC, u32(VERSION), u32(entries.length), ...parts]);
|
||||
return {bytes, entries, assets, metaFiles};
|
||||
}
|
||||
|
||||
export type Outcome = 'pending' | 'ok' | 'refused';
|
||||
|
||||
// The reply, read as it arrives. Strict from the first byte: the command's stdout
|
||||
// is either a reply followed by its report, or it is a refusal, and a refusal is
|
||||
// relayed whole. Nothing that isn't the reply is skipped over to find one, since
|
||||
// anything printing on that stream ahead of the command is something the deploy
|
||||
// shouldn't be talking through -- a shell profile, a filter -- and finding out is
|
||||
// better than working around it.
|
||||
export class Reply {
|
||||
outcome: Outcome = 'pending';
|
||||
// Why a refusal was ours rather than the command's, or null where the command
|
||||
// said so itself and its words were relayed.
|
||||
error: string | null = null;
|
||||
wanted = new Set<string>();
|
||||
#manifest: Manifest;
|
||||
#sink: (chunk: Buffer) => void;
|
||||
#buf = Buffer.alloc(0);
|
||||
#count = -1;
|
||||
#seen = new Set<number>();
|
||||
|
||||
constructor(manifest: Manifest, sink: (chunk: Buffer) => void) {
|
||||
this.#manifest = manifest;
|
||||
this.#sink = sink;
|
||||
}
|
||||
|
||||
feed(chunk: Buffer): void {
|
||||
if (this.outcome !== 'pending') {
|
||||
this.#sink(chunk);
|
||||
return;
|
||||
}
|
||||
this.#buf = Buffer.concat([this.#buf, chunk]);
|
||||
// Refused as early as the bytes disagree, not once there are enough of
|
||||
// them: a one-line message is shorter than a header.
|
||||
let head = this.#buf.subarray(0, MAGIC.length);
|
||||
if (!head.equals(MAGIC.subarray(0, head.length))) {
|
||||
this.#refuse('the command didn\'t answer the manifest');
|
||||
return;
|
||||
}
|
||||
if (this.#buf.length < HEADER_SIZE) {
|
||||
return;
|
||||
}
|
||||
if (this.#count < 0) {
|
||||
let version = this.#buf.readUInt32BE(MAGIC.length);
|
||||
if (version !== VERSION) {
|
||||
this.#refuse(`the command answered with version ${version}, and this speaks ${VERSION}`);
|
||||
return;
|
||||
}
|
||||
this.#count = this.#buf.readUInt32BE(MAGIC.length + 4);
|
||||
}
|
||||
let offset = HEADER_SIZE + 4 * this.#seen.size;
|
||||
while (this.#seen.size < this.#count && this.#buf.length >= offset + 4) {
|
||||
let index = this.#buf.readUInt32BE(offset);
|
||||
offset += 4;
|
||||
let dst = this.#manifest.entries[index];
|
||||
if (dst === undefined || !this.#manifest.assets.has(index)) {
|
||||
this.#refuse(`the command asked for position ${index}, which isn't an asset in the manifest`);
|
||||
return;
|
||||
}
|
||||
if (this.#seen.has(index)) {
|
||||
this.#refuse(`the command asked for ${dst} twice`);
|
||||
return;
|
||||
}
|
||||
this.#seen.add(index);
|
||||
this.wanted.add(dst);
|
||||
}
|
||||
if (this.#seen.size === this.#count) {
|
||||
this.outcome = 'ok';
|
||||
let rest = this.#buf.subarray(offset);
|
||||
this.#buf = Buffer.alloc(0);
|
||||
if (rest.length > 0) {
|
||||
this.#sink(rest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The stream closed. A reply still being read is a command that went away
|
||||
// without finishing its answer.
|
||||
end(): void {
|
||||
if (this.outcome === 'pending') {
|
||||
this.#refuse('the command hung up before answering the manifest');
|
||||
}
|
||||
}
|
||||
|
||||
#refuse(error: string): void {
|
||||
this.outcome = 'refused';
|
||||
this.error = error;
|
||||
let held = this.#buf;
|
||||
this.#buf = Buffer.alloc(0);
|
||||
if (held.length > 0) {
|
||||
this.#sink(held);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Send the manifest and read the reply: the names to tar, or null where the
|
||||
// command refused. On a refusal stdin is closed, so a command waiting for a tar
|
||||
// sees the end of the stream rather than a deploy waiting for it; everything it
|
||||
// printed is relayed to `sink` either way, and goes on being relayed after.
|
||||
export function negotiate(child: ChildProcess, manifest: Manifest,
|
||||
sink: (chunk: Buffer) => void = chunk => { process.stdout.write(chunk); })
|
||||
: Promise<{wanted: Set<string>} | {error: string | null}> {
|
||||
let stdin = child.stdin;
|
||||
let stdout = child.stdout;
|
||||
if (stdin === null || stdout === null) {
|
||||
throw new Error('negotiate needs the command\'s stdin and stdout piped');
|
||||
}
|
||||
return new Promise(resolve => {
|
||||
let reply = new Reply(manifest, sink);
|
||||
let settled = false;
|
||||
let settle = () => {
|
||||
if (settled || reply.outcome === 'pending') {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
if (reply.outcome === 'refused') {
|
||||
stdin.end();
|
||||
resolve({error: reply.error});
|
||||
} else {
|
||||
resolve({wanted: reply.wanted});
|
||||
}
|
||||
};
|
||||
// Attached before the manifest goes out and kept for the whole run: the
|
||||
// reply can be longer than a pipe holds, and the report after it has to
|
||||
// land somewhere while the tar is going the other way.
|
||||
stdout.on('data', (chunk: Buffer) => {
|
||||
reply.feed(chunk);
|
||||
settle();
|
||||
});
|
||||
stdout.on('end', () => {
|
||||
reply.end();
|
||||
settle();
|
||||
});
|
||||
// A command that dies on the manifest closes the pipe under this write;
|
||||
// its exit code is the report, not the EPIPE.
|
||||
stdin.write(manifest.bytes, () => {});
|
||||
});
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
|
||||
import * as fs from 'node:fs/promises';
|
||||
import {createWriteStream} from 'node:fs';
|
||||
import * as nodePath from 'node:path';
|
||||
import tar from 'tar-stream';
|
||||
|
||||
import {BuildError} from '../build/errors.ts';
|
||||
|
||||
type Op = {
|
||||
type: 'Write',
|
||||
@@ -31,6 +31,15 @@ type DebugEntry = {
|
||||
|
||||
export type LogEntry = OpEntry | DebugEntry;
|
||||
|
||||
// The output root a publish keeps out of the served tree.
|
||||
export let META = '__meta';
|
||||
|
||||
export type Published = {
|
||||
files: number,
|
||||
placed: number,
|
||||
kept: number,
|
||||
};
|
||||
|
||||
// What a link names, resolved lexically against the directory it sits in: the
|
||||
// tree it describes isn't on disk anywhere yet, so there is nothing else to
|
||||
// resolve against. Null where it walks out of the tree, or names the tree
|
||||
@@ -52,6 +61,29 @@ function resolveLink(dst: string, target: string): string | null {
|
||||
return parts.length === 0 ? null : parts.join('/');
|
||||
}
|
||||
|
||||
// The path under META, or null for a dst outside it.
|
||||
function metaRest(dst: string): string | null {
|
||||
if (dst === META) {
|
||||
return '';
|
||||
}
|
||||
return dst.startsWith(`${META}/`) ? dst.slice(META.length + 1) : null;
|
||||
}
|
||||
|
||||
// Write-then-rename: a served name is cached forever, so nothing may ever
|
||||
// read it half written.
|
||||
async function place(dst: string, write: (tmp: string) => Promise<void>): Promise<void> {
|
||||
await fs.mkdir(nodePath.dirname(dst), {recursive: true});
|
||||
let tmp = nodePath.join(nodePath.dirname(dst),
|
||||
`.${nodePath.basename(dst)}.${process.pid}.${Math.random().toString(36).slice(2)}`);
|
||||
try {
|
||||
await write(tmp);
|
||||
await fs.rename(tmp, dst);
|
||||
} catch (err) {
|
||||
await fs.rm(tmp, {force: true});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export class ActionQueue {
|
||||
#seen: Map<string, OpEntry | 'MoreThan1'>;
|
||||
// Have an accessor for this in the future? idk
|
||||
@@ -93,6 +125,9 @@ export class ActionQueue {
|
||||
if (nodePath.isAbsolute(dst)) {
|
||||
this.valid = false;
|
||||
entry.valid = 'Absolute';
|
||||
} else if (dst === '..' || dst.startsWith('../')) {
|
||||
this.valid = false;
|
||||
entry.valid = 'Escapes';
|
||||
} else {
|
||||
let lastEntry = this.#seen.get(dst);
|
||||
if (lastEntry === undefined) {
|
||||
@@ -116,8 +151,7 @@ export class ActionQueue {
|
||||
this.#pushOp({type: 'Write', data}, dst);
|
||||
}
|
||||
|
||||
// A link the receiving side extracts as root, so its target may only name
|
||||
// something else in this tree.
|
||||
// A link's target may only name something else in this tree.
|
||||
symlink(target: string, dst: string) {
|
||||
let entry = this.#pushOp({type: 'Symlink', target}, dst);
|
||||
if (nodePath.isAbsolute(target) || resolveLink(entry.dst, target) === null) {
|
||||
@@ -163,86 +197,144 @@ export class ActionQueue {
|
||||
}
|
||||
}
|
||||
|
||||
// `onFile` is called once per published name as it lands, for progress.
|
||||
async run(dir: string, mode: 'link' | 'copy' | 'tar', filter?: (dst: string) => boolean,
|
||||
onFile?: () => void) {
|
||||
#ops(): OpEntry[] {
|
||||
if (!this.valid)
|
||||
throw new Error(`Invalid ActionQueue`);
|
||||
if (mode !== 'tar') {
|
||||
for (let entry of this.log) {
|
||||
if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst)))
|
||||
continue;
|
||||
let op = entry.op;
|
||||
let dst = nodePath.join(dir, entry.dst);
|
||||
await fs.mkdir(nodePath.dirname(dst), {recursive: true});
|
||||
if (op.type === 'Copy'){
|
||||
// Read-only sources are CAS objects; their mode must not
|
||||
// leak into deploy trees (rsync -a would ship it), and a
|
||||
// hardlink cannot carry its own mode, so copy those.
|
||||
if (mode === 'link' && ((await fs.stat(op.src)).mode & 0o200) !== 0) {
|
||||
await fs.link(op.src, dst);
|
||||
} else {
|
||||
await fs.copyFile(op.src, dst);
|
||||
await fs.chmod(dst, 0o644);
|
||||
}
|
||||
} else if (op.type === 'Write') {
|
||||
await fs.writeFile(dst, op.data);
|
||||
} else if (op.type === 'Symlink') {
|
||||
// A link and not the file it names, in every mode: what
|
||||
// the tar carries is what a directory has to hold too.
|
||||
// symlink() has no truncating open behind it, so the name
|
||||
// is cleared first to keep a rerun over an existing tree
|
||||
// working the way copy and write already do.
|
||||
await fs.rm(dst, {force: true});
|
||||
await fs.symlink(op.target, dst);
|
||||
return this.log.filter(entry => entry.type === 'Op');
|
||||
}
|
||||
|
||||
async run(dir: string, mode: 'link' | 'copy', filter?: (dst: string) => boolean) {
|
||||
for (let entry of this.#ops()) {
|
||||
if (filter !== undefined && !filter(entry.dst))
|
||||
continue;
|
||||
let op = entry.op;
|
||||
let dst = nodePath.join(dir, entry.dst);
|
||||
await fs.mkdir(nodePath.dirname(dst), {recursive: true});
|
||||
if (op.type === 'Copy'){
|
||||
// Read-only sources are CAS objects; their mode must not
|
||||
// leak into deploy trees (rsync -a would ship it), and a
|
||||
// hardlink cannot carry its own mode, so copy those.
|
||||
if (mode === 'link' && ((await fs.stat(op.src)).mode & 0o200) !== 0) {
|
||||
await fs.link(op.src, dst);
|
||||
} else {
|
||||
await fs.copyFile(op.src, dst);
|
||||
await fs.chmod(dst, 0o644);
|
||||
}
|
||||
onFile?.();
|
||||
} else if (op.type === 'Write') {
|
||||
await fs.writeFile(dst, op.data);
|
||||
} else if (op.type === 'Symlink') {
|
||||
// symlink() has no truncating open behind it, so the name
|
||||
// is cleared first to keep a rerun over an existing tree
|
||||
// working the way copy and write already do.
|
||||
await fs.rm(dst, {force: true});
|
||||
await fs.symlink(op.target, dst);
|
||||
}
|
||||
} else {
|
||||
// In this case, I guess its a file rather than a dir.
|
||||
let out = createWriteStream(dir);
|
||||
(await this.pack(filter, onFile)).pipe(out);
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
out.on('error', reject);
|
||||
out.on('finish', () => resolve());
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Entries are handed to the pack as fast as they can be read, but each
|
||||
// one's callback waits on the pack draining into whoever is reading it,
|
||||
// so `onFile` tracks what the consumer has taken rather than what has
|
||||
// been queued for it.
|
||||
async pack(filter?: (dst: string) => boolean, onFile?: () => void)
|
||||
: Promise<NodeJS.ReadableStream> {
|
||||
if (!this.valid)
|
||||
throw new Error(`Invalid ActionQueue`);
|
||||
let t = tar.pack();
|
||||
// An entry's callback also carries the error the consumer is already
|
||||
// reporting; count the entry only where it made it through.
|
||||
let took = onFile === undefined ? undefined : (err?: Error | null) => {
|
||||
if (err === null || err === undefined) {
|
||||
onFile();
|
||||
}
|
||||
// Into a served tree and the directory beside it holding what isn't
|
||||
// served. The tree is add-only: every name in it carries a content hash
|
||||
// and is cached forever, so a name already there is the same file and is
|
||||
// touched, and one there at another size is refused. `meta` is this
|
||||
// publish's own and is written whole; it gets `__files`, the tree names
|
||||
// this publish stands behind, and `__digest`, the caller's digest of
|
||||
// everything published.
|
||||
async publish(tree: string, meta: string, digest: string): Promise<Published> {
|
||||
let ops = this.#ops();
|
||||
let dsts = new Set(ops.map(entry => entry.dst));
|
||||
let files: string[] = [];
|
||||
let placed = 0;
|
||||
let kept = 0;
|
||||
|
||||
let landed = (dst: string): string => {
|
||||
let rest = metaRest(dst);
|
||||
return rest === null ? nodePath.join(tree, dst) : nodePath.join(meta, rest);
|
||||
};
|
||||
for (let entry of this.log) {
|
||||
if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst)))
|
||||
continue;
|
||||
let op = entry.op;
|
||||
if (op.type === 'Symlink') {
|
||||
t.entry({name: entry.dst, type: 'symlink', linkname: op.target}, took)
|
||||
.on('error', () => {});
|
||||
|
||||
for (let entry of ops) {
|
||||
let rest = metaRest(entry.dst);
|
||||
if (rest === '' || rest?.split('/').some(part => part.startsWith('__'))) {
|
||||
throw new BuildError(`${META} names starting with __ are the publish's own: ${entry.dst}`);
|
||||
}
|
||||
if (entry.op.type === 'Symlink') {
|
||||
if (rest === null) {
|
||||
throw new BuildError(`A served tree holds files, not links: ${entry.dst}`);
|
||||
}
|
||||
// The validity check already settled that it resolves.
|
||||
let target = resolveLink(entry.dst, entry.op.target)!;
|
||||
if (!dsts.has(target)) {
|
||||
throw new BuildError(`A link to something this publish doesn't carry: ${entry.dst} -> ${target}`);
|
||||
}
|
||||
let dst = landed(entry.dst);
|
||||
await fs.mkdir(nodePath.dirname(dst), {recursive: true});
|
||||
await fs.rm(dst, {force: true});
|
||||
await fs.symlink(nodePath.relative(nodePath.dirname(dst), landed(target)), dst);
|
||||
continue;
|
||||
}
|
||||
let data = op.type === 'Copy' ? await fs.readFile(op.src) : op.data;
|
||||
// A dying consumer destroys the pack and every pending entry
|
||||
// sink, and each sink emits the error; the consumer is the one
|
||||
// reporting the failure, so keep the sinks quiet.
|
||||
t.entry({name: entry.dst}, data, took).on('error', () => {});
|
||||
let op = entry.op;
|
||||
let dst = landed(entry.dst);
|
||||
if (rest !== null) {
|
||||
await fs.mkdir(nodePath.dirname(dst), {recursive: true});
|
||||
if (op.type === 'Copy') {
|
||||
await fs.copyFile(op.src, dst);
|
||||
await fs.chmod(dst, 0o644);
|
||||
} else {
|
||||
await fs.writeFile(dst, op.data);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
files.push(entry.dst);
|
||||
let size = op.type === 'Copy' ? (await fs.stat(op.src)).size : Buffer.byteLength(op.data);
|
||||
let have = await fs.stat(dst).catch((err: NodeJS.ErrnoException) => {
|
||||
if (err.code === 'ENOENT') {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (have !== null) {
|
||||
if (have.size !== size) {
|
||||
throw new BuildError(`Different file size: ${entry.dst} is already in the tree holding `
|
||||
+ `something else, and a name in there is served as if it can never change.`);
|
||||
}
|
||||
let now = new Date();
|
||||
await fs.utimes(dst, now, now);
|
||||
kept++;
|
||||
continue;
|
||||
}
|
||||
if (op.type === 'Copy') {
|
||||
// A CAS object is read-only and never changes, so the tree can
|
||||
// share it; a source file is copied, since the checkout it
|
||||
// sits in moves on.
|
||||
let src = op.src;
|
||||
let linked = false;
|
||||
if (((await fs.stat(src)).mode & 0o200) === 0) {
|
||||
await fs.mkdir(nodePath.dirname(dst), {recursive: true});
|
||||
try {
|
||||
await fs.link(src, dst);
|
||||
linked = true;
|
||||
} catch (err) {
|
||||
// Another filesystem: a copy, like a source
|
||||
if ((err as NodeJS.ErrnoException).code !== 'EXDEV') {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!linked) {
|
||||
await place(dst, async tmp => {
|
||||
await fs.copyFile(src, tmp);
|
||||
await fs.chmod(tmp, 0o644);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
let data = op.data;
|
||||
await place(dst, tmp => fs.writeFile(tmp, data, {mode: 0o644}));
|
||||
}
|
||||
placed++;
|
||||
}
|
||||
// Without this the archive has no end-of-archive marker, and strict
|
||||
// readers (Python tarfile in stream mode) die on the truncation.
|
||||
t.finalize();
|
||||
return t;
|
||||
|
||||
await fs.mkdir(meta, {recursive: true});
|
||||
await fs.writeFile(nodePath.join(meta, '__files'), files.sort().map(f => `${f}\n`).join(''));
|
||||
await fs.writeFile(nodePath.join(meta, '__digest'), `${digest}\n`);
|
||||
return {files: files.length, placed, kept};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,16 +4,15 @@ import {createHash} from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as pathlib from 'node:path';
|
||||
import * as stream from 'node:stream';
|
||||
import {beforeEach, test} from 'node:test';
|
||||
|
||||
import b32encode from 'base32-encode';
|
||||
import tar from 'tar-stream';
|
||||
|
||||
import {resetDecls, rule} from '../../build/artifact.ts';
|
||||
import {casPath} from '../../build/cas.ts';
|
||||
import {makeCtx} from '../api.ts';
|
||||
import {ActionQueue} from '../queue.ts';
|
||||
import {BuildError} from '../../build/errors.ts';
|
||||
|
||||
beforeEach(resetDecls);
|
||||
|
||||
@@ -33,6 +32,7 @@ function makeArtifact(casDir: string, content: string, ext: string) {
|
||||
let obj = casPath(casDir, digest, ext);
|
||||
fs.mkdirSync(pathlib.dirname(obj), {recursive: true});
|
||||
fs.writeFileSync(obj, content);
|
||||
fs.chmodSync(obj, 0o444);
|
||||
return artifact;
|
||||
}
|
||||
|
||||
@@ -89,91 +89,16 @@ test('ctx.list sorts, parses extensions, skips dotfiles and directories', async
|
||||
]);
|
||||
});
|
||||
|
||||
type PackedEntry = {name: string, data: string, type?: string, linkname?: string};
|
||||
|
||||
async function packedEntries(aq: ActionQueue, filter?: (dst: string) => boolean)
|
||||
: Promise<PackedEntry[]> {
|
||||
let packed = await aq.pack(filter);
|
||||
return new Promise((resolve, reject) => {
|
||||
let extract = tar.extract();
|
||||
let entries: PackedEntry[] = [];
|
||||
extract.on('entry', (header, stream, next) => {
|
||||
let chunks: Buffer[] = [];
|
||||
stream.on('data', c => chunks.push(c));
|
||||
stream.on('end', () => {
|
||||
let entry: PackedEntry = {name: header.name, data: Buffer.concat(chunks).toString()};
|
||||
if (header.type === 'symlink') {
|
||||
entry.type = header.type;
|
||||
entry.linkname = header.linkname ?? undefined;
|
||||
}
|
||||
entries.push(entry);
|
||||
next();
|
||||
});
|
||||
});
|
||||
extract.on('finish', () => resolve(entries));
|
||||
extract.on('error', reject);
|
||||
packed.pipe(extract);
|
||||
});
|
||||
}
|
||||
|
||||
test('pack preserves op order', async () => {
|
||||
let aq = new ActionQueue();
|
||||
aq.write('zzz', 'z.txt');
|
||||
aq.write('aaa', 'a.txt');
|
||||
assert.deepEqual(await packedEntries(aq), [
|
||||
{name: 'z.txt', data: 'zzz'},
|
||||
{name: 'a.txt', data: 'aaa'},
|
||||
]);
|
||||
});
|
||||
|
||||
test('pack with a filter packs only matching entries in order', async () => {
|
||||
let aq = new ActionQueue();
|
||||
aq.write('1', 'xy/a.png');
|
||||
aq.write('2', 'meta/m.json');
|
||||
aq.write('3', 'xy/b.png');
|
||||
let subset = await packedEntries(aq, dst => dst.startsWith('xy/'));
|
||||
assert.deepEqual(subset, [
|
||||
{name: 'xy/a.png', data: '1'},
|
||||
{name: 'xy/b.png', data: '3'},
|
||||
]);
|
||||
});
|
||||
|
||||
test('pack counts an entry for the progress bar as the consumer takes it', async () => {
|
||||
let aq = new ActionQueue();
|
||||
for (let i = 0; i < 4; i++) {
|
||||
aq.write('x'.repeat(50_000), `f${i}.txt`);
|
||||
}
|
||||
let counted = 0;
|
||||
let packed = await aq.pack(undefined, () => counted++);
|
||||
// Nothing has read the pack yet: an entry queued is not an entry shipped.
|
||||
assert.equal(counted, 0);
|
||||
let sink = new stream.Writable({highWaterMark: 1, write: (_c, _e, cb) => setImmediate(cb)});
|
||||
await new Promise((resolve, reject) => {
|
||||
sink.on('finish', resolve);
|
||||
sink.on('error', reject);
|
||||
packed.pipe(sink);
|
||||
});
|
||||
assert.equal(counted, 4);
|
||||
});
|
||||
|
||||
test('run counts each materialized entry for the progress bar, filter and all', async () => {
|
||||
let dir = tmpdir();
|
||||
let aq = new ActionQueue();
|
||||
aq.write('1', 'ani/a.gif');
|
||||
aq.symlink('a.gif', 'ani/b.gif');
|
||||
aq.write('2', 'dex/c.png');
|
||||
let counted: number[] = [];
|
||||
await aq.run(pathlib.join(dir, 'deploy'), 'copy', dst => dst.startsWith('ani/'),
|
||||
() => counted.push(counted.length + 1));
|
||||
assert.deepEqual(counted, [1, 2]);
|
||||
});
|
||||
|
||||
test('duplicate and absolute destinations invalidate the queue', async () => {
|
||||
let dup = new ActionQueue();
|
||||
dup.write('a', 'x.txt');
|
||||
dup.write('b', 'x.txt');
|
||||
assert.ok(!dup.valid);
|
||||
await assert.rejects(() => dup.pack(), /Invalid ActionQueue/);
|
||||
await assert.rejects(() => dup.run(tmpdir(), 'copy'), /Invalid ActionQueue/);
|
||||
|
||||
let up = new ActionQueue();
|
||||
up.write('a', 'x/../../y.txt');
|
||||
assert.ok(!up.valid);
|
||||
|
||||
let abs = new ActionQueue();
|
||||
abs.write('a', '/etc/passwd');
|
||||
@@ -203,16 +128,6 @@ test('copy-mode materialization restores 0644 on read-only sources', async () =>
|
||||
assert.equal(fs.statSync(pathlib.join(out, 'out/x.png')).mode & 0o777, 0o644);
|
||||
});
|
||||
|
||||
test('pack carries a symlink as a link, not the bytes it names', async () => {
|
||||
let aq = new ActionQueue();
|
||||
aq.write('1', 'links/xy/a-XXXX.gif');
|
||||
aq.symlink('a-XXXX.gif', 'links/xy/a.gif');
|
||||
assert.deepEqual(await packedEntries(aq), [
|
||||
{name: 'links/xy/a-XXXX.gif', data: '1'},
|
||||
{name: 'links/xy/a.gif', data: '', type: 'symlink', linkname: 'a-XXXX.gif'},
|
||||
]);
|
||||
});
|
||||
|
||||
test('run materializes a symlink resolving beside its target', async () => {
|
||||
let dir = tmpdir();
|
||||
let aq = new ActionQueue();
|
||||
@@ -262,3 +177,75 @@ test('ctx.symlink writes the path from the link to what it names', async () => {
|
||||
'../sprites/a-XXXX.gif',
|
||||
]);
|
||||
});
|
||||
|
||||
// A tree and a meta directory under one scratch root, with a CAS beside them.
|
||||
function publishDirs() {
|
||||
let dir = tmpdir();
|
||||
return {dir, tree: pathlib.join(dir, 'tree'), meta: pathlib.join(dir, 'meta'), cas: pathlib.join(dir, 'cas')};
|
||||
}
|
||||
|
||||
test('publish places the tree, retargets links into it and records what it stands behind', async () => {
|
||||
let {dir, tree, meta, cas} = publishDirs();
|
||||
let art = makeArtifact(cas, 'gif bytes', 'gif');
|
||||
let src = pathlib.join(dir, 'src.png');
|
||||
fs.writeFileSync(src, 'png bytes');
|
||||
let aq = new ActionQueue();
|
||||
let ctx = makeCtx(cas, aq);
|
||||
ctx.copy(art, 'xy/a-HASH.gif');
|
||||
ctx.copy(src, 'xy/b-HASH.png');
|
||||
ctx.write('spritesheet-HASH.css', 'css');
|
||||
ctx.write('__meta/xy/manifest.json', '{}');
|
||||
ctx.symlink('__meta/links/xy/a.gif', 'xy/a-HASH.gif');
|
||||
ctx.symlink('__meta/other.json', '__meta/xy/manifest.json');
|
||||
assert.deepEqual(await aq.publish(tree, meta, 'd1g3st'), {files: 3, placed: 3, kept: 0});
|
||||
|
||||
// The CAS object is shared, the source is copied, the write is a file
|
||||
assert.equal(fs.statSync(pathlib.join(tree, 'xy/a-HASH.gif')).ino, fs.statSync(casPath(cas, art.hash, 'gif')).ino);
|
||||
assert.notEqual(fs.statSync(pathlib.join(tree, 'xy/b-HASH.png')).ino, fs.statSync(src).ino);
|
||||
assert.equal(fs.statSync(pathlib.join(tree, 'xy/b-HASH.png')).mode & 0o777, 0o644);
|
||||
assert.equal(fs.readFileSync(pathlib.join(tree, 'spritesheet-HASH.css'), 'utf8'), 'css');
|
||||
assert.equal(fs.readFileSync(pathlib.join(meta, 'xy/manifest.json'), 'utf8'), '{}');
|
||||
// Both ends landed apart; the link follows the file into the tree
|
||||
let link = pathlib.join(meta, 'links/xy/a.gif');
|
||||
assert.equal(fs.readlinkSync(link), '../../../tree/xy/a-HASH.gif');
|
||||
assert.equal(fs.readFileSync(link, 'utf8'), 'gif bytes');
|
||||
assert.equal(fs.readFileSync(pathlib.join(meta, 'other.json'), 'utf8'), '{}');
|
||||
assert.equal(fs.readFileSync(pathlib.join(meta, '__files'), 'utf8'),
|
||||
'spritesheet-HASH.css\nxy/a-HASH.gif\nxy/b-HASH.png\n');
|
||||
assert.equal(fs.readFileSync(pathlib.join(meta, '__digest'), 'utf8'), 'd1g3st\n');
|
||||
assert.ok(!fs.readdirSync(pathlib.join(tree, 'xy')).some(name => name.startsWith('.')), 'no temp files left');
|
||||
});
|
||||
|
||||
test('publish keeps what the tree has and refuses a name holding something else', async () => {
|
||||
let {tree, meta, cas} = publishDirs();
|
||||
fs.mkdirSync(pathlib.join(tree, 'xy'), {recursive: true});
|
||||
fs.writeFileSync(pathlib.join(tree, 'xy/a-HASH.gif'), 'same size');
|
||||
let old = new Date(2020, 0, 1);
|
||||
fs.utimesSync(pathlib.join(tree, 'xy/a-HASH.gif'), old, old);
|
||||
let aq = new ActionQueue();
|
||||
aq.write('SAME SIZE', 'xy/a-HASH.gif');
|
||||
assert.deepEqual(await aq.publish(tree, meta, 'd'), {files: 1, placed: 0, kept: 1});
|
||||
assert.equal(fs.readFileSync(pathlib.join(tree, 'xy/a-HASH.gif'), 'utf8'), 'same size', 'not rewritten');
|
||||
assert.ok(fs.statSync(pathlib.join(tree, 'xy/a-HASH.gif')).mtime > old, 'touched');
|
||||
|
||||
let other = new ActionQueue();
|
||||
other.write('a different length', 'xy/a-HASH.gif');
|
||||
await assert.rejects(() => other.publish(tree, pathlib.join(meta, '2'), 'd'), /Different file size/);
|
||||
void cas;
|
||||
});
|
||||
|
||||
test('publish refuses links in the tree, links to what it does not carry, and __ meta names', async () => {
|
||||
let {tree, meta} = publishDirs();
|
||||
let inTree = new ActionQueue();
|
||||
inTree.write('a', 'xy/a-HASH.gif');
|
||||
inTree.symlink('a-HASH.gif', 'xy/a.gif');
|
||||
await assert.rejects(() => inTree.publish(tree, meta, 'd'), BuildError);
|
||||
|
||||
let dangling = new ActionQueue();
|
||||
dangling.symlink('../../xy/a-HASH.gif', '__meta/links/xy/a.gif');
|
||||
await assert.rejects(() => dangling.publish(tree, meta, 'd'), /doesn't carry/);
|
||||
|
||||
let reserved = new ActionQueue();
|
||||
reserved.write('a', '__meta/__files');
|
||||
await assert.rejects(() => reserved.publish(tree, meta, 'd'), /publish's own/);
|
||||
});
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as pathlib from 'node:path';
|
||||
import {test} from 'node:test';
|
||||
|
||||
import {loadDeployConfig, matchSubsets} from '../config.ts';
|
||||
|
||||
function configFile(text: string): string {
|
||||
let dir = fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-config-test-'));
|
||||
let p = pathlib.join(dir, 'deploy.json5');
|
||||
fs.writeFileSync(p, text);
|
||||
return p;
|
||||
}
|
||||
|
||||
test('loadDeployConfig parses json5 with comments and trailing commas', async () => {
|
||||
let config = await loadDeployConfig(configFile(`{
|
||||
// dex assets
|
||||
assets: {
|
||||
buildFile: "assets.build.ts",
|
||||
deploy: [
|
||||
{subset: ["**"], cmd: "cat > /dev/null"},
|
||||
],
|
||||
},
|
||||
}`));
|
||||
assert.deepEqual([...config.keys()], ['assets']);
|
||||
assert.deepEqual(config.get('assets'), {
|
||||
buildFile: 'assets.build.ts',
|
||||
deploy: [{subset: ['**'], cmd: 'cat > /dev/null'}],
|
||||
});
|
||||
});
|
||||
|
||||
test('loadDeployConfig ties the dir flag to %d in the cmd', async () => {
|
||||
let dir = await loadDeployConfig(configFile(
|
||||
'{ps: {buildFile: "ps.build.ts", deploy: [{subset: ["ani/**"], dir: true, cmd: "rsync -a %d/ani/ h:a/"}]}}'));
|
||||
assert.equal(dir.get('ps')!.deploy[0]!.dir, true);
|
||||
await assert.rejects(() => loadDeployConfig(configFile(
|
||||
'{a: {buildFile: "x.build.ts", deploy: [{subset: ["**"], dir: true, cmd: "rsync -a h:a/"}]}}')),
|
||||
/dir entry's cmd must use %d/);
|
||||
await assert.rejects(() => loadDeployConfig(configFile(
|
||||
'{a: {buildFile: "x.build.ts", deploy: [{subset: ["**"], cmd: "rsync -a %d/ h:a/"}]}}')),
|
||||
/must not use %d \(missing dir: true\?\)/);
|
||||
});
|
||||
|
||||
test('loadDeployConfig rejects missing files and malformed shapes', async () => {
|
||||
await assert.rejects(() => loadDeployConfig('/nonexistent/deploy.json5'), /missing/);
|
||||
await assert.rejects(() => loadDeployConfig(configFile('[1]')), /must be an object/);
|
||||
await assert.rejects(() => loadDeployConfig(configFile('{a: {deploy: []}}')), /expected \{buildFile/);
|
||||
await assert.rejects(() => loadDeployConfig(configFile(
|
||||
'{a: {buildFile: "x.build.ts", deploy: [{subset: "**", cmd: "c"}]}}')),
|
||||
/needs \{subset/);
|
||||
await assert.rejects(() => loadDeployConfig(configFile('{a: {buildFile:')), /deploy.json5:/);
|
||||
});
|
||||
|
||||
test('matchSubsets routes dsts to entries, overlap allowed', () => {
|
||||
let dsts = ['xy/a.gif', 'xy/manifest.json', 'xyicons/b.png'];
|
||||
let [xy, manifests] = matchSubsets(dsts, [
|
||||
{subset: ['xy/**', 'xyicons/**'], cmd: 'one'},
|
||||
{subset: ['**/manifest.json'], cmd: 'two'},
|
||||
]);
|
||||
assert.deepEqual([...xy!.matched].sort(), dsts);
|
||||
assert.equal(xy!.entry.cmd, 'one');
|
||||
assert.deepEqual([...manifests!.matched], ['xy/manifest.json']);
|
||||
});
|
||||
|
||||
test('matchSubsets errors on a glob matching nothing', () => {
|
||||
assert.throws(() => matchSubsets(['xy/a.gif'], [{subset: ['xy/**', 'zz/**'], cmd: 'c'}]),
|
||||
/matches no outputs: zz\/\*\*/);
|
||||
});
|
||||
|
||||
test('matchSubsets errors on uncovered outputs, listing them', () => {
|
||||
assert.throws(() => matchSubsets(['xy/a.gif', 'stray.txt'], [{subset: ['xy/**'], cmd: 'c'}]),
|
||||
/not covered by any deploy entry:\n {2}stray.txt/);
|
||||
});
|
||||
@@ -1,76 +0,0 @@
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {test} from 'node:test';
|
||||
|
||||
import {Progress, type Term} from '../progress.ts';
|
||||
|
||||
function term(overrides: Partial<Term> = {}): Term & {written: string[]} {
|
||||
let written: string[] = [];
|
||||
return {isTTY: true, columns: 80, write: (s: string) => written.push(s), written, ...overrides};
|
||||
}
|
||||
|
||||
// Every write the bar makes, with the clear-and-return prefix stripped.
|
||||
function lines(t: {written: string[]}): string[] {
|
||||
return t.written.map(s => s.replace('\x1b[2K\r', ''));
|
||||
}
|
||||
|
||||
test('a bar draws nothing when stderr is not a terminal', () => {
|
||||
let t = term({isTTY: false});
|
||||
let bar = new Progress('deploy: uploading', 4, t);
|
||||
for (let i = 0; i < 4; i++) {
|
||||
bar.tick();
|
||||
}
|
||||
bar.finish();
|
||||
assert.deepEqual(t.written, []);
|
||||
});
|
||||
|
||||
test('a bar draws its label, a filled share of the width, and the counts', () => {
|
||||
let t = term();
|
||||
let bar = new Progress('up', 100, t);
|
||||
// The timer holds the redraws back, so drive it to the end and read the
|
||||
// first draw, which the constructor always makes.
|
||||
let [first] = lines(t);
|
||||
assert.equal(first, `up [${'-'.repeat(66)}] 0/100`);
|
||||
assert.equal(first!.length, 79);
|
||||
for (let i = 0; i < 50; i++) {
|
||||
bar.tick();
|
||||
}
|
||||
assert.equal(lines(t).length, 1);
|
||||
});
|
||||
|
||||
test('a bar never writes past the terminal width', () => {
|
||||
for (let columns of [20, 40, 80, 200]) {
|
||||
let t = term({columns});
|
||||
new Progress('deploying something', 1000, t);
|
||||
for (let line of lines(t)) {
|
||||
assert.ok(line.length < columns, `${line.length} >= ${columns} at ${columns} columns`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('a narrow terminal gets the counts without a bar', () => {
|
||||
let t = term({columns: 24});
|
||||
new Progress('up', 1000, t);
|
||||
assert.deepEqual(lines(t), ['up 0/1000']);
|
||||
});
|
||||
|
||||
test('a bar clears its line once it is full, leaving the rest of the wait alone', () => {
|
||||
let t = term();
|
||||
let bar = new Progress('up', 2, t);
|
||||
bar.tick();
|
||||
bar.tick();
|
||||
assert.equal(t.written.at(-1), '\x1b[2K\r');
|
||||
assert.equal(lines(t).at(-1), '');
|
||||
// Already cleared: finishing again is not a second clear.
|
||||
let count = t.written.length;
|
||||
bar.finish();
|
||||
assert.equal(t.written.length, count);
|
||||
});
|
||||
|
||||
test('finishing a bar mid-run clears the line it drew', () => {
|
||||
let t = term();
|
||||
let bar = new Progress('up', 100, t);
|
||||
bar.tick();
|
||||
bar.finish();
|
||||
assert.equal(t.written.at(-1), '\x1b[2K\r');
|
||||
});
|
||||
@@ -1,212 +0,0 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import {spawn} from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as pathlib from 'node:path';
|
||||
import {test} from 'node:test';
|
||||
|
||||
import {MAGIC, Reply, manifestOf, negotiate, type Manifest} from '../protocol.ts';
|
||||
import {ActionQueue} from '../queue.ts';
|
||||
|
||||
let here = pathlib.dirname(new URL(import.meta.url).pathname);
|
||||
let TIMEOUT = 10_000;
|
||||
|
||||
function tmpdir(): string {
|
||||
return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'deploy-protocol-test-'));
|
||||
}
|
||||
|
||||
// A queue shaped like a small deploy: two stamped assets, one of them named
|
||||
// awkwardly, a manifest under __meta/ and a link mirroring an asset.
|
||||
async function sampleQueue(dir: string): Promise<{aq: ActionQueue, manifest: Manifest}> {
|
||||
let aq = new ActionQueue();
|
||||
let src = pathlib.join(dir, 'a.gif');
|
||||
fs.writeFileSync(src, 'gif bytes');
|
||||
aq.copy(src, 'sprites/xy/a-HASH.gif');
|
||||
aq.write('héllo', 'sprites/xy/b c\td-HASH.txt');
|
||||
aq.write('{}', '__meta/xy/manifest.json');
|
||||
aq.symlink('../../sprites/xy/a-HASH.gif', '__meta/links/xy/a.gif');
|
||||
return {aq, manifest: await manifestOf(aq)};
|
||||
}
|
||||
|
||||
function u32(n: number): Buffer {
|
||||
let b = Buffer.alloc(4);
|
||||
b.writeUInt32BE(n);
|
||||
return b;
|
||||
}
|
||||
|
||||
function reply(indices: number[], version = 1): Buffer {
|
||||
return Buffer.concat([MAGIC, u32(version), u32(indices.length), ...indices.map(u32)]);
|
||||
}
|
||||
|
||||
test('manifestOf spells the wire format, in queue order, sizes as the tar would carry them', async () => {
|
||||
let {manifest} = await sampleQueue(tmpdir());
|
||||
assert.deepEqual(manifest.entries,
|
||||
['sprites/xy/a-HASH.gif', 'sprites/xy/b c\td-HASH.txt', '__meta/xy/manifest.json', '__meta/links/xy/a.gif']);
|
||||
assert.deepEqual([...manifest.assets], [0, 1]);
|
||||
assert.deepEqual([...manifest.metaFiles], ['__meta/xy/manifest.json']);
|
||||
|
||||
let b = manifest.bytes;
|
||||
assert.ok(b.subarray(0, 16).equals(MAGIC));
|
||||
assert.equal(b.readUInt32BE(16), 1, 'version');
|
||||
assert.equal(b.readUInt32BE(20), 4, 'count');
|
||||
let at = 24;
|
||||
let expectFile = (kind: number, size: number, name: string) => {
|
||||
assert.equal(b[at], kind, `kind of ${name}`);
|
||||
assert.equal(Number(b.readBigUInt64BE(at + 1)), size, `size of ${name}`);
|
||||
let raw = Buffer.from(name);
|
||||
assert.equal(b.readUInt32BE(at + 9), raw.length, `name length of ${name}`);
|
||||
assert.ok(b.subarray(at + 13, at + 13 + raw.length).equals(raw), `name of ${name}`);
|
||||
at += 13 + raw.length;
|
||||
};
|
||||
expectFile(1, 'gif bytes'.length, 'sprites/xy/a-HASH.gif');
|
||||
// Six characters, seven bytes: the size is the utf-8's, since that is what
|
||||
// tar-stream writes, and the tab rides inside the length-prefixed name.
|
||||
expectFile(1, 6, 'sprites/xy/b c\td-HASH.txt');
|
||||
expectFile(2, 2, '__meta/xy/manifest.json');
|
||||
assert.equal(b[at], 3, 'a link');
|
||||
let name = Buffer.from('__meta/links/xy/a.gif');
|
||||
let target = Buffer.from('../../sprites/xy/a-HASH.gif');
|
||||
assert.equal(b.readUInt32BE(at + 1), name.length);
|
||||
assert.ok(b.subarray(at + 5, at + 5 + name.length).equals(name));
|
||||
at += 5 + name.length;
|
||||
assert.equal(b.readUInt32BE(at), target.length);
|
||||
assert.ok(b.subarray(at + 4, at + 4 + target.length).equals(target));
|
||||
assert.equal(at + 4 + target.length, b.length, 'nothing after the last entry');
|
||||
});
|
||||
|
||||
test('manifestOf takes only what the filter selects, positions renumbered', async () => {
|
||||
let {aq} = await sampleQueue(tmpdir());
|
||||
let manifest = await manifestOf(aq, dst => dst.startsWith('__meta/'));
|
||||
assert.deepEqual(manifest.entries, ['__meta/xy/manifest.json', '__meta/links/xy/a.gif']);
|
||||
assert.equal(manifest.assets.size, 0);
|
||||
assert.equal(manifest.bytes.readUInt32BE(20), 2);
|
||||
});
|
||||
|
||||
test('Reply reads a reply however it is chunked and relays the report after it', async () => {
|
||||
let {manifest} = await sampleQueue(tmpdir());
|
||||
let bytes = Buffer.concat([reply([1, 0]), Buffer.from('* 2 files\n')]);
|
||||
for (let step of [bytes.length, 1, 7]) {
|
||||
let relayed: Buffer[] = [];
|
||||
let r = new Reply(manifest, chunk => relayed.push(chunk));
|
||||
for (let i = 0; i < bytes.length; i += step) {
|
||||
r.feed(bytes.subarray(i, i + step));
|
||||
}
|
||||
assert.equal(r.outcome, 'ok', `chunked by ${step}`);
|
||||
assert.deepEqual([...r.wanted].sort(), ['sprites/xy/a-HASH.gif', 'sprites/xy/b c\td-HASH.txt']);
|
||||
assert.equal(Buffer.concat(relayed).toString(), '* 2 files\n', `report chunked by ${step}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('Reply refuses anything that is not a reply, and relays it whole', async () => {
|
||||
let {manifest} = await sampleQueue(tmpdir());
|
||||
|
||||
// Text where the magic should be: a refusal, a shell profile, a filter
|
||||
let relayed: Buffer[] = [];
|
||||
let r = new Reply(manifest, chunk => relayed.push(chunk));
|
||||
r.feed(Buffer.from('No such asset set: sprites\n'));
|
||||
assert.equal(r.outcome, 'refused');
|
||||
assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites\n');
|
||||
// And what comes after goes through too
|
||||
r.feed(Buffer.from('more\n'));
|
||||
assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites\nmore\n');
|
||||
|
||||
// Even where it starts out looking right
|
||||
r = new Reply(manifest, () => {});
|
||||
r.feed(Buffer.from('smogonctl asset'));
|
||||
assert.equal(r.outcome, 'pending', 'a prefix of the magic is still possible');
|
||||
r.feed(Buffer.from('!'));
|
||||
assert.equal(r.outcome, 'refused');
|
||||
|
||||
r = new Reply(manifest, () => {});
|
||||
r.feed(reply([0], 2));
|
||||
assert.equal(r.outcome, 'refused');
|
||||
assert.match(r.error!, /version 2/);
|
||||
|
||||
r = new Reply(manifest, () => {});
|
||||
r.feed(reply([7]));
|
||||
assert.equal(r.outcome, 'refused');
|
||||
assert.match(r.error!, /position 7/);
|
||||
|
||||
r = new Reply(manifest, () => {});
|
||||
r.feed(reply([2]));
|
||||
assert.equal(r.outcome, 'refused', 'a __meta/ file is not something to ask for');
|
||||
|
||||
r = new Reply(manifest, () => {});
|
||||
r.feed(reply([0, 0]));
|
||||
assert.equal(r.outcome, 'refused');
|
||||
assert.match(r.error!, /twice/);
|
||||
|
||||
// Hung up partway through the header, and partway through the positions
|
||||
r = new Reply(manifest, () => {});
|
||||
r.feed(reply([0, 1]).subarray(0, 10));
|
||||
r.end();
|
||||
assert.equal(r.outcome, 'refused');
|
||||
assert.match(r.error!, /hung up/);
|
||||
r = new Reply(manifest, () => {});
|
||||
r.feed(reply([0, 1]).subarray(0, 26));
|
||||
r.end();
|
||||
assert.equal(r.outcome, 'refused');
|
||||
|
||||
// An empty answer is an answer
|
||||
r = new Reply(manifest, () => {});
|
||||
r.feed(reply([]));
|
||||
assert.equal(r.outcome, 'ok');
|
||||
assert.equal(r.wanted.size, 0);
|
||||
});
|
||||
|
||||
function run(cmd: string, args: string[]) {
|
||||
let child = spawn(cmd, args, {stdio: ['pipe', 'pipe', 'inherit']});
|
||||
child.stdin!.on('error', () => {});
|
||||
let exit = new Promise<number | null>(resolve => child.on('close', resolve));
|
||||
return {child, exit};
|
||||
}
|
||||
|
||||
test('negotiate against a command that answers: only what it asked for travels', {timeout: TIMEOUT}, async () => {
|
||||
let dir = tmpdir();
|
||||
let {aq, manifest} = await sampleQueue(dir);
|
||||
let out = pathlib.join(dir, 'seen.json');
|
||||
let {child, exit} = run(process.execPath, [pathlib.join(here, 'servers/answer.ts'), out]);
|
||||
let relayed: Buffer[] = [];
|
||||
let result = await negotiate(child, manifest, chunk => relayed.push(chunk));
|
||||
assert.ok('wanted' in result, `should be answered, got ${JSON.stringify(result)}`);
|
||||
// Every other asset: position 0 and not 1
|
||||
assert.deepEqual([...result.wanted], ['sprites/xy/a-HASH.gif']);
|
||||
let send = new Set([...result.wanted, ...manifest.metaFiles]);
|
||||
(await aq.pack(dst => send.has(dst))).pipe(child.stdin!);
|
||||
assert.equal(await exit, 0);
|
||||
let seen = JSON.parse(fs.readFileSync(out, 'utf8'));
|
||||
assert.deepEqual(seen.manifest.map((e: {name: string}) => e.name), manifest.entries);
|
||||
assert.deepEqual(seen.received, [
|
||||
{name: 'sprites/xy/a-HASH.gif', size: 9},
|
||||
{name: '__meta/xy/manifest.json', size: 2},
|
||||
]);
|
||||
assert.equal(Buffer.concat(relayed).toString(), '* 2 received\n', 'the report after the reply is relayed');
|
||||
});
|
||||
|
||||
test('negotiate against a command that refuses: relayed, stdin closed, its exit code kept', {timeout: TIMEOUT}, async () => {
|
||||
let {manifest} = await sampleQueue(tmpdir());
|
||||
let {child, exit} = run(process.execPath, [pathlib.join(here, 'servers/refuse.ts')]);
|
||||
let relayed: Buffer[] = [];
|
||||
let result = await negotiate(child, manifest, chunk => relayed.push(chunk));
|
||||
assert.ok('error' in result);
|
||||
assert.equal(Buffer.concat(relayed).toString(), 'No such asset set: sprites.\n');
|
||||
assert.equal(await exit, 3);
|
||||
});
|
||||
|
||||
test('negotiate against cat: an echo of the manifest is not a reply', {timeout: TIMEOUT}, async () => {
|
||||
let {manifest} = await sampleQueue(tmpdir());
|
||||
let {child, exit} = run('cat', []);
|
||||
let result = await negotiate(child, manifest, () => {});
|
||||
assert.ok('error' in result, 'the manifest coming back starts with the magic but is no reply');
|
||||
// stdin was closed for it, so cat finishes instead of waiting
|
||||
assert.equal(await exit, 0);
|
||||
});
|
||||
|
||||
test('negotiate against a command that says nothing: refused when it hangs up', {timeout: TIMEOUT}, async () => {
|
||||
let {manifest} = await sampleQueue(tmpdir());
|
||||
let {child, exit} = run('true', []);
|
||||
let result = await negotiate(child, manifest, () => {});
|
||||
assert.ok('error' in result);
|
||||
assert.match(result.error!, /hung up|didn't answer/);
|
||||
await exit;
|
||||
});
|
||||
@@ -1,94 +0,0 @@
|
||||
// A stand-in for `smogonctl assets upload`: reads the manifest, asks for every
|
||||
// other asset, takes the tar, and writes what it saw to the path in argv[2] as
|
||||
// JSON. Prints one line of report after the reply, the way the real one does.
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import tar from 'tar-stream';
|
||||
|
||||
let MAGIC = Buffer.from('smogonctl assets');
|
||||
let out = process.argv[2]!;
|
||||
|
||||
let buf = Buffer.alloc(0);
|
||||
let pending: {n: number, resolve: (b: Buffer) => void} | null = null;
|
||||
let ended = false;
|
||||
|
||||
process.stdin.on('data', (chunk: Buffer) => {
|
||||
buf = Buffer.concat([buf, chunk]);
|
||||
serve();
|
||||
});
|
||||
process.stdin.on('end', () => {
|
||||
ended = true;
|
||||
serve();
|
||||
});
|
||||
|
||||
function serve() {
|
||||
if (pending !== null && buf.length >= pending.n) {
|
||||
let {n, resolve} = pending;
|
||||
pending = null;
|
||||
let head = buf.subarray(0, n);
|
||||
buf = buf.subarray(n);
|
||||
resolve(head);
|
||||
} else if (pending !== null && ended) {
|
||||
throw new Error('manifest ended early');
|
||||
}
|
||||
}
|
||||
|
||||
function take(n: number): Promise<Buffer> {
|
||||
return new Promise(resolve => {
|
||||
pending = {n, resolve};
|
||||
serve();
|
||||
});
|
||||
}
|
||||
|
||||
async function name(): Promise<string> {
|
||||
let len = (await take(4)).readUInt32BE(0);
|
||||
return (await take(len)).toString();
|
||||
}
|
||||
|
||||
let header = await take(24);
|
||||
if (!header.subarray(0, 16).equals(MAGIC)) {
|
||||
throw new Error('not a manifest');
|
||||
}
|
||||
let count = header.readUInt32BE(20);
|
||||
let manifest: {kind: number, name: string, size?: number, target?: string}[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
let kind = (await take(1))[0]!;
|
||||
if (kind === 3) {
|
||||
manifest.push({kind, name: await name(), target: await name()});
|
||||
} else {
|
||||
let size = Number((await take(8)).readBigUInt64BE(0));
|
||||
manifest.push({kind, name: await name(), size});
|
||||
}
|
||||
}
|
||||
let asked = manifest.map((e, i) => i).filter(i => manifest[i]!.kind === 1 && i % 2 === 0);
|
||||
let reply = Buffer.alloc(24 + 4 * asked.length);
|
||||
MAGIC.copy(reply);
|
||||
reply.writeUInt32BE(1, 16);
|
||||
reply.writeUInt32BE(asked.length, 20);
|
||||
asked.forEach((index, i) => reply.writeUInt32BE(index, 24 + 4 * i));
|
||||
process.stdout.write(reply);
|
||||
|
||||
// Whatever follows is the tar, or nothing at all.
|
||||
let received: {name: string, size: number}[] = [];
|
||||
let extract = tar.extract();
|
||||
extract.on('entry', (header, stream, next) => {
|
||||
received.push({name: header.name, size: header.size ?? 0});
|
||||
stream.on('end', next);
|
||||
stream.resume();
|
||||
});
|
||||
let done = new Promise<void>((resolve, reject) => {
|
||||
extract.on('finish', resolve);
|
||||
extract.on('error', reject);
|
||||
});
|
||||
if (buf.length > 0) {
|
||||
extract.write(buf);
|
||||
}
|
||||
buf = Buffer.alloc(0);
|
||||
if (ended) {
|
||||
extract.end();
|
||||
} else {
|
||||
process.stdin.pipe(extract);
|
||||
}
|
||||
await done;
|
||||
fs.writeFileSync(out, JSON.stringify({manifest, asked, received}));
|
||||
process.stdout.write(`* ${received.length} received\n`);
|
||||
Reference in New Issue
Block a user