diff --git a/README.md b/README.md index 1bf68f96..395ec656 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,12 @@ covered by some entry. `deploy -o ` materializes each entry's subset under `///` instead of running its command, for eyeballing what would ship. +Where stderr is a terminal, each entry draws a progress bar over its files +while they go out -- an entry counts once the command has taken it, not once +it has been read off disk -- and the line is cleared again afterwards. Piped +or under CI nothing is drawn, and the lines the deploy prints are the same +either way. + This file is not committed, because it is where the hosts and paths this repo ships to are written down. diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index 68a35bb1..fe2bcf82 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -18,6 +18,7 @@ import {setConfig} from '../build/helpers.ts'; import * as db from '../build/store.ts'; import * as api from './api.ts'; import {loadDeployConfig, matchSubsets} from './config.ts'; +import {withProgress} from './progress.ts'; import {ActionQueue} from './queue.ts'; let root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..'); @@ -265,7 +266,8 @@ async function cmdDeploy(names: string[], opts: VerbOpts): Promise { // entries included) instead of running its command. if (opts.output !== undefined) { let dir = nodePath.join(opts.output, name, String(i)); - await aq.run(dir, 'copy', dst => matched.has(dst)); + await withProgress(`${name}: copying`, matched.size, + tick => aq.run(dir, 'copy', dst => matched.has(dst), tick)); console.log(` -> ${dir}`); continue; } @@ -273,7 +275,8 @@ async function cmdDeploy(names: string[], opts: VerbOpts): Promise { await fs.mkdir(TMP_DIR, {recursive: true}); let tmp = await fs.mkdtemp(nodePath.join(TMP_DIR, 'deploy-')); try { - await aq.run(tmp, 'copy', dst => matched.has(dst)); + await withProgress(`${name}: staging`, matched.size, + tick => aq.run(tmp, 'copy', dst => matched.has(dst), tick)); let cmd = spawn(entry.cmd.replaceAll('%d', tmp), {shell: true, stdio: ['ignore', 'inherit', 'inherit']}); if (await waitExit(cmd) !== 0) { @@ -293,10 +296,15 @@ async function cmdDeploy(names: string[], opts: VerbOpts): Promise { // also crash on the resulting EPIPE, which reaches both // stdin and (via streamx's destroy propagation) the pack. stdin.on('error', () => {}); - let pack = await aq.pack(dst => matched.has(dst)); - pack.on('error', () => {}); - pack.pipe(stdin); - if (await waitExit(upload) !== 0) { + // The bar tracks the upload itself: an entry counts once the + // command has taken it, not once it has been read off disk. + let code = await withProgress(`${name}: uploading`, matched.size, async tick => { + let pack = await aq.pack(dst => matched.has(dst), tick); + pack.on('error', () => {}); + pack.pipe(stdin); + return await waitExit(upload); + }); + if (code !== 0) { return 1; } } diff --git a/tools/deploy/progress.ts b/tools/deploy/progress.ts new file mode 100644 index 00000000..18b14f04 --- /dev/null +++ b/tools/deploy/progress.ts @@ -0,0 +1,106 @@ +// A one-line progress bar for the publishing step of a deploy, which is +// otherwise a long silent wait: thousands of files being packed into an +// upload or copied into a staging tree, with nothing on the terminal between +// the line naming the command and the line after it finishes. +// +// Only drawn when stderr is a terminal. Piped or under CI a redraw per file +// is noise in the log, and the deploy's own stdout lines are the record of +// what happened; those are printed either way, and never carry the bar. + +let REDRAW_MS = 100; +// Below this a bar is more misleading than a bare count. +let MIN_BAR = 10; + +// The parts of a terminal a bar uses, so a test can hand it one. +export type Term = { + isTTY?: boolean | undefined, + columns?: number | undefined, + write(s: string): unknown, +}; + +export class Progress { + #label: string; + #total: number; + #term: Term; + #done: number; + #drawn: boolean; + #lastDraw: number; + + constructor(label: string, total: number, term: Term = process.stderr) { + this.#label = label; + this.#total = total; + this.#term = term; + this.#done = 0; + this.#drawn = false; + this.#lastDraw = 0; + if (this.#interactive) { + this.#draw(); + } + } + + get #interactive(): boolean { + return this.#term.isTTY === true && this.#total > 0; + } + + tick(n = 1) { + this.#done = Math.min(this.#done + n, this.#total); + if (!this.#interactive) { + return; + } + // The line is the bar's only while there is progress left to show. + // What the command has to say about the rest of the wait -- it is + // still running, with the terminal to itself -- lands on a clean line + // instead of on the end of a finished bar. + if (this.#done === this.#total) { + this.finish(); + return; + } + // Redraw on a timer rather than per file: a deploy ticks tens of + // thousands of times. + if (Date.now() - this.#lastDraw < REDRAW_MS) { + return; + } + this.#draw(); + } + + // Leave the line as the deploy found it: the bar is a live thing, and + // what stays in the scrollback is the deploy's own output. + finish() { + if (this.#drawn) { + this.#term.write('\x1b[2K\r'); + this.#drawn = false; + } + } + + #draw() { + this.#lastDraw = Date.now(); + let columns = this.#term.columns || 80; + // Pad the count so the bar keeps its width as the numbers grow into + // each other's columns. + let counts = `${String(this.#done).padStart(String(this.#total).length)}/${this.#total}`; + // label, a space, '[', ']', a space, counts, and a column to spare. + let width = columns - this.#label.length - counts.length - 5; + let line; + if (width < MIN_BAR) { + line = `${this.#label} ${counts}`; + } else { + let filled = Math.round(width * this.#done / this.#total); + line = `${this.#label} [${'#'.repeat(filled)}${'-'.repeat(width - filled)}] ${counts}`; + } + // One column short of the width: writing the last one wraps to the + // next line on some terminals, which leaves the bar behind. + this.#term.write(`\x1b[2K\r${line.slice(0, columns - 1)}`); + this.#drawn = true; + } +} + +// Draw a bar over `body`'s work, and leave the line clean however it goes. +export async function withProgress(label: string, total: number, + body: (tick: () => void) => Promise): Promise { + let bar = new Progress(label, total); + try { + return await body(() => bar.tick()); + } finally { + bar.finish(); + } +} diff --git a/tools/deploy/queue.ts b/tools/deploy/queue.ts index dcfe3954..693d9c1c 100644 --- a/tools/deploy/queue.ts +++ b/tools/deploy/queue.ts @@ -163,7 +163,9 @@ export class ActionQueue { } } - async run(dir: string, mode: 'link' | 'copy' | 'tar', filter?: (dst: string) => boolean) { + // `onFile` is called once per published name as it lands, for progress. + async run(dir: string, mode: 'link' | 'copy' | 'tar', filter?: (dst: string) => boolean, + onFile?: () => void) { if (!this.valid) throw new Error(`Invalid ActionQueue`); if (mode !== 'tar') { @@ -194,11 +196,12 @@ export class ActionQueue { await fs.rm(dst, {force: true}); await fs.symlink(op.target, dst); } + onFile?.(); } } else { // In this case, I guess its a file rather than a dir. let out = createWriteStream(dir); - (await this.pack(filter)).pipe(out); + (await this.pack(filter, onFile)).pipe(out); return new Promise((resolve, reject) => { out.on('error', reject); out.on('finish', () => resolve()); @@ -206,23 +209,36 @@ export class ActionQueue { } } - async pack(filter?: (dst: string) => boolean): Promise { + // Entries are handed to the pack as fast as they can be read, but each + // one's callback waits on the pack draining into whoever is reading it, + // so `onFile` tracks what the consumer has taken rather than what has + // been queued for it. + async pack(filter?: (dst: string) => boolean, onFile?: () => void) + : Promise { if (!this.valid) throw new Error(`Invalid ActionQueue`); let t = tar.pack(); + // An entry's callback also carries the error the consumer is already + // reporting; count the entry only where it made it through. + let took = onFile === undefined ? undefined : (err?: Error | null) => { + if (err === null || err === undefined) { + onFile(); + } + }; for (let entry of this.log) { if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) continue; let op = entry.op; if (op.type === 'Symlink') { - t.entry({name: entry.dst, type: 'symlink', linkname: op.target}).on('error', () => {}); + t.entry({name: entry.dst, type: 'symlink', linkname: op.target}, took) + .on('error', () => {}); continue; } let data = op.type === 'Copy' ? await fs.readFile(op.src) : op.data; // A dying consumer destroys the pack and every pending entry // sink, and each sink emits the error; the consumer is the one // reporting the failure, so keep the sinks quiet. - t.entry({name: entry.dst}, data).on('error', () => {}); + t.entry({name: entry.dst}, data, took).on('error', () => {}); } // Without this the archive has no end-of-archive marker, and strict // readers (Python tarfile in stream mode) die on the truncation. diff --git a/tools/deploy/test/api.test.ts b/tools/deploy/test/api.test.ts index f5b1bf68..a39338ef 100644 --- a/tools/deploy/test/api.test.ts +++ b/tools/deploy/test/api.test.ts @@ -4,6 +4,7 @@ import {createHash} from 'node:crypto'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as pathlib from 'node:path'; +import * as stream from 'node:stream'; import {beforeEach, test} from 'node:test'; import b32encode from 'base32-encode'; @@ -137,6 +138,36 @@ test('pack with a filter packs only matching entries in order', async () => { ]); }); +test('pack counts an entry for the progress bar as the consumer takes it', async () => { + let aq = new ActionQueue(); + for (let i = 0; i < 4; i++) { + aq.write('x'.repeat(50_000), `f${i}.txt`); + } + let counted = 0; + let packed = await aq.pack(undefined, () => counted++); + // Nothing has read the pack yet: an entry queued is not an entry shipped. + assert.equal(counted, 0); + let sink = new stream.Writable({highWaterMark: 1, write: (_c, _e, cb) => setImmediate(cb)}); + await new Promise((resolve, reject) => { + sink.on('finish', resolve); + sink.on('error', reject); + packed.pipe(sink); + }); + assert.equal(counted, 4); +}); + +test('run counts each materialized entry for the progress bar, filter and all', async () => { + let dir = tmpdir(); + let aq = new ActionQueue(); + aq.write('1', 'ani/a.gif'); + aq.symlink('a.gif', 'ani/b.gif'); + aq.write('2', 'dex/c.png'); + let counted: number[] = []; + await aq.run(pathlib.join(dir, 'deploy'), 'copy', dst => dst.startsWith('ani/'), + () => counted.push(counted.length + 1)); + assert.deepEqual(counted, [1, 2]); +}); + test('duplicate and absolute destinations invalidate the queue', async () => { let dup = new ActionQueue(); dup.write('a', 'x.txt'); diff --git a/tools/deploy/test/progress.test.ts b/tools/deploy/test/progress.test.ts new file mode 100644 index 00000000..55bc884c --- /dev/null +++ b/tools/deploy/test/progress.test.ts @@ -0,0 +1,76 @@ + +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {Progress, type Term} from '../progress.ts'; + +function term(overrides: Partial = {}): Term & {written: string[]} { + let written: string[] = []; + return {isTTY: true, columns: 80, write: (s: string) => written.push(s), written, ...overrides}; +} + +// Every write the bar makes, with the clear-and-return prefix stripped. +function lines(t: {written: string[]}): string[] { + return t.written.map(s => s.replace('\x1b[2K\r', '')); +} + +test('a bar draws nothing when stderr is not a terminal', () => { + let t = term({isTTY: false}); + let bar = new Progress('deploy: uploading', 4, t); + for (let i = 0; i < 4; i++) { + bar.tick(); + } + bar.finish(); + assert.deepEqual(t.written, []); +}); + +test('a bar draws its label, a filled share of the width, and the counts', () => { + let t = term(); + let bar = new Progress('up', 100, t); + // The timer holds the redraws back, so drive it to the end and read the + // first draw, which the constructor always makes. + let [first] = lines(t); + assert.equal(first, `up [${'-'.repeat(66)}] 0/100`); + assert.equal(first!.length, 79); + for (let i = 0; i < 50; i++) { + bar.tick(); + } + assert.equal(lines(t).length, 1); +}); + +test('a bar never writes past the terminal width', () => { + for (let columns of [20, 40, 80, 200]) { + let t = term({columns}); + new Progress('deploying something', 1000, t); + for (let line of lines(t)) { + assert.ok(line.length < columns, `${line.length} >= ${columns} at ${columns} columns`); + } + } +}); + +test('a narrow terminal gets the counts without a bar', () => { + let t = term({columns: 24}); + new Progress('up', 1000, t); + assert.deepEqual(lines(t), ['up 0/1000']); +}); + +test('a bar clears its line once it is full, leaving the rest of the wait alone', () => { + let t = term(); + let bar = new Progress('up', 2, t); + bar.tick(); + bar.tick(); + assert.equal(t.written.at(-1), '\x1b[2K\r'); + assert.equal(lines(t).at(-1), ''); + // Already cleared: finishing again is not a second clear. + let count = t.written.length; + bar.finish(); + assert.equal(t.written.length, count); +}); + +test('finishing a bar mid-run clears the line it drew', () => { + let t = term(); + let bar = new Progress('up', 100, t); + bar.tick(); + bar.finish(); + assert.equal(t.written.at(-1), '\x1b[2K\r'); +});