Merge pull request #11 from smogon/claude/interactive-deploy-progress-bar-njnbln

Draw a progress bar for deploy on a terminal

README.md

@@ -101,6 +101,12 @@ covered by some entry. `deploy <name> -o <dir>` materializes each entry's
 subset under `<dir>/<name>/<entry index>/` instead of running its command,
 for eyeballing what would ship.
 
+Where stderr is a terminal, each entry draws a progress bar over its files
+while they go out -- an entry counts once the command has taken it, not once
+it has been read off disk -- and the line is cleared again afterwards. Piped
+or under CI nothing is drawn, and the lines the deploy prints are the same
+either way.
+
 This file is not committed, because it is where the hosts and paths this
 repo ships to are written down.
 

tools/deploy/index.ts

@@ -18,6 +18,7 @@ import {setConfig} from '../build/helpers.ts';
 import * as db from '../build/store.ts';
 import * as api from './api.ts';
 import {loadDeployConfig, matchSubsets} from './config.ts';
+import {withProgress} from './progress.ts';
 import {ActionQueue} from './queue.ts';
 
 let root = nodePath.resolve(fileURLToPath(import.meta.url), '../../..');
@@ -265,7 +266,8 @@ async function cmdDeploy(names: string[], opts: VerbOpts): Promise<void> {
                 // entries included) instead of running its command.
                 if (opts.output !== undefined) {
                     let dir = nodePath.join(opts.output, name, String(i));
-                    await aq.run(dir, 'copy', dst => matched.has(dst));
+                    await withProgress(`${name}: copying`, matched.size,
+                        tick => aq.run(dir, 'copy', dst => matched.has(dst), tick));
                     console.log(`    -> ${dir}`);
                     continue;
                 }
@@ -273,7 +275,8 @@ async function cmdDeploy(names: string[], opts: VerbOpts): Promise<void> {
                     await fs.mkdir(TMP_DIR, {recursive: true});
                     let tmp = await fs.mkdtemp(nodePath.join(TMP_DIR, 'deploy-'));
                     try {
-                        await aq.run(tmp, 'copy', dst => matched.has(dst));
+                        await withProgress(`${name}: staging`, matched.size,
+                            tick => aq.run(tmp, 'copy', dst => matched.has(dst), tick));
                         let cmd = spawn(entry.cmd.replaceAll('%d', tmp),
                             {shell: true, stdio: ['ignore', 'inherit', 'inherit']});
                         if (await waitExit(cmd) !== 0) {
@@ -293,10 +296,15 @@ async function cmdDeploy(names: string[], opts: VerbOpts): Promise<void> {
                 // also crash on the resulting EPIPE, which reaches both
                 // stdin and (via streamx's destroy propagation) the pack.
                 stdin.on('error', () => {});
-                let pack = await aq.pack(dst => matched.has(dst));
-                pack.on('error', () => {});
-                pack.pipe(stdin);
-                if (await waitExit(upload) !== 0) {
+                // The bar tracks the upload itself: an entry counts once the
+                // command has taken it, not once it has been read off disk.
+                let code = await withProgress(`${name}: uploading`, matched.size, async tick => {
+                    let pack = await aq.pack(dst => matched.has(dst), tick);
+                    pack.on('error', () => {});
+                    pack.pipe(stdin);
+                    return await waitExit(upload);
+                });
+                if (code !== 0) {
                     return 1;
                 }
             }

tools/deploy/progress.ts

@@ -0,0 +1,106 @@
+// A one-line progress bar for the publishing step of a deploy, which is
+// otherwise a long silent wait: thousands of files being packed into an
+// upload or copied into a staging tree, with nothing on the terminal between
+// the line naming the command and the line after it finishes.
+//
+// Only drawn when stderr is a terminal. Piped or under CI a redraw per file
+// is noise in the log, and the deploy's own stdout lines are the record of
+// what happened; those are printed either way, and never carry the bar.
+
+let REDRAW_MS = 100;
+// Below this a bar is more misleading than a bare count.
+let MIN_BAR = 10;
+
+// The parts of a terminal a bar uses, so a test can hand it one.
+export type Term = {
+    isTTY?: boolean | undefined,
+    columns?: number | undefined,
+    write(s: string): unknown,
+};
+
+export class Progress {
+    #label: string;
+    #total: number;
+    #term: Term;
+    #done: number;
+    #drawn: boolean;
+    #lastDraw: number;
+
+    constructor(label: string, total: number, term: Term = process.stderr) {
+        this.#label = label;
+        this.#total = total;
+        this.#term = term;
+        this.#done = 0;
+        this.#drawn = false;
+        this.#lastDraw = 0;
+        if (this.#interactive) {
+            this.#draw();
+        }
+    }
+
+    get #interactive(): boolean {
+        return this.#term.isTTY === true && this.#total > 0;
+    }
+
+    tick(n = 1) {
+        this.#done = Math.min(this.#done + n, this.#total);
+        if (!this.#interactive) {
+            return;
+        }
+        // The line is the bar's only while there is progress left to show.
+        // What the command has to say about the rest of the wait -- it is
+        // still running, with the terminal to itself -- lands on a clean line
+        // instead of on the end of a finished bar.
+        if (this.#done === this.#total) {
+            this.finish();
+            return;
+        }
+        // Redraw on a timer rather than per file: a deploy ticks tens of
+        // thousands of times.
+        if (Date.now() - this.#lastDraw < REDRAW_MS) {
+            return;
+        }
+        this.#draw();
+    }
+
+    // Leave the line as the deploy found it: the bar is a live thing, and
+    // what stays in the scrollback is the deploy's own output.
+    finish() {
+        if (this.#drawn) {
+            this.#term.write('\x1b[2K\r');
+            this.#drawn = false;
+        }
+    }
+
+    #draw() {
+        this.#lastDraw = Date.now();
+        let columns = this.#term.columns || 80;
+        // Pad the count so the bar keeps its width as the numbers grow into
+        // each other's columns.
+        let counts = `${String(this.#done).padStart(String(this.#total).length)}/${this.#total}`;
+        // label, a space, '[', ']', a space, counts, and a column to spare.
+        let width = columns - this.#label.length - counts.length - 5;
+        let line;
+        if (width < MIN_BAR) {
+            line = `${this.#label} ${counts}`;
+        } else {
+            let filled = Math.round(width * this.#done / this.#total);
+            line = `${this.#label} [${'#'.repeat(filled)}${'-'.repeat(width - filled)}] ${counts}`;
+        }
+        // One column short of the width: writing the last one wraps to the
+        // next line on some terminals, which leaves the bar behind.
+        this.#term.write(`\x1b[2K\r${line.slice(0, columns - 1)}`);
+        this.#drawn = true;
+    }
+}
+
+// Draw a bar over `body`'s work, and leave the line clean however it goes.
+export async function withProgress<T>(label: string, total: number,
+                                      body: (tick: () => void) => Promise<T>): Promise<T> {
+    let bar = new Progress(label, total);
+    try {
+        return await body(() => bar.tick());
+    } finally {
+        bar.finish();
+    }
+}

tools/deploy/queue.ts

@@ -163,7 +163,9 @@ export class ActionQueue {
         }
     }
 
-    async run(dir: string, mode: 'link' | 'copy' | 'tar', filter?: (dst: string) => boolean) {
+    // `onFile` is called once per published name as it lands, for progress.
+    async run(dir: string, mode: 'link' | 'copy' | 'tar', filter?: (dst: string) => boolean,
+              onFile?: () => void) {
         if (!this.valid)
             throw new Error(`Invalid ActionQueue`);
         if (mode !== 'tar') {
@@ -194,11 +196,12 @@ export class ActionQueue {
                     await fs.rm(dst, {force: true});
                     await fs.symlink(op.target, dst);
                 }
+                onFile?.();
             }
         } else {
             // In this case, I guess its a file rather than a dir.
             let out = createWriteStream(dir);
-            (await this.pack(filter)).pipe(out);
+            (await this.pack(filter, onFile)).pipe(out);
             return new Promise<void>((resolve, reject) => {
                 out.on('error', reject);
                 out.on('finish', () => resolve());
@@ -206,23 +209,36 @@ export class ActionQueue {
         }
     }
 
-    async pack(filter?: (dst: string) => boolean): Promise<NodeJS.ReadableStream> {
+    // Entries are handed to the pack as fast as they can be read, but each
+    // one's callback waits on the pack draining into whoever is reading it,
+    // so `onFile` tracks what the consumer has taken rather than what has
+    // been queued for it.
+    async pack(filter?: (dst: string) => boolean, onFile?: () => void)
+        : Promise<NodeJS.ReadableStream> {
         if (!this.valid)
             throw new Error(`Invalid ActionQueue`);
         let t = tar.pack();
+        // An entry's callback also carries the error the consumer is already
+        // reporting; count the entry only where it made it through.
+        let took = onFile === undefined ? undefined : (err?: Error | null) => {
+            if (err === null || err === undefined) {
+                onFile();
+            }
+        };
         for (let entry of this.log) {
             if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst)))
                 continue;
             let op = entry.op;
             if (op.type === 'Symlink') {
-                t.entry({name: entry.dst, type: 'symlink', linkname: op.target}).on('error', () => {});
+                t.entry({name: entry.dst, type: 'symlink', linkname: op.target}, took)
+                    .on('error', () => {});
                 continue;
             }
             let data = op.type === 'Copy' ? await fs.readFile(op.src) : op.data;
             // A dying consumer destroys the pack and every pending entry
             // sink, and each sink emits the error; the consumer is the one
             // reporting the failure, so keep the sinks quiet.
-            t.entry({name: entry.dst}, data).on('error', () => {});
+            t.entry({name: entry.dst}, data, took).on('error', () => {});
         }
         // Without this the archive has no end-of-archive marker, and strict
         // readers (Python tarfile in stream mode) die on the truncation.

tools/deploy/test/api.test.ts

@@ -4,6 +4,7 @@ import {createHash} from 'node:crypto';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as pathlib from 'node:path';
+import * as stream from 'node:stream';
 import {beforeEach, test} from 'node:test';
 
 import b32encode from 'base32-encode';
@@ -137,6 +138,36 @@ test('pack with a filter packs only matching entries in order', async () => {
     ]);
 });
 
+test('pack counts an entry for the progress bar as the consumer takes it', async () => {
+    let aq = new ActionQueue();
+    for (let i = 0; i < 4; i++) {
+        aq.write('x'.repeat(50_000), `f${i}.txt`);
+    }
+    let counted = 0;
+    let packed = await aq.pack(undefined, () => counted++);
+    // Nothing has read the pack yet: an entry queued is not an entry shipped.
+    assert.equal(counted, 0);
+    let sink = new stream.Writable({highWaterMark: 1, write: (_c, _e, cb) => setImmediate(cb)});
+    await new Promise((resolve, reject) => {
+        sink.on('finish', resolve);
+        sink.on('error', reject);
+        packed.pipe(sink);
+    });
+    assert.equal(counted, 4);
+});
+
+test('run counts each materialized entry for the progress bar, filter and all', async () => {
+    let dir = tmpdir();
+    let aq = new ActionQueue();
+    aq.write('1', 'ani/a.gif');
+    aq.symlink('a.gif', 'ani/b.gif');
+    aq.write('2', 'dex/c.png');
+    let counted: number[] = [];
+    await aq.run(pathlib.join(dir, 'deploy'), 'copy', dst => dst.startsWith('ani/'),
+        () => counted.push(counted.length + 1));
+    assert.deepEqual(counted, [1, 2]);
+});
+
 test('duplicate and absolute destinations invalidate the queue', async () => {
     let dup = new ActionQueue();
     dup.write('a', 'x.txt');

tools/deploy/test/progress.test.ts

@@ -0,0 +1,76 @@
+
+import assert from 'node:assert/strict';
+import {test} from 'node:test';
+
+import {Progress, type Term} from '../progress.ts';
+
+function term(overrides: Partial<Term> = {}): Term & {written: string[]} {
+    let written: string[] = [];
+    return {isTTY: true, columns: 80, write: (s: string) => written.push(s), written, ...overrides};
+}
+
+// Every write the bar makes, with the clear-and-return prefix stripped.
+function lines(t: {written: string[]}): string[] {
+    return t.written.map(s => s.replace('\x1b[2K\r', ''));
+}
+
+test('a bar draws nothing when stderr is not a terminal', () => {
+    let t = term({isTTY: false});
+    let bar = new Progress('deploy: uploading', 4, t);
+    for (let i = 0; i < 4; i++) {
+        bar.tick();
+    }
+    bar.finish();
+    assert.deepEqual(t.written, []);
+});
+
+test('a bar draws its label, a filled share of the width, and the counts', () => {
+    let t = term();
+    let bar = new Progress('up', 100, t);
+    // The timer holds the redraws back, so drive it to the end and read the
+    // first draw, which the constructor always makes.
+    let [first] = lines(t);
+    assert.equal(first, `up [${'-'.repeat(66)}]   0/100`);
+    assert.equal(first!.length, 79);
+    for (let i = 0; i < 50; i++) {
+        bar.tick();
+    }
+    assert.equal(lines(t).length, 1);
+});
+
+test('a bar never writes past the terminal width', () => {
+    for (let columns of [20, 40, 80, 200]) {
+        let t = term({columns});
+        new Progress('deploying something', 1000, t);
+        for (let line of lines(t)) {
+            assert.ok(line.length < columns, `${line.length} >= ${columns} at ${columns} columns`);
+        }
+    }
+});
+
+test('a narrow terminal gets the counts without a bar', () => {
+    let t = term({columns: 24});
+    new Progress('up', 1000, t);
+    assert.deepEqual(lines(t), ['up    0/1000']);
+});
+
+test('a bar clears its line once it is full, leaving the rest of the wait alone', () => {
+    let t = term();
+    let bar = new Progress('up', 2, t);
+    bar.tick();
+    bar.tick();
+    assert.equal(t.written.at(-1), '\x1b[2K\r');
+    assert.equal(lines(t).at(-1), '');
+    // Already cleared: finishing again is not a second clear.
+    let count = t.written.length;
+    bar.finish();
+    assert.equal(t.written.length, count);
+});
+
+test('finishing a bar mid-run clears the line it drew', () => {
+    let t = term();
+    let bar = new Progress('up', 100, t);
+    bar.tick();
+    bar.finish();
+    assert.equal(t.written.at(-1), '\x1b[2K\r');
+});