diff --git a/tools/deploy/api.ts b/tools/deploy/api.ts index 416344e7..4fa48d3a 100644 --- a/tools/deploy/api.ts +++ b/tools/deploy/api.ts @@ -1,6 +1,7 @@ import * as crypto from 'node:crypto'; import * as fs from 'node:fs/promises'; +import * as nodePath from 'node:path'; import b32encode from 'base32-encode'; @@ -24,6 +25,11 @@ export type DeployCtx = { // read, list, and hash do file I/O. copy(src: CopySource, dst: string): void, write(dst: string, data: string): void, + // A second name for something else this tree publishes, carried as a link + // rather than a second copy. `names` is that thing's destination, not a + // link target: what gets written is the path from `dst` to it, since the + // two ends can be moved apart by whoever unpacks the tree. + symlink(dst: string, names: string): void, read(src: CopySource): Promise, list(dir: string): Promise, // 8-char base32 content stamp. One source: the digest of its bytes, @@ -76,6 +82,9 @@ export function makeCtx(casDir: string, queue: ActionQueue): DeployCtx { write(dst: string, data: string): void { queue.write(data, dst); }, + symlink(dst: string, names: string): void { + queue.symlink(nodePath.relative(nodePath.dirname(nodePath.normalize(dst)), names), dst); + }, async read(src: CopySource): Promise { return await fs.readFile(srcPath(src), 'utf8'); }, diff --git a/tools/deploy/index.ts b/tools/deploy/index.ts index d923c969..68a35bb1 100644 --- a/tools/deploy/index.ts +++ b/tools/deploy/index.ts @@ -337,6 +337,12 @@ async function outputs(aq: ActionQueue): Promise> { out.set(e.dst, crypto.createHash('sha256').update(e.op.data).digest('hex')); continue; } + // A link has no bytes; what it publishes is the name it points at, so + // that is what a retarget has to show up as. + if (e.op.type === 'Symlink') { + out.set(e.dst, `symlink:${e.op.target}`); + continue; + } // A CAS path spells its own digest, so only raw sources are read. let cas = new RegExp(`^${CAS_DIR}/[0-9a-f]{2}/([0-9a-f]{64})\\.`).exec(e.op.src); out.set(e.dst, cas ? cas[1]! : (await hashFile(e.op.src)).toString('hex')); diff --git a/tools/deploy/queue.ts b/tools/deploy/queue.ts index e82df206..dcfe3954 100644 --- a/tools/deploy/queue.ts +++ b/tools/deploy/queue.ts @@ -10,13 +10,16 @@ type Op = { } | { type: 'Copy', src: string, +} | { + type: 'Symlink', + target: string, }; type OpEntry = { type: 'Op', op: Op, dst: string, - valid: 'Success' | 'Absolute' | 'Multiple', + valid: 'Success' | 'Absolute' | 'Multiple' | 'Escapes', debugObjs: unknown[] }; @@ -28,6 +31,27 @@ type DebugEntry = { export type LogEntry = OpEntry | DebugEntry; +// What a link names, resolved lexically against the directory it sits in: the +// tree it describes isn't on disk anywhere yet, so there is nothing else to +// resolve against. Null where it walks out of the tree, or names the tree +// itself, neither of which is a name this tree can publish. +function resolveLink(dst: string, target: string): string | null { + let parts = nodePath.dirname(dst).split('/').filter(p => p !== '' && p !== '.'); + for (let part of target.split('/')) { + if (part === '' || part === '.') { + continue; + } + if (part !== '..') { + parts.push(part); + } else if (parts.length > 0) { + parts.pop(); + } else { + return null; + } + } + return parts.length === 0 ? null : parts.join('/'); +} + export class ActionQueue { #seen: Map; // Have an accessor for this in the future? idk @@ -55,7 +79,7 @@ export class ActionQueue { this.log.push({type: 'Debug', obj, stray}); } - #pushOp(op: Op, dst: string) { + #pushOp(op: Op, dst: string): OpEntry { dst = nodePath.normalize(dst); let entry: OpEntry = { type: 'Op', @@ -81,6 +105,7 @@ export class ActionQueue { } } } + return entry; } copy(src: string, dst: string) { @@ -91,6 +116,16 @@ export class ActionQueue { this.#pushOp({type: 'Write', data}, dst); } + // A link the receiving side extracts as root, so its target may only name + // something else in this tree. + symlink(target: string, dst: string) { + let entry = this.#pushOp({type: 'Symlink', target}, dst); + if (nodePath.isAbsolute(target) || resolveLink(entry.dst, target) === null) { + this.valid = false; + entry.valid = 'Escapes'; + } + } + skip() { for (let obj of this.#debugBuffer) { this.gdebug(obj, true); @@ -115,6 +150,8 @@ export class ActionQueue { console.error(`COPY${addendum}: ${op.src} ==> ${entry.dst}`); } else if (op.type === 'Write') { console.error(`WRITE${addendum}: ${op.data.length} characters ==> ${entry.dst}`); + } else if (op.type === 'Symlink') { + console.error(`SYMLINK${addendum}: ${entry.dst} -> ${op.target}`); } } else if (entry.type === 'Debug') { let addendum = ''; @@ -148,6 +185,14 @@ export class ActionQueue { } } else if (op.type === 'Write') { await fs.writeFile(dst, op.data); + } else if (op.type === 'Symlink') { + // A link and not the file it names, in every mode: what + // the tar carries is what a directory has to hold too. + // symlink() has no truncating open behind it, so the name + // is cleared first to keep a rerun over an existing tree + // working the way copy and write already do. + await fs.rm(dst, {force: true}); + await fs.symlink(op.target, dst); } } } else { @@ -169,6 +214,10 @@ export class ActionQueue { if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst))) continue; let op = entry.op; + if (op.type === 'Symlink') { + t.entry({name: entry.dst, type: 'symlink', linkname: op.target}).on('error', () => {}); + continue; + } let data = op.type === 'Copy' ? await fs.readFile(op.src) : op.data; // A dying consumer destroys the pack and every pending entry // sink, and each sink emits the error; the consumer is the one diff --git a/tools/deploy/test/api.test.ts b/tools/deploy/test/api.test.ts index ef794fd3..f5b1bf68 100644 --- a/tools/deploy/test/api.test.ts +++ b/tools/deploy/test/api.test.ts @@ -88,17 +88,24 @@ test('ctx.list sorts, parses extensions, skips dotfiles and directories', async ]); }); +type PackedEntry = {name: string, data: string, type?: string, linkname?: string}; + async function packedEntries(aq: ActionQueue, filter?: (dst: string) => boolean) - : Promise<{name: string, data: string}[]> { + : Promise { let packed = await aq.pack(filter); return new Promise((resolve, reject) => { let extract = tar.extract(); - let entries: {name: string, data: string}[] = []; + let entries: PackedEntry[] = []; extract.on('entry', (header, stream, next) => { let chunks: Buffer[] = []; stream.on('data', c => chunks.push(c)); stream.on('end', () => { - entries.push({name: header.name, data: Buffer.concat(chunks).toString()}); + let entry: PackedEntry = {name: header.name, data: Buffer.concat(chunks).toString()}; + if (header.type === 'symlink') { + entry.type = header.type; + entry.linkname = header.linkname ?? undefined; + } + entries.push(entry); next(); }); }); @@ -164,3 +171,63 @@ test('copy-mode materialization restores 0644 on read-only sources', async () => await aq.run(out, 'copy'); assert.equal(fs.statSync(pathlib.join(out, 'out/x.png')).mode & 0o777, 0o644); }); + +test('pack carries a symlink as a link, not the bytes it names', async () => { + let aq = new ActionQueue(); + aq.write('1', 'links/xy/a-XXXX.gif'); + aq.symlink('a-XXXX.gif', 'links/xy/a.gif'); + assert.deepEqual(await packedEntries(aq), [ + {name: 'links/xy/a-XXXX.gif', data: '1'}, + {name: 'links/xy/a.gif', data: '', type: 'symlink', linkname: 'a-XXXX.gif'}, + ]); +}); + +test('run materializes a symlink resolving beside its target', async () => { + let dir = tmpdir(); + let aq = new ActionQueue(); + aq.write('bytes', 'xy/a-XXXX.gif'); + aq.symlink('a-XXXX.gif', 'xy/a.gif'); + let out = pathlib.join(dir, 'deploy'); + await aq.run(out, 'copy'); + let link = pathlib.join(out, 'xy/a.gif'); + assert.ok(fs.lstatSync(link).isSymbolicLink()); + assert.equal(fs.readlinkSync(link), 'a-XXXX.gif'); + assert.equal(fs.readFileSync(link, 'utf8'), 'bytes'); + // A second pass over the same tree is what a rerun looks like + await aq.run(out, 'copy'); + assert.equal(fs.readlinkSync(link), 'a-XXXX.gif'); +}); + +test('a symlink target naming anything outside the tree invalidates the queue', async () => { + let escaping = new ActionQueue(); + escaping.symlink('../../etc/passwd', 'xy/a.gif'); + assert.ok(!escaping.valid); + + let absolute = new ActionQueue(); + absolute.symlink('/etc/passwd', 'xy/a.gif'); + assert.ok(!absolute.valid); + + let tree = new ActionQueue(); + tree.symlink('..', 'xy/a.gif'); + assert.ok(!tree.valid); + + // A target that walks up and back down again stays in the tree, which is + // what a link between two of its subtrees looks like. + let across = new ActionQueue(); + across.symlink('../sprites/a-XXXX.gif', '__meta/a.gif'); + across.symlink('a-XXXX.gif', 'xy/a.gif'); + assert.ok(across.valid); +}); + +test('ctx.symlink writes the path from the link to what it names', async () => { + let aq = new ActionQueue(); + let ctx = makeCtx('cas', aq); + ctx.symlink('__meta/links/sprites/xy/a.gif', 'sprites/xy/a-XXXX.gif'); + ctx.symlink('__meta/a.gif', 'sprites/a-XXXX.gif'); + let ops = aq.log.filter(e => e.type === 'Op'); + assert.ok(aq.valid); + assert.deepEqual(ops.map(e => (e.op as {target: string}).target), [ + '../../../../sprites/xy/a-XXXX.gif', + '../sprites/a-XXXX.gif', + ]); +});