Leahnaya/sprites
1
0
Fork 0
mirror of https://github.com/smogon/sprites.git synced 2026-09-10 11:05:36 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Let a deploy publish a link as well as a file

Browse Source 
A served name carries a content hash, so a reader that composes a fixed path
can't find one. The way out is to publish the set twice, once under the stamped
names and once under the bare ones, and a link is the one thing the queue
couldn't say: Copy dereferences and Write has bytes.

Add a Symlink op beside them. What ctx.symlink takes is not a link target but
the destination of the thing being named, since a published tree is unpacked
somewhere else and its halves can land apart; the path between the two is
computed here and is what gets written. It takes the same duplicate-dst and
absolute-dst checks the other ops do, plus one of its own -- resolved lexically
against the directory it sits in, a target has to stay inside the tree, since
the receiving side extracts as root and a link that walks out of it is the same
write to anywhere that an absolute member name is.

run() makes a real symlink in every mode, clearing the name first so a rerun
over an existing tree behaves the way copy and write already do, and pack()
emits a symlink member with no body, which is what a stream reader on the other
end sees.

refactor digests bytes, and a link has none; what it publishes is the name it
points at, so that is what it records and what a retarget shows up as.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Christopher Monsanto
2026-08-23 15:22:45 -04:00
parent 79f0ef6497
commit a648c9e622
4 changed files with 136 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tools/deploy/api.ts
View File

@@ -1,6 +1,7 @@
import * as crypto from 'node:crypto';
import * as fs from 'node:fs/promises';
import * as nodePath from 'node:path';
import b32encode from 'base32-encode';
@@ -24,6 +25,11 @@ export type DeployCtx = {
// read, list, and hash do file I/O.
copy(src: CopySource, dst: string): void,
write(dst: string, data: string): void,
// A second name for something else this tree publishes, carried as a link
// rather than a second copy. `names` is that thing's destination, not a
// link target: what gets written is the path from `dst` to it, since the
// two ends can be moved apart by whoever unpacks the tree.
symlink(dst: string, names: string): void,
read(src: CopySource): Promise<string>,
list(dir: string): Promise<SrcFile[]>,
// 8-char base32 content stamp. One source: the digest of its bytes,
@@ -76,6 +82,9 @@ export function makeCtx(casDir: string, queue: ActionQueue): DeployCtx {
write(dst: string, data: string): void {
queue.write(data, dst);
},
symlink(dst: string, names: string): void {
queue.symlink(nodePath.relative(nodePath.dirname(nodePath.normalize(dst)), names), dst);
},
async read(src: CopySource): Promise<string> {
return await fs.readFile(srcPath(src), 'utf8');
},

6
tools/deploy/index.ts
View File

@@ -337,6 +337,12 @@ async function outputs(aq: ActionQueue): Promise<Map<string, string>> {
out.set(e.dst, crypto.createHash('sha256').update(e.op.data).digest('hex'));
continue;
}
// A link has no bytes; what it publishes is the name it points at, so
// that is what a retarget has to show up as.
if (e.op.type === 'Symlink') {
out.set(e.dst, `symlink:${e.op.target}`);
continue;
}
// A CAS path spells its own digest, so only raw sources are read.
let cas = new RegExp(`^${CAS_DIR}/[0-9a-f]{2}/([0-9a-f]{64})\\.`).exec(e.op.src);
out.set(e.dst, cas ? cas[1]! : (await hashFile(e.op.src)).toString('hex'));

53
tools/deploy/queue.ts
View File

@@ -10,13 +10,16 @@ type Op = {
} | {
type: 'Copy',
src: string,
} | {
type: 'Symlink',
target: string,
};
type OpEntry = {
type: 'Op',
op: Op,
dst: string,
valid: 'Success' | 'Absolute' | 'Multiple',
valid: 'Success' | 'Absolute' | 'Multiple' | 'Escapes',
debugObjs: unknown[]
};
@@ -28,6 +31,27 @@ type DebugEntry = {
export type LogEntry = OpEntry | DebugEntry;
// What a link names, resolved lexically against the directory it sits in: the
// tree it describes isn't on disk anywhere yet, so there is nothing else to
// resolve against. Null where it walks out of the tree, or names the tree
// itself, neither of which is a name this tree can publish.
function resolveLink(dst: string, target: string): string | null {
let parts = nodePath.dirname(dst).split('/').filter(p => p !== '' && p !== '.');
for (let part of target.split('/')) {
if (part === '' || part === '.') {
continue;
}
if (part !== '..') {
parts.push(part);
} else if (parts.length > 0) {
parts.pop();
} else {
return null;
}
}
return parts.length === 0 ? null : parts.join('/');
}
export class ActionQueue {
#seen: Map<string, OpEntry | 'MoreThan1'>;
// Have an accessor for this in the future? idk
@@ -55,7 +79,7 @@ export class ActionQueue {
this.log.push({type: 'Debug', obj, stray});
}
#pushOp(op: Op, dst: string) {
#pushOp(op: Op, dst: string): OpEntry {
dst = nodePath.normalize(dst);
let entry: OpEntry = {
type: 'Op',
@@ -81,6 +105,7 @@ export class ActionQueue {
}
}
}
return entry;
}
copy(src: string, dst: string) {
@@ -91,6 +116,16 @@ export class ActionQueue {
this.#pushOp({type: 'Write', data}, dst);
}
// A link the receiving side extracts as root, so its target may only name
// something else in this tree.
symlink(target: string, dst: string) {
let entry = this.#pushOp({type: 'Symlink', target}, dst);
if (nodePath.isAbsolute(target) || resolveLink(entry.dst, target) === null) {
this.valid = false;
entry.valid = 'Escapes';
}
}
skip() {
for (let obj of this.#debugBuffer) {
this.gdebug(obj, true);
@@ -115,6 +150,8 @@ export class ActionQueue {
console.error(`COPY${addendum}: ${op.src} ==> ${entry.dst}`);
} else if (op.type === 'Write') {
console.error(`WRITE${addendum}: ${op.data.length} characters ==> ${entry.dst}`);
} else if (op.type === 'Symlink') {
console.error(`SYMLINK${addendum}: ${entry.dst} -> ${op.target}`);
}
} else if (entry.type === 'Debug') {
let addendum = '';
@@ -148,6 +185,14 @@ export class ActionQueue {
}
} else if (op.type === 'Write') {
await fs.writeFile(dst, op.data);
} else if (op.type === 'Symlink') {
// A link and not the file it names, in every mode: what
// the tar carries is what a directory has to hold too.
// symlink() has no truncating open behind it, so the name
// is cleared first to keep a rerun over an existing tree
// working the way copy and write already do.
await fs.rm(dst, {force: true});
await fs.symlink(op.target, dst);
}
}
} else {
@@ -169,6 +214,10 @@ export class ActionQueue {
if (entry.type !== 'Op' || (filter !== undefined && !filter(entry.dst)))
continue;
let op = entry.op;
if (op.type === 'Symlink') {
t.entry({name: entry.dst, type: 'symlink', linkname: op.target}).on('error', () => {});
continue;
}
let data = op.type === 'Copy' ? await fs.readFile(op.src) : op.data;
// A dying consumer destroys the pack and every pending entry
// sink, and each sink emits the error; the consumer is the one

73
tools/deploy/test/api.test.ts
View File

@@ -88,17 +88,24 @@ test('ctx.list sorts, parses extensions, skips dotfiles and directories', async
]);
});
type PackedEntry = {name: string, data: string, type?: string, linkname?: string};
async function packedEntries(aq: ActionQueue, filter?: (dst: string) => boolean)
: Promise<{name: string, data: string}[]> {
: Promise<PackedEntry[]> {
let packed = await aq.pack(filter);
return new Promise((resolve, reject) => {
let extract = tar.extract();
let entries: {name: string, data: string}[] = [];
let entries: PackedEntry[] = [];
extract.on('entry', (header, stream, next) => {
let chunks: Buffer[] = [];
stream.on('data', c => chunks.push(c));
stream.on('end', () => {
entries.push({name: header.name, data: Buffer.concat(chunks).toString()});
let entry: PackedEntry = {name: header.name, data: Buffer.concat(chunks).toString()};
if (header.type === 'symlink') {
entry.type = header.type;
entry.linkname = header.linkname ?? undefined;
}
entries.push(entry);
next();
});
});
@@ -164,3 +171,63 @@ test('copy-mode materialization restores 0644 on read-only sources', async () =>
await aq.run(out, 'copy');
assert.equal(fs.statSync(pathlib.join(out, 'out/x.png')).mode & 0o777, 0o644);
});
test('pack carries a symlink as a link, not the bytes it names', async () => {
let aq = new ActionQueue();
aq.write('1', 'links/xy/a-XXXX.gif');
aq.symlink('a-XXXX.gif', 'links/xy/a.gif');
assert.deepEqual(await packedEntries(aq), [
{name: 'links/xy/a-XXXX.gif', data: '1'},
{name: 'links/xy/a.gif', data: '', type: 'symlink', linkname: 'a-XXXX.gif'},
]);
});
test('run materializes a symlink resolving beside its target', async () => {
let dir = tmpdir();
let aq = new ActionQueue();
aq.write('bytes', 'xy/a-XXXX.gif');
aq.symlink('a-XXXX.gif', 'xy/a.gif');
let out = pathlib.join(dir, 'deploy');
await aq.run(out, 'copy');
let link = pathlib.join(out, 'xy/a.gif');
assert.ok(fs.lstatSync(link).isSymbolicLink());
assert.equal(fs.readlinkSync(link), 'a-XXXX.gif');
assert.equal(fs.readFileSync(link, 'utf8'), 'bytes');
// A second pass over the same tree is what a rerun looks like
await aq.run(out, 'copy');
assert.equal(fs.readlinkSync(link), 'a-XXXX.gif');
});
test('a symlink target naming anything outside the tree invalidates the queue', async () => {
let escaping = new ActionQueue();
escaping.symlink('../../etc/passwd', 'xy/a.gif');
assert.ok(!escaping.valid);
let absolute = new ActionQueue();
absolute.symlink('/etc/passwd', 'xy/a.gif');
assert.ok(!absolute.valid);
let tree = new ActionQueue();
tree.symlink('..', 'xy/a.gif');
assert.ok(!tree.valid);
// A target that walks up and back down again stays in the tree, which is
// what a link between two of its subtrees looks like.
let across = new ActionQueue();
across.symlink('../sprites/a-XXXX.gif', '__meta/a.gif');
across.symlink('a-XXXX.gif', 'xy/a.gif');
assert.ok(across.valid);
});
test('ctx.symlink writes the path from the link to what it names', async () => {
let aq = new ActionQueue();
let ctx = makeCtx('cas', aq);
ctx.symlink('__meta/links/sprites/xy/a.gif', 'sprites/xy/a-XXXX.gif');
ctx.symlink('__meta/a.gif', 'sprites/a-XXXX.gif');
let ops = aq.log.filter(e => e.type === 'Op');
assert.ok(aq.valid);
assert.deepEqual(ops.map(e => (e.op as {target: string}).target), [
'../../../../sprites/xy/a-XXXX.gif',
'../sprites/a-XXXX.gif',
]);
});