Add the content-addressed store (cas.ts)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

tools/build/cas.ts

@@ -0,0 +1,72 @@
+
+import fs from 'fs';
+import pathlib from 'path';
+
+import {hashFileSync} from './hash.ts';
+
+// Content-addressed store for build outputs. Objects live at
+// <casDir>/<hh>/<sha256hex>.<ext> (hh = first two hex chars), carry their
+// extension so format-sniffing tools can consume them directly, and are
+// read-only: a command substituting a CAS path for %f cannot corrupt the
+// store. Writers stage the file elsewhere on the same filesystem and insert
+// it with an atomic rename.
+
+export function casPath(casDir : string, digest : string, ext : string) : string {
+    return pathlib.join(casDir, digest.slice(0, 2), `${digest}.${ext}`);
+}
+
+export function casExists(casDir : string, digest : string, ext : string) : boolean {
+    try {
+        return fs.statSync(casPath(casDir, digest, ext)).isFile();
+    } catch {
+        return false;
+    }
+}
+
+// Move tmpPath into the store, returning the content digest (hex). If the
+// object already exists the temp file is simply discarded.
+export function casInsert(casDir : string, tmpPath : string, ext : string) : string {
+    const digest = hashFileSync(tmpPath).toString('hex');
+    const target = casPath(casDir, digest, ext);
+    if (fs.existsSync(target)) {
+        fs.unlinkSync(tmpPath);
+        return digest;
+    }
+    fs.mkdirSync(pathlib.dirname(target), {recursive: true});
+    fs.chmodSync(tmpPath, 0o444);
+    fs.renameSync(tmpPath, target);
+    return digest;
+}
+
+// Remove every object not in `live` (keys are "<digest>.<ext>", the object
+// basename) and prune emptied fanout directories. Returns the removal count.
+export function casSweep(casDir : string, live : Set<string>) : number {
+    let removed = 0;
+    let fanout : fs.Dirent[];
+    try {
+        fanout = fs.readdirSync(casDir, {withFileTypes: true});
+    } catch (err) {
+        if ((err as {code? : string}).code === 'ENOENT') {
+            return 0;
+        }
+        throw err;
+    }
+    for (const dir of fanout) {
+        if (!dir.isDirectory()) {
+            continue;
+        }
+        const dirPath = pathlib.join(casDir, dir.name);
+        for (const name of fs.readdirSync(dirPath)) {
+            if (!live.has(name)) {
+                fs.unlinkSync(pathlib.join(dirPath, name));
+                removed++;
+            }
+        }
+        try {
+            fs.rmdirSync(dirPath);
+        } catch {
+            // not empty
+        }
+    }
+    return removed;
+}

tools/build/test/cas.test.ts

@@ -0,0 +1,66 @@
+
+import assert from 'node:assert/strict';
+import {createHash} from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import pathlib from 'node:path';
+import {test} from 'node:test';
+
+import {casExists, casInsert, casPath, casSweep} from '../cas.ts';
+
+function makeTmpRoot() : string {
+    return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'cas-test-'));
+}
+
+function stage(root : string, data : string) : string {
+    const p = pathlib.join(root, `stage-${Math.random().toString(36).slice(2)}`);
+    fs.writeFileSync(p, data);
+    return p;
+}
+
+test('casInsert stores by content digest, read-only', () => {
+    const root = makeTmpRoot();
+    const cas = pathlib.join(root, 'cas');
+    const digest = casInsert(cas, stage(root, 'hello'), 'png');
+    assert.equal(digest, createHash('sha256').update('hello').digest('hex'));
+    const obj = casPath(cas, digest, 'png');
+    assert.equal(fs.readFileSync(obj, 'utf8'), 'hello');
+    assert.equal(fs.statSync(obj).mode & 0o777, 0o444);
+    assert.ok(casExists(cas, digest, 'png'));
+    assert.ok(!casExists(cas, digest, 'gif'));
+    fs.rmSync(root, {recursive: true, force: true});
+});
+
+test('casInsert dedupes an existing object and discards the temp', () => {
+    const root = makeTmpRoot();
+    const cas = pathlib.join(root, 'cas');
+    const d1 = casInsert(cas, stage(root, 'same'), 'png');
+    const tmp2 = stage(root, 'same');
+    const d2 = casInsert(cas, tmp2, 'png');
+    assert.equal(d1, d2);
+    assert.ok(!fs.existsSync(tmp2));
+    // Same bytes under a different extension is a distinct object.
+    const d3 = casInsert(cas, stage(root, 'same'), 'gif');
+    assert.equal(d1, d3);
+    assert.ok(casExists(cas, d1, 'png'));
+    assert.ok(casExists(cas, d1, 'gif'));
+    fs.rmSync(root, {recursive: true, force: true});
+});
+
+test('casSweep removes non-live objects and prunes empty fanout dirs', () => {
+    const root = makeTmpRoot();
+    const cas = pathlib.join(root, 'cas');
+    const keep = casInsert(cas, stage(root, 'keep'), 'png');
+    const drop = casInsert(cas, stage(root, 'drop'), 'png');
+    const removed = casSweep(cas, new Set([`${keep}.png`]));
+    assert.equal(removed, 1);
+    assert.ok(casExists(cas, keep, 'png'));
+    assert.ok(!casExists(cas, drop, 'png'));
+    assert.ok(!fs.existsSync(pathlib.join(cas, drop.slice(0, 2))));
+    assert.ok(fs.existsSync(pathlib.join(cas, keep.slice(0, 2))));
+    fs.rmSync(root, {recursive: true, force: true});
+});
+
+test('casSweep on a missing store is a no-op', () => {
+    assert.equal(casSweep(pathlib.join(makeTmpRoot(), 'nope'), new Set()), 0);
+});