diff --git a/tools/build/cas.ts b/tools/build/cas.ts new file mode 100644 index 00000000..ec6bcffa --- /dev/null +++ b/tools/build/cas.ts @@ -0,0 +1,72 @@ + +import fs from 'fs'; +import pathlib from 'path'; + +import {hashFileSync} from './hash.ts'; + +// Content-addressed store for build outputs. Objects live at +// //. (hh = first two hex chars), carry their +// extension so format-sniffing tools can consume them directly, and are +// read-only: a command substituting a CAS path for %f cannot corrupt the +// store. Writers stage the file elsewhere on the same filesystem and insert +// it with an atomic rename. + +export function casPath(casDir : string, digest : string, ext : string) : string { + return pathlib.join(casDir, digest.slice(0, 2), `${digest}.${ext}`); +} + +export function casExists(casDir : string, digest : string, ext : string) : boolean { + try { + return fs.statSync(casPath(casDir, digest, ext)).isFile(); + } catch { + return false; + } +} + +// Move tmpPath into the store, returning the content digest (hex). If the +// object already exists the temp file is simply discarded. +export function casInsert(casDir : string, tmpPath : string, ext : string) : string { + const digest = hashFileSync(tmpPath).toString('hex'); + const target = casPath(casDir, digest, ext); + if (fs.existsSync(target)) { + fs.unlinkSync(tmpPath); + return digest; + } + fs.mkdirSync(pathlib.dirname(target), {recursive: true}); + fs.chmodSync(tmpPath, 0o444); + fs.renameSync(tmpPath, target); + return digest; +} + +// Remove every object not in `live` (keys are ".", the object +// basename) and prune emptied fanout directories. Returns the removal count. +export function casSweep(casDir : string, live : Set) : number { + let removed = 0; + let fanout : fs.Dirent[]; + try { + fanout = fs.readdirSync(casDir, {withFileTypes: true}); + } catch (err) { + if ((err as {code? : string}).code === 'ENOENT') { + return 0; + } + throw err; + } + for (const dir of fanout) { + if (!dir.isDirectory()) { + continue; + } + const dirPath = pathlib.join(casDir, dir.name); + for (const name of fs.readdirSync(dirPath)) { + if (!live.has(name)) { + fs.unlinkSync(pathlib.join(dirPath, name)); + removed++; + } + } + try { + fs.rmdirSync(dirPath); + } catch { + // not empty + } + } + return removed; +} diff --git a/tools/build/test/cas.test.ts b/tools/build/test/cas.test.ts new file mode 100644 index 00000000..9c19b396 --- /dev/null +++ b/tools/build/test/cas.test.ts @@ -0,0 +1,66 @@ + +import assert from 'node:assert/strict'; +import {createHash} from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import pathlib from 'node:path'; +import {test} from 'node:test'; + +import {casExists, casInsert, casPath, casSweep} from '../cas.ts'; + +function makeTmpRoot() : string { + return fs.mkdtempSync(pathlib.join(os.tmpdir(), 'cas-test-')); +} + +function stage(root : string, data : string) : string { + const p = pathlib.join(root, `stage-${Math.random().toString(36).slice(2)}`); + fs.writeFileSync(p, data); + return p; +} + +test('casInsert stores by content digest, read-only', () => { + const root = makeTmpRoot(); + const cas = pathlib.join(root, 'cas'); + const digest = casInsert(cas, stage(root, 'hello'), 'png'); + assert.equal(digest, createHash('sha256').update('hello').digest('hex')); + const obj = casPath(cas, digest, 'png'); + assert.equal(fs.readFileSync(obj, 'utf8'), 'hello'); + assert.equal(fs.statSync(obj).mode & 0o777, 0o444); + assert.ok(casExists(cas, digest, 'png')); + assert.ok(!casExists(cas, digest, 'gif')); + fs.rmSync(root, {recursive: true, force: true}); +}); + +test('casInsert dedupes an existing object and discards the temp', () => { + const root = makeTmpRoot(); + const cas = pathlib.join(root, 'cas'); + const d1 = casInsert(cas, stage(root, 'same'), 'png'); + const tmp2 = stage(root, 'same'); + const d2 = casInsert(cas, tmp2, 'png'); + assert.equal(d1, d2); + assert.ok(!fs.existsSync(tmp2)); + // Same bytes under a different extension is a distinct object. + const d3 = casInsert(cas, stage(root, 'same'), 'gif'); + assert.equal(d1, d3); + assert.ok(casExists(cas, d1, 'png')); + assert.ok(casExists(cas, d1, 'gif')); + fs.rmSync(root, {recursive: true, force: true}); +}); + +test('casSweep removes non-live objects and prunes empty fanout dirs', () => { + const root = makeTmpRoot(); + const cas = pathlib.join(root, 'cas'); + const keep = casInsert(cas, stage(root, 'keep'), 'png'); + const drop = casInsert(cas, stage(root, 'drop'), 'png'); + const removed = casSweep(cas, new Set([`${keep}.png`])); + assert.equal(removed, 1); + assert.ok(casExists(cas, keep, 'png')); + assert.ok(!casExists(cas, drop, 'png')); + assert.ok(!fs.existsSync(pathlib.join(cas, drop.slice(0, 2)))); + assert.ok(fs.existsSync(pathlib.join(cas, keep.slice(0, 2)))); + fs.rmSync(root, {recursive: true, force: true}); +}); + +test('casSweep on a missing store is a no-op', () => { + assert.equal(casSweep(pathlib.join(makeTmpRoot(), 'nope'), new Set()), 0); +});