Move updaters onto public and private storage

Updaters take a storage object instead of touching the filesystem: a
publicStorage (what the site serves: data/, assets/) and a privateStorage
(updater state). Two implementations share one small interface
(exists / readJson / writeJson / readBytes / writeText / writeBytes):

- FilesystemStorage over a directory, used by npm run splatnet, so dist/
  and storage/ end up exactly as they always have and the S3 sync is
  untouched;
- BucketStorage over an R2 bucket binding (MemoryBucket in tests), with a
  thin per-run cache: exists() lists a directory once and answers from
  memory, readJson() parses a document once, and writeJson() skips the put
  when the content is unchanged.

The updater base class and localization processor keep their original
structure with awaits added; the processor reads its document lazily and
writes once per update, as splatoon3.ink's does. Objects written to a
bucket carry the content type and cache-control the S3 sync used to apply.

Tests cover the storage contract (run against both implementations), the
BucketStorage cache behaviour, MemoryBucket's R2-style listing, the
localization processor, the updater base, and each concrete updater.
scripts/compare-data.mjs compares two data directories ignoring SplatNet's
random ordering and ICS timestamps.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Matt Isenhower
2026-09-04 20:12:56 -07:00
parent 2c636257f3
commit 19d1fb5d3d
26 changed files with 975 additions and 178 deletions

77
scripts/compare-data.mjs Normal file
View File

@@ -0,0 +1,77 @@
// Compare two data directories (e.g. a saved baseline and dist/data) the way the site
// cares about: SplatNet returns object keys and some arrays in random order per request,
// and ICS files carry a per-run DTSTAMP, so byte diffs are noise. Usage:
//
// node scripts/compare-data.mjs <baseline-dir> <current-dir>
//
// Exits non-zero when a real difference is found.
import fs from 'node:fs';
import path from 'node:path';
const [base, current] = process.argv.slice(2);
if (!base || !current) {
console.error('Usage: node scripts/compare-data.mjs <baseline-dir> <current-dir>');
process.exit(2);
}
const canon = value => Array.isArray(value)
? value.map(canon)
: (value && typeof value === 'object')
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canon(value[key])]))
: value;
function firstDifference(a, b, at = '$') {
if (Array.isArray(a) && Array.isArray(b)) {
if (a.length !== b.length)
return `${at}.length ${a.length} vs ${b.length}`;
// Festival result lists come back in random order
if (a.length && a[0] && typeof a[0] === 'object' && 'festival_id' in a[0]) {
let key = item => JSON.stringify(canon(item));
let sa = a.map(key).sort(), sb = b.map(key).sort();
return sa.every((v, i) => v === sb[i]) ? null : `${at}: set of items differs`;
}
for (let i = 0; i < a.length; i++) {
let d = firstDifference(a[i], b[i], `${at}[${i}]`);
if (d) return d;
}
return null;
}
if (a && b && typeof a === 'object' && typeof b === 'object') {
for (let key of new Set([...Object.keys(a), ...Object.keys(b)])) {
let d = firstDifference(a[key], b[key], `${at}.${key}`);
if (d) return d;
}
return null;
}
return a === b ? null : `${at}: ${JSON.stringify(a)?.slice(0, 60)} vs ${JSON.stringify(b)?.slice(0, 60)}`;
}
const walk = dir => fs.readdirSync(dir, { withFileTypes: true })
.flatMap(entry => entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]);
let problems = 0;
for (let file of walk(base)) {
let relative = path.relative(base, file);
let other = path.join(current, relative);
if (!fs.existsSync(other)) {
console.log(`${relative}: missing in ${current}`);
problems++;
continue;
}
let a = fs.readFileSync(file), b = fs.readFileSync(other);
if (a.equals(b))
continue;
if (relative.endsWith('.json')) {
let d = firstDifference(JSON.parse(a), JSON.parse(b));
if (d) { console.log(`${relative}: ${d}`); problems++; }
} else if (relative.endsWith('.ics')) {
let strip = s => s.toString().split(/\r?\n/).filter(line => !line.startsWith('DTSTAMP')).join('\n');
if (strip(a) !== strip(b)) { console.log(`${relative}: calendar content differs`); problems++; }
} else {
console.log(`${relative}: binary content differs`);
problems++;
}
}
console.log(problems ? `${problems} real difference(s)` : 'semantically identical');
process.exit(problems ? 1 : 0);