Files
splatoon3.ink/app/data/ArchiveCompressor.mjs
Matt Isenhower 1480138e15 Report Worker errors to Sentry and drop Sentry from the Node CLI
Both Workers and their Durable Objects are wrapped with @sentry/cloudflare, so the shared
code's @sentry/core calls (failed updaters, social posts, archive snapshots) reach Sentry.
A lane run that fails outright, a failed credential renewal and a failed archive compression
are reported too. Durable Object RPC methods use instrumentPrototypeMethods; without it they
run with no Sentry client and their events are dropped.

The Node runner and the compressor's container no longer use Sentry; @sentry/node is gone.
POST /sentry-test sends one test event from the updater Worker and one from a Durable Object.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 14:05:01 -08:00

180 lines
4.4 KiB
JavaScript

import { S3Client } from '@aws-sdk/client-s3';
import prefixedConsole from '../common/prefixedConsole.mjs';
import ArchiveBuilder from './ArchiveBuilder.mjs';
import ArchiveInventory from './ArchiveInventory.mjs';
const quietPeriod = 30 * 60 * 1000;
export function compressArchives(maxDays = Infinity, dryRun = false) {
let compressor = new ArchiveCompressor;
compressor.maxDays = maxDays;
compressor.dryRun = dryRun;
return compressor.process();
}
export function compressArchivesFromCli(args) {
let dryRun = false;
let maxDays = Infinity;
for (let i = 0; i < args.length; i++) {
if (args[i] === '--dry-run') {
dryRun = true;
} else if (args[i] === '--max-days' && /^\d+$/.test(args[i + 1])) {
maxDays = Number(args[++i]);
} else {
throw new Error(
'Usage: npm run data:archive:compress -- [--dry-run] [--max-days DAYS]',
);
}
}
if (maxDays < 1) {
throw new Error('--max-days must be at least 1');
}
return compressArchives(maxDays, dryRun);
}
export default class ArchiveCompressor
{
dryRun = false;
maxDays = Infinity;
constructor(s3Client, archiveBuilder, archiveInventory) {
this._client = s3Client;
this._archiveBuilder = archiveBuilder;
this._archiveInventory = archiveInventory;
}
async process() {
if (!this.canRun) {
this.console.log('Skipping archive compressor');
return;
}
let currentDate;
let compressed = 0;
try {
let candidates = await this.getCandidates();
this.console.log(`Found ${candidates.length} dates to archive`);
for (let candidate of candidates) {
if (compressed >= this.maxDays) {
break;
}
currentDate = candidate.date;
let completed = this.dryRun
? await this.previewDate(candidate)
: await this.archiveDate(candidate);
if (completed) {
compressed++;
}
}
} catch (e) {
this.console.error(currentDate ? `Failed on ${currentDate}:` : 'Failed:', e);
throw e;
}
this.console.log(
this.dryRun
? `Would compress ${compressed} daily archives`
: `Compressed ${compressed} daily archives`,
);
}
// Properties
get console() {
this._console ??= prefixedConsole('Archive Compressor');
return this._console;
}
get canRun() {
return process.env.AWS_S3_ENDPOINT
&& process.env.AWS_REGION
&& process.env.AWS_S3_ARCHIVE_BUCKET
&& process.env.AWS_ACCESS_KEY_ID
&& process.env.AWS_SECRET_ACCESS_KEY;
}
get s3Client() {
return this._client ??= new S3Client({
endpoint: process.env.AWS_S3_ENDPOINT,
region: process.env.AWS_REGION,
requestChecksumCalculation: 'WHEN_REQUIRED',
responseChecksumValidation: 'WHEN_REQUIRED',
credentials: {
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
},
});
}
get archiveBuilder() {
return this._archiveBuilder ??= new ArchiveBuilder(this.s3Client);
}
get archiveInventory() {
return this._archiveInventory ??= new ArchiveInventory(this.s3Client);
}
// Archive compression
async previewDate(candidate) {
let objects = await this.getReadyObjects(candidate);
if (!objects) {
return false;
}
this.console.log(
`Would create ${candidate.archiveKey} and ${candidate.manifestKey} `
+ `from ${objects.length} files`,
);
return true;
}
async archiveDate(candidate) {
let objects = await this.getReadyObjects(candidate);
if (!objects) {
return false;
}
this.console.log(`Creating ${candidate.archiveKey}`);
await this.archiveBuilder.build(candidate, objects);
return true;
}
async getReadyObjects(candidate) {
let objects = await this.archiveInventory.getSourceObjects(candidate.prefix);
if (objects.length === 0) {
return null;
}
let newestObject = Math.max(...objects.map(object => object.lastModified.getTime()));
if (newestObject > Date.now() - quietPeriod) {
this.console.log(`Skipping ${candidate.date}; its source files are still changing`);
return null;
}
return objects;
}
// S3 discovery
async getCandidates() {
let today = new Date().toISOString().slice(0, 10);
return (await this.archiveInventory.getDates())
.filter(item => item.hasSource && !item.hasArchive && item.date < today);
}
}