mirror of
https://github.com/misenhower/splatoon3.ink.git
synced 2026-11-18 16:58:18 -06:00
Compress archive days in a Cloudflare Container
A separate compressor Worker runs app/data/ArchiveCompressor.mjs in a container (standard-1) once a day at 00:30 UTC, against the archive Space. Its Durable Object starts the container and keeps the last 30 runs with their log lines. An alarm every few seconds collects the container's output from its log server and ends the run when the container reports its exit code, so a run survives the object being evicted from memory. Only dev is deployed; production waits until spectre stops compressing at the same time. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -34,6 +34,7 @@ npm run updater:deploy:dry-run # Bundle the dev Worker
|
||||
npm run updater:deploy # Deploy dev (production: updater:deploy:production)
|
||||
npm run admin:dev # Admin panel with a simulated API (no Worker needed)
|
||||
npm run admin:build # Build the admin panel (deploys do this first)
|
||||
npm run compressor:deploy # Deploy the dev archive compressor (workers/compressor/, needs Docker)
|
||||
```
|
||||
|
||||
## Architecture
|
||||
@@ -48,7 +49,7 @@ npm run admin:build # Build the admin panel (deploys do this first
|
||||
|
||||
**Admin panel** (`workers/updater/admin/`): a separate small Vue + Tailwind app (its own Vite config, not part of the public site's bundle), served by the updater Worker on its own hostname behind Cloudflare Access. Its API is `workers/updater/src/admin/routes.mjs`; `admin/mock/api.mjs` simulates that API for `npm run admin:dev` and must be kept in step with it. Pure logic lives in `admin/src/lib/` with tests alongside.
|
||||
|
||||
**Scheduling**: In Cloudflare, `workers/updater/` runs two Scheduler Durable Object lanes (fast: time-sensitive data + social; slow: X-Rank, rankings, avatars) and one RegionAuth Durable Object per region for Nintendo tokens. Locally, `app/cron.mjs` runs the same steps on a node-cron schedule.
|
||||
**Scheduling**: In Cloudflare, `workers/updater/` runs two Scheduler Durable Object lanes (fast: time-sensitive data + social; slow: X-Rank, rankings, avatars) and one RegionAuth Durable Object per region for Nintendo tokens. `workers/compressor/` compresses each finished day of archive snapshots into a tar.zst once a day, running `app/data/ArchiveCompressor.mjs` in a Cloudflare Container (deploying it needs Docker or OrbStack). Locally, `app/cron.mjs` runs the same steps on a node-cron schedule.
|
||||
|
||||
## Code Style
|
||||
|
||||
@@ -66,6 +67,6 @@ Tests use Vitest. Test files live alongside source: `app/**/*.test.mjs`, `src/**
|
||||
## Deployment
|
||||
|
||||
- Frontend: built to `dist/` and served by Cloudflare Pages
|
||||
- Backend: being migrated to Cloudflare Workers; there is no longer a Docker image or server deployment built from this repo
|
||||
- Backend: being migrated to Cloudflare Workers; there is no longer a server deployment built from this repo. The only Docker image is the archive compressor's container (`workers/compressor/Dockerfile`), which `wrangler deploy` builds and pushes to Cloudflare
|
||||
- `dist/` is not emptied on build (preserves generated `dist/data/` from backend)
|
||||
- Local screenshots run on the host with puppeteer-core and an installed Chrome; no separate browser service is needed
|
||||
|
||||
@@ -34,6 +34,10 @@
|
||||
"updater:deploy": "npm run admin:build && wrangler deploy --config workers/updater/wrangler.jsonc --env dev",
|
||||
"updater:deploy:production": "npm run admin:build && wrangler deploy --config workers/updater/wrangler.jsonc --env production",
|
||||
"updater:tail": "wrangler tail --config workers/updater/wrangler.jsonc --env dev",
|
||||
"compressor:deploy:dry-run": "wrangler deploy --config workers/compressor/wrangler.jsonc --env dev --dry-run",
|
||||
"compressor:deploy": "wrangler deploy --config workers/compressor/wrangler.jsonc --env dev",
|
||||
"compressor:deploy:production": "wrangler deploy --config workers/compressor/wrangler.jsonc --env production",
|
||||
"compressor:tail": "wrangler tail --config workers/compressor/wrangler.jsonc --env dev",
|
||||
"cron": "node app/index.mjs cron",
|
||||
"start": "npm run splatnet:quick && npm run social && npm run cron",
|
||||
"social": "node app/index.mjs social",
|
||||
|
||||
@@ -14,6 +14,7 @@ export default defineConfig({
|
||||
'app/**/*.test.mjs',
|
||||
'src/**/*.test.{js,mjs}',
|
||||
'workers/updater/admin/src/**/*.test.js',
|
||||
'workers/compressor/src/**/*.test.mjs',
|
||||
],
|
||||
setupFiles: ['app/test/setup.mjs'],
|
||||
},
|
||||
|
||||
24
workers/compressor/Dockerfile
Normal file
24
workers/compressor/Dockerfile
Normal file
@@ -0,0 +1,24 @@
|
||||
# The archive compressor's container. Built with the repository root as its context (see
|
||||
# image_build_context in wrangler.jsonc); Dockerfile.dockerignore keeps that context small.
|
||||
FROM node:24-slim
|
||||
|
||||
# tar is in the base image; zstd does the compression (app/data/TarZstdWriter.mjs).
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends zstd \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# The same locked dependencies as the rest of the backend. Install scripts are skipped: the
|
||||
# compressor only needs the S3 client and Sentry, not sharp's native setup.
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force
|
||||
|
||||
COPY app ./app
|
||||
COPY workers/compressor/container ./workers/compressor/container
|
||||
|
||||
USER node
|
||||
ENV NODE_ENV=production
|
||||
# Where the container serves its log lines to the Durable Object (container/logServer.mjs).
|
||||
EXPOSE 8080
|
||||
CMD ["node", "workers/compressor/container/compress.mjs"]
|
||||
7
workers/compressor/Dockerfile.dockerignore
Normal file
7
workers/compressor/Dockerfile.dockerignore
Normal file
@@ -0,0 +1,7 @@
|
||||
# The build context is the whole repository; send only what the image copies.
|
||||
*
|
||||
!package.json
|
||||
!package-lock.json
|
||||
!app
|
||||
!workers/compressor/container
|
||||
**/*.test.mjs
|
||||
74
workers/compressor/README.md
Normal file
74
workers/compressor/README.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# splatoon3.ink archive compressor Worker
|
||||
|
||||
Once a day, bundles each finished day of data snapshots in the archive Space (DigitalOcean
|
||||
Spaces) into `YYYY/MM/YYYY-MM-DD.tar.zst` plus a `.manifest.json`, using the same code as
|
||||
`npm run data:archive:compress` (`app/data/ArchiveCompressor.mjs`). The updater Worker writes
|
||||
the snapshots; this Worker never touches R2.
|
||||
|
||||
## How it runs
|
||||
|
||||
- A cron trigger at 00:30 UTC asks the `Compressor` Durable Object (named `archive`) to start
|
||||
a run. Days still being written (today, or anything changed in the last 30 minutes) are
|
||||
skipped, and a run compresses at most 10 days.
|
||||
- The work happens in a Cloudflare Container (`Dockerfile`, entry point
|
||||
`container/compress.mjs`) because it needs `tar` and `zstd`. It runs on `standard-1`
|
||||
(½ vCPU, 4 GiB, 8 GB disk); a production day (~430 MB of snapshots) takes under a minute.
|
||||
- What the container prints (the `[Archive Compressor]` lines) goes to stdout, which is in
|
||||
Workers Logs (not `wrangler tail`). The container also serves it on port 8080
|
||||
(`container/logServer.mjs`), and once the work is done, its exit code too.
|
||||
- While a run goes on, an alarm every three seconds fetches the new lines (up to 500 are kept
|
||||
with each run) and checks on the container. The run ends when the container reports its exit
|
||||
code; the container waits (up to two minutes) until that report has been fetched, then
|
||||
exits. The object keeps the last 30 runs (exit code, error, timing, log), and only one run
|
||||
happens at a time.
|
||||
- Nothing depends on the Durable Object staying in memory: Cloudflare can evict it mid-run (it
|
||||
did, before the alarm took over), and the next alarm carries on from what is stored.
|
||||
- A container that stops without reporting (a crash, running out of memory) fails the run
|
||||
with "The container stopped before it reported how the run went"; its log shows how far it
|
||||
got. A run past three hours is stopped.
|
||||
- The updater's admin panel shows the runs (timeline and Controls) and can start one, through
|
||||
its `COMPRESSOR` binding to this Worker's Durable Object.
|
||||
|
||||
The compressor doesn't delete the source snapshots; `npm run data:archive:prune` does that,
|
||||
by hand.
|
||||
|
||||
## Setup (once per environment)
|
||||
|
||||
1. Docker or a Docker-compatible CLI (OrbStack) must be running: `wrangler deploy` builds the
|
||||
image from the repository root and pushes it to Cloudflare's registry.
|
||||
2. Secrets (`npx wrangler secret put NAME --config workers/compressor/wrangler.jsonc --env dev`):
|
||||
`RUN_TOKEN`, `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (a Spaces key for the archive
|
||||
bucket), optional `SENTRY_DSN`.
|
||||
3. The bucket is a var: dev uses `splatoon3ink-archive-test` and must never point at the
|
||||
production `splatoon3ink-archive`.
|
||||
4. Production: deploy only once spectre stops, which also compresses at 00:30 UTC.
|
||||
|
||||
## Commands
|
||||
|
||||
```sh
|
||||
npm run compressor:deploy:dry-run # build the image and bundle the dev Worker
|
||||
npm run compressor:deploy # deploy dev
|
||||
npm run compressor:deploy:production # deploy production
|
||||
npm run compressor:tail # stream dev Worker logs (not the container's output)
|
||||
```
|
||||
|
||||
With `RUN_TOKEN` in `workers/compressor/.dev.vars`:
|
||||
|
||||
```sh
|
||||
AUTH="Authorization: Bearer $RUN_TOKEN"
|
||||
BASE=https://splatoon3-ink-dev-compressor.ike.workers.dev
|
||||
curl -H "$AUTH" "$BASE/status" # the run in progress and recent runs
|
||||
curl -H "$AUTH" "$BASE/run?id=RUN_ID" # one run with its log lines
|
||||
curl -X POST -H "$AUTH" "$BASE/run" # compress now (202, or 409 if running)
|
||||
curl -X POST -H "$AUTH" "$BASE/run?dryRun=true" # list what would be compressed
|
||||
curl -X POST -H "$AUTH" "$BASE/run?maxDays=30" # catch up on more days
|
||||
```
|
||||
|
||||
To try the image locally (OrbStack), against dev's bucket only:
|
||||
|
||||
```sh
|
||||
docker build --platform linux/amd64 -f workers/compressor/Dockerfile -t splatoon3ink-compressor .
|
||||
docker run --rm --platform linux/amd64 -e AWS_S3_ENDPOINT -e AWS_REGION -e AWS_ACCESS_KEY_ID \
|
||||
-e AWS_SECRET_ACCESS_KEY -e AWS_S3_ARCHIVE_BUCKET=splatoon3ink-archive-test -e DRY_RUN=true \
|
||||
splatoon3ink-compressor
|
||||
```
|
||||
28
workers/compressor/container/compress.mjs
Normal file
28
workers/compressor/container/compress.mjs
Normal file
@@ -0,0 +1,28 @@
|
||||
// Container entry point: compress finished days in the archive Space into daily tar.zst files,
|
||||
// then exit. The Compressor Durable Object starts it and reads the exit code.
|
||||
//
|
||||
// Environment: AWS_S3_ENDPOINT, AWS_REGION, AWS_S3_ARCHIVE_BUCKET, AWS_ACCESS_KEY_ID and
|
||||
// AWS_SECRET_ACCESS_KEY (see ArchiveCompressor), MAX_DAYS, DRY_RUN, optional SENTRY_DSN, and
|
||||
// LOG_PORT (where the Durable Object fetches what this prints).
|
||||
|
||||
import * as Sentry from '@sentry/node';
|
||||
import { sentryInit } from '../../../app/common/sentry.mjs';
|
||||
import { compressArchives } from '../../../app/data/ArchiveCompressor.mjs';
|
||||
import { serveLogs } from './logServer.mjs';
|
||||
|
||||
let logs = serveLogs(process.env.LOG_PORT);
|
||||
|
||||
sentryInit();
|
||||
|
||||
let maxDays = Number(process.env.MAX_DAYS || 10);
|
||||
let dryRun = process.env.DRY_RUN === 'true';
|
||||
|
||||
try {
|
||||
// Errors are logged and sent to Sentry by the compressor itself.
|
||||
await compressArchives(maxDays, dryRun);
|
||||
} catch {
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await Sentry.flush(2000).catch(() => {});
|
||||
await logs.close(process.exitCode ?? 0);
|
||||
}
|
||||
98
workers/compressor/container/logServer.mjs
Normal file
98
workers/compressor/container/logServer.mjs
Normal file
@@ -0,0 +1,98 @@
|
||||
// Keeps what the compressor prints and serves it to the Compressor Durable Object, which fetches
|
||||
// it every few seconds (GET /log?after=N) and keeps it with the run for the admin panel. Once
|
||||
// the work is done the answer also carries the exit code, which is how the object learns how
|
||||
// the run went. Without LOG_PORT (a local `docker run`) lines only go to stdout.
|
||||
|
||||
import http from 'node:http';
|
||||
import { format } from 'node:util';
|
||||
|
||||
const LEVELS = { debug: 'debug', log: 'info', info: 'info', warn: 'warn', error: 'error' };
|
||||
const MAX_LINE_LENGTH = 600;
|
||||
// How long to wait at the end for the object to fetch the report and the last lines. It checks
|
||||
// every few seconds, so this only runs out if it has stopped checking.
|
||||
const COLLECT_TIMEOUT_MS = 2 * 60 * 1000;
|
||||
|
||||
function describe(value) {
|
||||
if (typeof value === 'string') {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (value instanceof Error) {
|
||||
return value.cause ? `${value.message} (cause: ${describe(value.cause)})` : value.message;
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.stringify(value) ?? String(value);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep console output and serve it on `port`.
|
||||
* @param {string|undefined} port
|
||||
* @returns {{ close: (code: number) => Promise<void> }} close() reports the exit code, waits
|
||||
* until it and every line have been fetched (or a timeout), then stops serving; await it
|
||||
* before exiting
|
||||
*/
|
||||
export function serveLogs(port) {
|
||||
if (!port) {
|
||||
return { close: async () => {} };
|
||||
}
|
||||
|
||||
let lines = [];
|
||||
let done = false;
|
||||
let exitCode = null;
|
||||
let collected = () => {};
|
||||
|
||||
for (let [method, level] of Object.entries(LEVELS)) {
|
||||
let original = console[method].bind(console);
|
||||
|
||||
console[method] = (...args) => {
|
||||
// printf-style placeholders are filled in the way console does it.
|
||||
let text = typeof args[0] === 'string' && /%[sdifjoOc]/.test(args[0])
|
||||
? format(...args)
|
||||
: args.map(describe).join(' ');
|
||||
|
||||
lines.push({ at: Date.now(), level, text: text.length > MAX_LINE_LENGTH ? `${text.slice(0, MAX_LINE_LENGTH)}…` : text });
|
||||
original(...args);
|
||||
};
|
||||
}
|
||||
|
||||
let server = http.createServer((request, response) => {
|
||||
let url = new URL(request.url, 'http://container');
|
||||
|
||||
if (request.method !== 'GET' || url.pathname !== '/log') {
|
||||
response.writeHead(404).end();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
let after = Math.max(0, Number(url.searchParams.get('after')) || 0);
|
||||
let finished = done;
|
||||
|
||||
response.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
response.end(JSON.stringify({ lines: lines.slice(after), next: lines.length, done: finished, exitCode }));
|
||||
|
||||
if (finished) {
|
||||
collected();
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(Number(port));
|
||||
|
||||
return {
|
||||
async close(code) {
|
||||
exitCode = code;
|
||||
done = true;
|
||||
|
||||
await Promise.race([
|
||||
new Promise(resolve => { collected = resolve; }),
|
||||
new Promise(resolve => setTimeout(resolve, COLLECT_TIMEOUT_MS).unref()),
|
||||
]);
|
||||
|
||||
// Lets the last response finish sending.
|
||||
await new Promise(resolve => server.close(resolve));
|
||||
},
|
||||
};
|
||||
}
|
||||
230
workers/compressor/src/Compressor.mjs
Normal file
230
workers/compressor/src/Compressor.mjs
Normal file
@@ -0,0 +1,230 @@
|
||||
// One Durable Object that runs the archive compressor's container to completion and keeps a
|
||||
// short history of runs. The container does the work (container/compress.mjs) and serves what
|
||||
// it prints, and at the end how it went, on LOG_PORT (container/logServer.mjs).
|
||||
//
|
||||
// While a run goes on, an alarm every few seconds fetches the new lines and stores them with
|
||||
// the run. Nothing depends on this object staying in memory: it can be evicted mid-run, and the
|
||||
// next alarm carries on from what is stored. The run ends when the container reports its exit
|
||||
// code; the container waits until that report has been fetched before it exits.
|
||||
|
||||
import { DurableObject } from 'cloudflare:workers';
|
||||
import { nextScheduledRun } from './schedule.mjs';
|
||||
|
||||
const HISTORY = 30;
|
||||
// How often the alarm fetches log lines and checks on the container while a run goes on.
|
||||
const CHECK_INTERVAL_MS = 3000;
|
||||
// The container keeps running this long without hearing from this object.
|
||||
const INACTIVITY_TIMEOUT_MS = 15 * 60 * 1000;
|
||||
// A run that goes on this long is stuck (a day takes a few minutes); stop it.
|
||||
const MAX_RUNTIME_MS = 3 * 60 * 60 * 1000;
|
||||
// A container that is not running this long after its run started has stopped.
|
||||
const START_GRACE_MS = 60 * 1000;
|
||||
const LOG_PORT = 8080;
|
||||
const MAX_LOG_LINES = 500;
|
||||
const STOPPED = 'The container stopped before it reported how the run went';
|
||||
|
||||
export class Compressor extends DurableObject {
|
||||
// While this object stays in memory, monitor() reports a crashed container's exit code.
|
||||
#watching = null;
|
||||
|
||||
/**
|
||||
* @returns {Promise<{ now: number, nextAt: number, current: object|null, runs: object[] }>} The
|
||||
* run in progress, recent runs (newest first), and when the next scheduled run starts
|
||||
*/
|
||||
async status() {
|
||||
let now = Date.now();
|
||||
|
||||
return {
|
||||
now,
|
||||
nextAt: nextScheduledRun(now),
|
||||
current: (await this.ctx.storage.get('current')) ?? null,
|
||||
runs: (await this.ctx.storage.get('runs')) ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* One run with what it printed, or null if it is no longer kept.
|
||||
* @param {string} id
|
||||
*/
|
||||
async run(id) {
|
||||
let current = await this.ctx.storage.get('current');
|
||||
let run = current?.id === id
|
||||
? current
|
||||
: ((await this.ctx.storage.get('runs')) ?? []).find(run => run.id === id);
|
||||
|
||||
if (!run) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let { lines, omitted } = (await this.ctx.storage.get(`log:${id}`)) ?? { lines: [], omitted: 0 };
|
||||
|
||||
return { ...run, running: run === current, logs: { lines, omitted } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a run unless one is going.
|
||||
* @param {{ trigger: 'scheduled'|'manual', maxDays?: number, dryRun?: boolean }} options
|
||||
*/
|
||||
async start({ trigger, maxDays = 10, dryRun = false }) {
|
||||
if (!Number.isInteger(maxDays) || maxDays < 1) {
|
||||
return { ok: false, error: 'maxDays must be a whole number of at least 1' };
|
||||
}
|
||||
|
||||
let current = await this.ctx.storage.get('current');
|
||||
if (current) {
|
||||
return { ok: false, busy: true, current };
|
||||
}
|
||||
|
||||
// The last run has reported and finished, but its container can take a moment to exit.
|
||||
if (this.ctx.container.running) {
|
||||
return { ok: false, busy: true, error: 'The last run\'s container is still stopping. Try again in a minute.' };
|
||||
}
|
||||
|
||||
current = { id: crypto.randomUUID(), trigger, maxDays, dryRun, startedAt: Date.now() };
|
||||
await this.ctx.storage.put('current', current);
|
||||
|
||||
try {
|
||||
this.ctx.container.start({ enableInternet: true, env: this.#containerEnv(current) });
|
||||
await this.ctx.container.setInactivityTimeout(INACTIVITY_TIMEOUT_MS);
|
||||
await this.ctx.storage.setAlarm(Date.now() + CHECK_INTERVAL_MS);
|
||||
} catch (e) {
|
||||
let run = await this.#finish({ ok: false, error: `Could not start the container: ${e?.message ?? e}` });
|
||||
|
||||
return { ok: false, run };
|
||||
}
|
||||
|
||||
this.#watch(current.id);
|
||||
|
||||
return { ok: true, current };
|
||||
}
|
||||
|
||||
async alarm() {
|
||||
let current = await this.ctx.storage.get('current');
|
||||
if (!current) {
|
||||
return;
|
||||
}
|
||||
|
||||
// The container is not listening for the first moment, so a failed fetch is expected.
|
||||
let report = await this.#fetchLogs(current.id).catch(() => null);
|
||||
|
||||
if (report?.done) {
|
||||
let exitCode = Number.isInteger(report.exitCode) ? report.exitCode : null;
|
||||
|
||||
await this.#finish(exitCode === 0 ? { ok: true, exitCode } : { ok: false, exitCode, error: `The compressor exited with code ${exitCode}` });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
let elapsed = Date.now() - current.startedAt;
|
||||
|
||||
if (!this.ctx.container.running && elapsed > START_GRACE_MS) {
|
||||
// It crashed, or was stopped, before reporting. Its log shows how far it got.
|
||||
await this.#finish({ ok: false, lost: true, exitCode: null, error: STOPPED });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (elapsed > MAX_RUNTIME_MS) {
|
||||
await this.ctx.container.destroy('Compression ran too long');
|
||||
await this.#finish({ ok: false, exitCode: null, error: 'Stopped after running for three hours' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.ctx.container.setInactivityTimeout(INACTIVITY_TIMEOUT_MS);
|
||||
await this.ctx.storage.setAlarm(Date.now() + CHECK_INTERVAL_MS);
|
||||
this.#watch(current.id);
|
||||
}
|
||||
|
||||
// Store the container's new log lines. Returns its report: done, and the exit code once done.
|
||||
async #fetchLogs(id) {
|
||||
let log = (await this.ctx.storage.get(`log:${id}`)) ?? { lines: [], omitted: 0, next: 0 };
|
||||
let response = await this.ctx.container.getTcpPort(LOG_PORT).fetch(`http://container/log?after=${log.next ?? 0}`);
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let { lines, next, done, exitCode } = await response.json();
|
||||
|
||||
if (lines.length) {
|
||||
await this.#appendLog(id, lines, next);
|
||||
}
|
||||
|
||||
return { done, exitCode };
|
||||
}
|
||||
|
||||
async #appendLog(id, lines, next) {
|
||||
let log = (await this.ctx.storage.get(`log:${id}`)) ?? { lines: [], omitted: 0, next: 0 };
|
||||
|
||||
log.next = next ?? log.next;
|
||||
|
||||
for (let { at, level, text } of lines) {
|
||||
if (log.lines.length >= MAX_LOG_LINES) {
|
||||
log.omitted++;
|
||||
} else {
|
||||
log.lines.push({ at: Number(at) || Date.now(), level: String(level ?? 'info'), text: String(text ?? '') });
|
||||
}
|
||||
}
|
||||
|
||||
await this.ctx.storage.put(`log:${id}`, log);
|
||||
}
|
||||
|
||||
#containerEnv({ maxDays, dryRun }) {
|
||||
let env = {
|
||||
AWS_S3_ENDPOINT: this.env.AWS_S3_ENDPOINT,
|
||||
AWS_REGION: this.env.AWS_REGION,
|
||||
AWS_S3_ARCHIVE_BUCKET: this.env.AWS_S3_ARCHIVE_BUCKET,
|
||||
AWS_ACCESS_KEY_ID: this.env.AWS_ACCESS_KEY_ID,
|
||||
AWS_SECRET_ACCESS_KEY: this.env.AWS_SECRET_ACCESS_KEY,
|
||||
SENTRY_DSN: this.env.SENTRY_DSN,
|
||||
LOG_PORT: String(LOG_PORT),
|
||||
MAX_DAYS: String(maxDays),
|
||||
DRY_RUN: String(dryRun),
|
||||
};
|
||||
|
||||
// Unset values are left out rather than passed as "undefined".
|
||||
return Object.fromEntries(Object.entries(env).filter(([, value]) => value));
|
||||
}
|
||||
|
||||
// A container that dies (out of memory, say) never reports; monitor() gives its exit code if
|
||||
// this object is still in memory. A clean exit comes after the report, which already ended
|
||||
// the run, so only failures count here.
|
||||
#watch(id) {
|
||||
if (this.#watching) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.#watching = this.ctx.container.monitor()
|
||||
.then(
|
||||
() => null,
|
||||
async error => {
|
||||
if ((await this.ctx.storage.get('current'))?.id === id) {
|
||||
await this.#finish({ ok: false, exitCode: error?.exitCode ?? null, error: String(error?.message ?? error) });
|
||||
}
|
||||
},
|
||||
)
|
||||
.finally(() => {
|
||||
this.#watching = null;
|
||||
});
|
||||
}
|
||||
|
||||
async #finish(result) {
|
||||
let current = await this.ctx.storage.get('current');
|
||||
if (!current) {
|
||||
return;
|
||||
}
|
||||
|
||||
let run = { ...current, ...result, finishedAt: Date.now() };
|
||||
let kept = [run, ...((await this.ctx.storage.get('runs')) ?? [])];
|
||||
|
||||
await this.ctx.storage.put('runs', kept.slice(0, HISTORY));
|
||||
await this.ctx.storage.delete(kept.slice(HISTORY).map(old => `log:${old.id}`));
|
||||
await this.ctx.storage.delete('current');
|
||||
await this.ctx.storage.deleteAlarm();
|
||||
|
||||
console[run.ok ? 'info' : 'error'](JSON.stringify({ event: 'compress-run', ...run }));
|
||||
|
||||
return run;
|
||||
}
|
||||
}
|
||||
74
workers/compressor/src/index.mjs
Normal file
74
workers/compressor/src/index.mjs
Normal file
@@ -0,0 +1,74 @@
|
||||
// splatoon3.ink archive compressor Worker: once a day, runs app/data/ArchiveCompressor.mjs in a
|
||||
// Cloudflare Container against the archive Space. See workers/compressor/README.md.
|
||||
|
||||
import { Compressor } from './Compressor.mjs';
|
||||
|
||||
export { Compressor };
|
||||
|
||||
function compressor(env) {
|
||||
return env.COMPRESSOR.getByName('archive');
|
||||
}
|
||||
|
||||
function timingSafeEqual(a, b) {
|
||||
let encoder = new TextEncoder();
|
||||
let left = encoder.encode(a);
|
||||
let right = encoder.encode(b);
|
||||
|
||||
return left.byteLength === right.byteLength && crypto.subtle.timingSafeEqual(left, right);
|
||||
}
|
||||
|
||||
function authorized(request, env) {
|
||||
if (!env.RUN_TOKEN) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let [scheme, token] = (request.headers.get('Authorization') ?? '').split(' ');
|
||||
|
||||
return scheme === 'Bearer' && !!token && timingSafeEqual(token, env.RUN_TOKEN);
|
||||
}
|
||||
|
||||
function json(body, status = 200) {
|
||||
return Response.json(body, { status, headers: { 'Cache-Control': 'no-store' } });
|
||||
}
|
||||
|
||||
export default {
|
||||
async fetch(request, env) {
|
||||
if (!authorized(request, env)) {
|
||||
return json({ ok: false, error: 'Unauthorized' }, 401);
|
||||
}
|
||||
|
||||
let url = new URL(request.url);
|
||||
let route = `${request.method} ${url.pathname}`;
|
||||
|
||||
if (route === 'GET /status') {
|
||||
return json(await compressor(env).status());
|
||||
}
|
||||
|
||||
if (route === 'GET /run') {
|
||||
let run = await compressor(env).run(url.searchParams.get('id') ?? '');
|
||||
|
||||
return run ? json(run) : json({ ok: false, error: 'Not found' }, 404);
|
||||
}
|
||||
|
||||
if (route === 'POST /run') {
|
||||
let maxDays = url.searchParams.get('maxDays');
|
||||
let result = await compressor(env).start({
|
||||
trigger: 'manual',
|
||||
maxDays: maxDays === null ? undefined : Number(maxDays),
|
||||
dryRun: url.searchParams.get('dryRun') === 'true',
|
||||
});
|
||||
|
||||
return json(result, result.ok ? 202 : result.busy ? 409 : 400);
|
||||
}
|
||||
|
||||
return json({ ok: false, error: 'Not found' }, 404);
|
||||
},
|
||||
|
||||
async scheduled(controller, env) {
|
||||
let result = await compressor(env).start({ trigger: 'scheduled' });
|
||||
|
||||
if (!result.ok) {
|
||||
console.warn(JSON.stringify({ event: 'compress-skipped', ...result }));
|
||||
}
|
||||
},
|
||||
};
|
||||
18
workers/compressor/src/schedule.mjs
Normal file
18
workers/compressor/src/schedule.mjs
Normal file
@@ -0,0 +1,18 @@
|
||||
// When the cron trigger in wrangler.jsonc ("30 0 * * *") starts a run. Keep the two in step.
|
||||
const HOUR_UTC = 0;
|
||||
const MINUTE_UTC = 30;
|
||||
|
||||
/**
|
||||
* @param {number} now Epoch milliseconds
|
||||
* @returns {number} The next scheduled run after now, in epoch milliseconds
|
||||
*/
|
||||
export function nextScheduledRun(now) {
|
||||
let next = new Date(now);
|
||||
next.setUTCHours(HOUR_UTC, MINUTE_UTC, 0, 0);
|
||||
|
||||
if (next.getTime() <= now) {
|
||||
next.setUTCDate(next.getUTCDate() + 1);
|
||||
}
|
||||
|
||||
return next.getTime();
|
||||
}
|
||||
12
workers/compressor/src/schedule.test.mjs
Normal file
12
workers/compressor/src/schedule.test.mjs
Normal file
@@ -0,0 +1,12 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { nextScheduledRun } from './schedule.mjs';
|
||||
|
||||
const next = iso => new Date(nextScheduledRun(Date.parse(iso))).toISOString();
|
||||
|
||||
describe('nextScheduledRun', () => {
|
||||
it('is 00:30 UTC today until then, and tomorrow from then on', () => {
|
||||
expect(next('2026-11-12T00:29:59Z')).toBe('2026-11-12T00:30:00.000Z');
|
||||
expect(next('2026-11-12T00:30:00Z')).toBe('2026-11-13T00:30:00.000Z');
|
||||
expect(next('2026-12-31T23:00:00Z')).toBe('2027-01-01T00:30:00.000Z');
|
||||
});
|
||||
});
|
||||
59
workers/compressor/wrangler.jsonc
Normal file
59
workers/compressor/wrangler.jsonc
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"$schema": "../../node_modules/wrangler/config-schema.json",
|
||||
"name": "splatoon3-ink-compressor",
|
||||
"main": "src/index.mjs",
|
||||
"compatibility_date": "2026-09-03",
|
||||
"compatibility_flags": ["nodejs_compat"],
|
||||
"workers_dev": true,
|
||||
"preview_urls": false,
|
||||
"migrations": [{ "tag": "v1", "new_sqlite_classes": ["Compressor"] }],
|
||||
"observability": { "enabled": true, "logs": { "head_sampling_rate": 1 } },
|
||||
|
||||
// Select --env dev or --env production explicitly. Deploying builds the container image with
|
||||
// a local Docker-compatible CLI (Dockerfile, built from the repository root).
|
||||
// Secrets: RUN_TOKEN, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY (a DigitalOcean Spaces key
|
||||
// that can read and write the archive bucket), optional SENTRY_DSN.
|
||||
"env": {
|
||||
"dev": {
|
||||
"name": "splatoon3-ink-dev-compressor",
|
||||
"vars": {
|
||||
// Dev's own archive Space, the one the dev updater writes. Never the production one.
|
||||
"AWS_S3_ENDPOINT": "https://nyc3.digitaloceanspaces.com",
|
||||
"AWS_REGION": "nyc3",
|
||||
"AWS_S3_ARCHIVE_BUCKET": "splatoon3ink-archive-test"
|
||||
},
|
||||
"triggers": { "crons": ["30 0 * * *"] },
|
||||
"containers": [
|
||||
{
|
||||
"class_name": "Compressor",
|
||||
"image": "./Dockerfile",
|
||||
"image_build_context": "../..",
|
||||
// 1/2 vCPU, 4 GiB, 8 GB disk: a day (~430 MB of snapshots) compresses in under a minute.
|
||||
"instance_type": "standard-1",
|
||||
"max_instances": 1
|
||||
}
|
||||
],
|
||||
"durable_objects": { "bindings": [{ "name": "COMPRESSOR", "class_name": "Compressor" }] }
|
||||
},
|
||||
"production": {
|
||||
"name": "splatoon3-ink-compressor",
|
||||
"vars": {
|
||||
"AWS_S3_ENDPOINT": "https://nyc3.digitaloceanspaces.com",
|
||||
"AWS_REGION": "nyc3",
|
||||
"AWS_S3_ARCHIVE_BUCKET": "splatoon3ink-archive"
|
||||
},
|
||||
// Spectre also compresses at 00:30 UTC until cutover; deploy this only once it stops.
|
||||
"triggers": { "crons": ["30 0 * * *"] },
|
||||
"containers": [
|
||||
{
|
||||
"class_name": "Compressor",
|
||||
"image": "./Dockerfile",
|
||||
"image_build_context": "../..",
|
||||
"instance_type": "standard-1",
|
||||
"max_instances": 1
|
||||
}
|
||||
],
|
||||
"durable_objects": { "bindings": [{ "name": "COMPRESSOR", "class_name": "Compressor" }] }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user