diff --git a/CLAUDE.md b/CLAUDE.md index 5ceb83e..ee33630 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,7 @@ npm run updater:deploy:dry-run # Bundle the dev Worker npm run updater:deploy # Deploy dev (production: updater:deploy:production) npm run admin:dev # Admin panel with a simulated API (no Worker needed) npm run admin:build # Build the admin panel (deploys do this first) +npm run compressor:deploy # Deploy the dev archive compressor (workers/compressor/, needs Docker) ``` ## Architecture @@ -48,7 +49,7 @@ npm run admin:build # Build the admin panel (deploys do this first **Admin panel** (`workers/updater/admin/`): a separate small Vue + Tailwind app (its own Vite config, not part of the public site's bundle), served by the updater Worker on its own hostname behind Cloudflare Access. Its API is `workers/updater/src/admin/routes.mjs`; `admin/mock/api.mjs` simulates that API for `npm run admin:dev` and must be kept in step with it. Pure logic lives in `admin/src/lib/` with tests alongside. -**Scheduling**: In Cloudflare, `workers/updater/` runs two Scheduler Durable Object lanes (fast: time-sensitive data + social; slow: X-Rank, rankings, avatars) and one RegionAuth Durable Object per region for Nintendo tokens. Locally, `app/cron.mjs` runs the same steps on a node-cron schedule. +**Scheduling**: In Cloudflare, `workers/updater/` runs two Scheduler Durable Object lanes (fast: time-sensitive data + social; slow: X-Rank, rankings, avatars) and one RegionAuth Durable Object per region for Nintendo tokens. `workers/compressor/` compresses each finished day of archive snapshots into a tar.zst once a day, running `app/data/ArchiveCompressor.mjs` in a Cloudflare Container (deploying it needs Docker or OrbStack). Locally, `app/cron.mjs` runs the same steps on a node-cron schedule. ## Code Style @@ -66,6 +67,6 @@ Tests use Vitest. Test files live alongside source: `app/**/*.test.mjs`, `src/** ## Deployment - Frontend: built to `dist/` and served by Cloudflare Pages -- Backend: being migrated to Cloudflare Workers; there is no longer a Docker image or server deployment built from this repo +- Backend: being migrated to Cloudflare Workers; there is no longer a server deployment built from this repo. The only Docker image is the archive compressor's container (`workers/compressor/Dockerfile`), which `wrangler deploy` builds and pushes to Cloudflare - `dist/` is not emptied on build (preserves generated `dist/data/` from backend) - Local screenshots run on the host with puppeteer-core and an installed Chrome; no separate browser service is needed diff --git a/package.json b/package.json index 4598154..8ce1934 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,10 @@ "updater:deploy": "npm run admin:build && wrangler deploy --config workers/updater/wrangler.jsonc --env dev", "updater:deploy:production": "npm run admin:build && wrangler deploy --config workers/updater/wrangler.jsonc --env production", "updater:tail": "wrangler tail --config workers/updater/wrangler.jsonc --env dev", + "compressor:deploy:dry-run": "wrangler deploy --config workers/compressor/wrangler.jsonc --env dev --dry-run", + "compressor:deploy": "wrangler deploy --config workers/compressor/wrangler.jsonc --env dev", + "compressor:deploy:production": "wrangler deploy --config workers/compressor/wrangler.jsonc --env production", + "compressor:tail": "wrangler tail --config workers/compressor/wrangler.jsonc --env dev", "cron": "node app/index.mjs cron", "start": "npm run splatnet:quick && npm run social && npm run cron", "social": "node app/index.mjs social", diff --git a/vitest.config.mjs b/vitest.config.mjs index d6b9ce7..227d08f 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -14,6 +14,7 @@ export default defineConfig({ 'app/**/*.test.mjs', 'src/**/*.test.{js,mjs}', 'workers/updater/admin/src/**/*.test.js', + 'workers/compressor/src/**/*.test.mjs', ], setupFiles: ['app/test/setup.mjs'], }, diff --git a/workers/compressor/Dockerfile b/workers/compressor/Dockerfile new file mode 100644 index 0000000..5bd2b6e --- /dev/null +++ b/workers/compressor/Dockerfile @@ -0,0 +1,24 @@ +# The archive compressor's container. Built with the repository root as its context (see +# image_build_context in wrangler.jsonc); Dockerfile.dockerignore keeps that context small. +FROM node:24-slim + +# tar is in the base image; zstd does the compression (app/data/TarZstdWriter.mjs). +RUN apt-get update \ + && apt-get install -y --no-install-recommends zstd \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# The same locked dependencies as the rest of the backend. Install scripts are skipped: the +# compressor only needs the S3 client and Sentry, not sharp's native setup. +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force + +COPY app ./app +COPY workers/compressor/container ./workers/compressor/container + +USER node +ENV NODE_ENV=production +# Where the container serves its log lines to the Durable Object (container/logServer.mjs). +EXPOSE 8080 +CMD ["node", "workers/compressor/container/compress.mjs"] diff --git a/workers/compressor/Dockerfile.dockerignore b/workers/compressor/Dockerfile.dockerignore new file mode 100644 index 0000000..17b0dc6 --- /dev/null +++ b/workers/compressor/Dockerfile.dockerignore @@ -0,0 +1,7 @@ +# The build context is the whole repository; send only what the image copies. +* +!package.json +!package-lock.json +!app +!workers/compressor/container +**/*.test.mjs diff --git a/workers/compressor/README.md b/workers/compressor/README.md new file mode 100644 index 0000000..a0fde2e --- /dev/null +++ b/workers/compressor/README.md @@ -0,0 +1,74 @@ +# splatoon3.ink archive compressor Worker + +Once a day, bundles each finished day of data snapshots in the archive Space (DigitalOcean +Spaces) into `YYYY/MM/YYYY-MM-DD.tar.zst` plus a `.manifest.json`, using the same code as +`npm run data:archive:compress` (`app/data/ArchiveCompressor.mjs`). The updater Worker writes +the snapshots; this Worker never touches R2. + +## How it runs + +- A cron trigger at 00:30 UTC asks the `Compressor` Durable Object (named `archive`) to start + a run. Days still being written (today, or anything changed in the last 30 minutes) are + skipped, and a run compresses at most 10 days. +- The work happens in a Cloudflare Container (`Dockerfile`, entry point + `container/compress.mjs`) because it needs `tar` and `zstd`. It runs on `standard-1` + (½ vCPU, 4 GiB, 8 GB disk); a production day (~430 MB of snapshots) takes under a minute. +- What the container prints (the `[Archive Compressor]` lines) goes to stdout, which is in + Workers Logs (not `wrangler tail`). The container also serves it on port 8080 + (`container/logServer.mjs`), and once the work is done, its exit code too. +- While a run goes on, an alarm every three seconds fetches the new lines (up to 500 are kept + with each run) and checks on the container. The run ends when the container reports its exit + code; the container waits (up to two minutes) until that report has been fetched, then + exits. The object keeps the last 30 runs (exit code, error, timing, log), and only one run + happens at a time. +- Nothing depends on the Durable Object staying in memory: Cloudflare can evict it mid-run (it + did, before the alarm took over), and the next alarm carries on from what is stored. +- A container that stops without reporting (a crash, running out of memory) fails the run + with "The container stopped before it reported how the run went"; its log shows how far it + got. A run past three hours is stopped. +- The updater's admin panel shows the runs (timeline and Controls) and can start one, through + its `COMPRESSOR` binding to this Worker's Durable Object. + +The compressor doesn't delete the source snapshots; `npm run data:archive:prune` does that, +by hand. + +## Setup (once per environment) + +1. Docker or a Docker-compatible CLI (OrbStack) must be running: `wrangler deploy` builds the + image from the repository root and pushes it to Cloudflare's registry. +2. Secrets (`npx wrangler secret put NAME --config workers/compressor/wrangler.jsonc --env dev`): + `RUN_TOKEN`, `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (a Spaces key for the archive + bucket), optional `SENTRY_DSN`. +3. The bucket is a var: dev uses `splatoon3ink-archive-test` and must never point at the + production `splatoon3ink-archive`. +4. Production: deploy only once spectre stops, which also compresses at 00:30 UTC. + +## Commands + +```sh +npm run compressor:deploy:dry-run # build the image and bundle the dev Worker +npm run compressor:deploy # deploy dev +npm run compressor:deploy:production # deploy production +npm run compressor:tail # stream dev Worker logs (not the container's output) +``` + +With `RUN_TOKEN` in `workers/compressor/.dev.vars`: + +```sh +AUTH="Authorization: Bearer $RUN_TOKEN" +BASE=https://splatoon3-ink-dev-compressor.ike.workers.dev +curl -H "$AUTH" "$BASE/status" # the run in progress and recent runs +curl -H "$AUTH" "$BASE/run?id=RUN_ID" # one run with its log lines +curl -X POST -H "$AUTH" "$BASE/run" # compress now (202, or 409 if running) +curl -X POST -H "$AUTH" "$BASE/run?dryRun=true" # list what would be compressed +curl -X POST -H "$AUTH" "$BASE/run?maxDays=30" # catch up on more days +``` + +To try the image locally (OrbStack), against dev's bucket only: + +```sh +docker build --platform linux/amd64 -f workers/compressor/Dockerfile -t splatoon3ink-compressor . +docker run --rm --platform linux/amd64 -e AWS_S3_ENDPOINT -e AWS_REGION -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY -e AWS_S3_ARCHIVE_BUCKET=splatoon3ink-archive-test -e DRY_RUN=true \ + splatoon3ink-compressor +``` diff --git a/workers/compressor/container/compress.mjs b/workers/compressor/container/compress.mjs new file mode 100644 index 0000000..a47a6c8 --- /dev/null +++ b/workers/compressor/container/compress.mjs @@ -0,0 +1,28 @@ +// Container entry point: compress finished days in the archive Space into daily tar.zst files, +// then exit. The Compressor Durable Object starts it and reads the exit code. +// +// Environment: AWS_S3_ENDPOINT, AWS_REGION, AWS_S3_ARCHIVE_BUCKET, AWS_ACCESS_KEY_ID and +// AWS_SECRET_ACCESS_KEY (see ArchiveCompressor), MAX_DAYS, DRY_RUN, optional SENTRY_DSN, and +// LOG_PORT (where the Durable Object fetches what this prints). + +import * as Sentry from '@sentry/node'; +import { sentryInit } from '../../../app/common/sentry.mjs'; +import { compressArchives } from '../../../app/data/ArchiveCompressor.mjs'; +import { serveLogs } from './logServer.mjs'; + +let logs = serveLogs(process.env.LOG_PORT); + +sentryInit(); + +let maxDays = Number(process.env.MAX_DAYS || 10); +let dryRun = process.env.DRY_RUN === 'true'; + +try { + // Errors are logged and sent to Sentry by the compressor itself. + await compressArchives(maxDays, dryRun); +} catch { + process.exitCode = 1; +} finally { + await Sentry.flush(2000).catch(() => {}); + await logs.close(process.exitCode ?? 0); +} diff --git a/workers/compressor/container/logServer.mjs b/workers/compressor/container/logServer.mjs new file mode 100644 index 0000000..70a3307 --- /dev/null +++ b/workers/compressor/container/logServer.mjs @@ -0,0 +1,98 @@ +// Keeps what the compressor prints and serves it to the Compressor Durable Object, which fetches +// it every few seconds (GET /log?after=N) and keeps it with the run for the admin panel. Once +// the work is done the answer also carries the exit code, which is how the object learns how +// the run went. Without LOG_PORT (a local `docker run`) lines only go to stdout. + +import http from 'node:http'; +import { format } from 'node:util'; + +const LEVELS = { debug: 'debug', log: 'info', info: 'info', warn: 'warn', error: 'error' }; +const MAX_LINE_LENGTH = 600; +// How long to wait at the end for the object to fetch the report and the last lines. It checks +// every few seconds, so this only runs out if it has stopped checking. +const COLLECT_TIMEOUT_MS = 2 * 60 * 1000; + +function describe(value) { + if (typeof value === 'string') { + return value; + } + + if (value instanceof Error) { + return value.cause ? `${value.message} (cause: ${describe(value.cause)})` : value.message; + } + + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} + +/** + * Keep console output and serve it on `port`. + * @param {string|undefined} port + * @returns {{ close: (code: number) => Promise }} close() reports the exit code, waits + * until it and every line have been fetched (or a timeout), then stops serving; await it + * before exiting + */ +export function serveLogs(port) { + if (!port) { + return { close: async () => {} }; + } + + let lines = []; + let done = false; + let exitCode = null; + let collected = () => {}; + + for (let [method, level] of Object.entries(LEVELS)) { + let original = console[method].bind(console); + + console[method] = (...args) => { + // printf-style placeholders are filled in the way console does it. + let text = typeof args[0] === 'string' && /%[sdifjoOc]/.test(args[0]) + ? format(...args) + : args.map(describe).join(' '); + + lines.push({ at: Date.now(), level, text: text.length > MAX_LINE_LENGTH ? `${text.slice(0, MAX_LINE_LENGTH)}…` : text }); + original(...args); + }; + } + + let server = http.createServer((request, response) => { + let url = new URL(request.url, 'http://container'); + + if (request.method !== 'GET' || url.pathname !== '/log') { + response.writeHead(404).end(); + + return; + } + + let after = Math.max(0, Number(url.searchParams.get('after')) || 0); + let finished = done; + + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify({ lines: lines.slice(after), next: lines.length, done: finished, exitCode })); + + if (finished) { + collected(); + } + }); + + server.listen(Number(port)); + + return { + async close(code) { + exitCode = code; + done = true; + + await Promise.race([ + new Promise(resolve => { collected = resolve; }), + new Promise(resolve => setTimeout(resolve, COLLECT_TIMEOUT_MS).unref()), + ]); + + // Lets the last response finish sending. + await new Promise(resolve => server.close(resolve)); + }, + }; +} diff --git a/workers/compressor/src/Compressor.mjs b/workers/compressor/src/Compressor.mjs new file mode 100644 index 0000000..b22f839 --- /dev/null +++ b/workers/compressor/src/Compressor.mjs @@ -0,0 +1,230 @@ +// One Durable Object that runs the archive compressor's container to completion and keeps a +// short history of runs. The container does the work (container/compress.mjs) and serves what +// it prints, and at the end how it went, on LOG_PORT (container/logServer.mjs). +// +// While a run goes on, an alarm every few seconds fetches the new lines and stores them with +// the run. Nothing depends on this object staying in memory: it can be evicted mid-run, and the +// next alarm carries on from what is stored. The run ends when the container reports its exit +// code; the container waits until that report has been fetched before it exits. + +import { DurableObject } from 'cloudflare:workers'; +import { nextScheduledRun } from './schedule.mjs'; + +const HISTORY = 30; +// How often the alarm fetches log lines and checks on the container while a run goes on. +const CHECK_INTERVAL_MS = 3000; +// The container keeps running this long without hearing from this object. +const INACTIVITY_TIMEOUT_MS = 15 * 60 * 1000; +// A run that goes on this long is stuck (a day takes a few minutes); stop it. +const MAX_RUNTIME_MS = 3 * 60 * 60 * 1000; +// A container that is not running this long after its run started has stopped. +const START_GRACE_MS = 60 * 1000; +const LOG_PORT = 8080; +const MAX_LOG_LINES = 500; +const STOPPED = 'The container stopped before it reported how the run went'; + +export class Compressor extends DurableObject { + // While this object stays in memory, monitor() reports a crashed container's exit code. + #watching = null; + + /** + * @returns {Promise<{ now: number, nextAt: number, current: object|null, runs: object[] }>} The + * run in progress, recent runs (newest first), and when the next scheduled run starts + */ + async status() { + let now = Date.now(); + + return { + now, + nextAt: nextScheduledRun(now), + current: (await this.ctx.storage.get('current')) ?? null, + runs: (await this.ctx.storage.get('runs')) ?? [], + }; + } + + /** + * One run with what it printed, or null if it is no longer kept. + * @param {string} id + */ + async run(id) { + let current = await this.ctx.storage.get('current'); + let run = current?.id === id + ? current + : ((await this.ctx.storage.get('runs')) ?? []).find(run => run.id === id); + + if (!run) { + return null; + } + + let { lines, omitted } = (await this.ctx.storage.get(`log:${id}`)) ?? { lines: [], omitted: 0 }; + + return { ...run, running: run === current, logs: { lines, omitted } }; + } + + /** + * Start a run unless one is going. + * @param {{ trigger: 'scheduled'|'manual', maxDays?: number, dryRun?: boolean }} options + */ + async start({ trigger, maxDays = 10, dryRun = false }) { + if (!Number.isInteger(maxDays) || maxDays < 1) { + return { ok: false, error: 'maxDays must be a whole number of at least 1' }; + } + + let current = await this.ctx.storage.get('current'); + if (current) { + return { ok: false, busy: true, current }; + } + + // The last run has reported and finished, but its container can take a moment to exit. + if (this.ctx.container.running) { + return { ok: false, busy: true, error: 'The last run\'s container is still stopping. Try again in a minute.' }; + } + + current = { id: crypto.randomUUID(), trigger, maxDays, dryRun, startedAt: Date.now() }; + await this.ctx.storage.put('current', current); + + try { + this.ctx.container.start({ enableInternet: true, env: this.#containerEnv(current) }); + await this.ctx.container.setInactivityTimeout(INACTIVITY_TIMEOUT_MS); + await this.ctx.storage.setAlarm(Date.now() + CHECK_INTERVAL_MS); + } catch (e) { + let run = await this.#finish({ ok: false, error: `Could not start the container: ${e?.message ?? e}` }); + + return { ok: false, run }; + } + + this.#watch(current.id); + + return { ok: true, current }; + } + + async alarm() { + let current = await this.ctx.storage.get('current'); + if (!current) { + return; + } + + // The container is not listening for the first moment, so a failed fetch is expected. + let report = await this.#fetchLogs(current.id).catch(() => null); + + if (report?.done) { + let exitCode = Number.isInteger(report.exitCode) ? report.exitCode : null; + + await this.#finish(exitCode === 0 ? { ok: true, exitCode } : { ok: false, exitCode, error: `The compressor exited with code ${exitCode}` }); + + return; + } + + let elapsed = Date.now() - current.startedAt; + + if (!this.ctx.container.running && elapsed > START_GRACE_MS) { + // It crashed, or was stopped, before reporting. Its log shows how far it got. + await this.#finish({ ok: false, lost: true, exitCode: null, error: STOPPED }); + + return; + } + + if (elapsed > MAX_RUNTIME_MS) { + await this.ctx.container.destroy('Compression ran too long'); + await this.#finish({ ok: false, exitCode: null, error: 'Stopped after running for three hours' }); + + return; + } + + await this.ctx.container.setInactivityTimeout(INACTIVITY_TIMEOUT_MS); + await this.ctx.storage.setAlarm(Date.now() + CHECK_INTERVAL_MS); + this.#watch(current.id); + } + + // Store the container's new log lines. Returns its report: done, and the exit code once done. + async #fetchLogs(id) { + let log = (await this.ctx.storage.get(`log:${id}`)) ?? { lines: [], omitted: 0, next: 0 }; + let response = await this.ctx.container.getTcpPort(LOG_PORT).fetch(`http://container/log?after=${log.next ?? 0}`); + + if (!response.ok) { + return null; + } + + let { lines, next, done, exitCode } = await response.json(); + + if (lines.length) { + await this.#appendLog(id, lines, next); + } + + return { done, exitCode }; + } + + async #appendLog(id, lines, next) { + let log = (await this.ctx.storage.get(`log:${id}`)) ?? { lines: [], omitted: 0, next: 0 }; + + log.next = next ?? log.next; + + for (let { at, level, text } of lines) { + if (log.lines.length >= MAX_LOG_LINES) { + log.omitted++; + } else { + log.lines.push({ at: Number(at) || Date.now(), level: String(level ?? 'info'), text: String(text ?? '') }); + } + } + + await this.ctx.storage.put(`log:${id}`, log); + } + + #containerEnv({ maxDays, dryRun }) { + let env = { + AWS_S3_ENDPOINT: this.env.AWS_S3_ENDPOINT, + AWS_REGION: this.env.AWS_REGION, + AWS_S3_ARCHIVE_BUCKET: this.env.AWS_S3_ARCHIVE_BUCKET, + AWS_ACCESS_KEY_ID: this.env.AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY: this.env.AWS_SECRET_ACCESS_KEY, + SENTRY_DSN: this.env.SENTRY_DSN, + LOG_PORT: String(LOG_PORT), + MAX_DAYS: String(maxDays), + DRY_RUN: String(dryRun), + }; + + // Unset values are left out rather than passed as "undefined". + return Object.fromEntries(Object.entries(env).filter(([, value]) => value)); + } + + // A container that dies (out of memory, say) never reports; monitor() gives its exit code if + // this object is still in memory. A clean exit comes after the report, which already ended + // the run, so only failures count here. + #watch(id) { + if (this.#watching) { + return; + } + + this.#watching = this.ctx.container.monitor() + .then( + () => null, + async error => { + if ((await this.ctx.storage.get('current'))?.id === id) { + await this.#finish({ ok: false, exitCode: error?.exitCode ?? null, error: String(error?.message ?? error) }); + } + }, + ) + .finally(() => { + this.#watching = null; + }); + } + + async #finish(result) { + let current = await this.ctx.storage.get('current'); + if (!current) { + return; + } + + let run = { ...current, ...result, finishedAt: Date.now() }; + let kept = [run, ...((await this.ctx.storage.get('runs')) ?? [])]; + + await this.ctx.storage.put('runs', kept.slice(0, HISTORY)); + await this.ctx.storage.delete(kept.slice(HISTORY).map(old => `log:${old.id}`)); + await this.ctx.storage.delete('current'); + await this.ctx.storage.deleteAlarm(); + + console[run.ok ? 'info' : 'error'](JSON.stringify({ event: 'compress-run', ...run })); + + return run; + } +} diff --git a/workers/compressor/src/index.mjs b/workers/compressor/src/index.mjs new file mode 100644 index 0000000..883473b --- /dev/null +++ b/workers/compressor/src/index.mjs @@ -0,0 +1,74 @@ +// splatoon3.ink archive compressor Worker: once a day, runs app/data/ArchiveCompressor.mjs in a +// Cloudflare Container against the archive Space. See workers/compressor/README.md. + +import { Compressor } from './Compressor.mjs'; + +export { Compressor }; + +function compressor(env) { + return env.COMPRESSOR.getByName('archive'); +} + +function timingSafeEqual(a, b) { + let encoder = new TextEncoder(); + let left = encoder.encode(a); + let right = encoder.encode(b); + + return left.byteLength === right.byteLength && crypto.subtle.timingSafeEqual(left, right); +} + +function authorized(request, env) { + if (!env.RUN_TOKEN) { + return false; + } + + let [scheme, token] = (request.headers.get('Authorization') ?? '').split(' '); + + return scheme === 'Bearer' && !!token && timingSafeEqual(token, env.RUN_TOKEN); +} + +function json(body, status = 200) { + return Response.json(body, { status, headers: { 'Cache-Control': 'no-store' } }); +} + +export default { + async fetch(request, env) { + if (!authorized(request, env)) { + return json({ ok: false, error: 'Unauthorized' }, 401); + } + + let url = new URL(request.url); + let route = `${request.method} ${url.pathname}`; + + if (route === 'GET /status') { + return json(await compressor(env).status()); + } + + if (route === 'GET /run') { + let run = await compressor(env).run(url.searchParams.get('id') ?? ''); + + return run ? json(run) : json({ ok: false, error: 'Not found' }, 404); + } + + if (route === 'POST /run') { + let maxDays = url.searchParams.get('maxDays'); + let result = await compressor(env).start({ + trigger: 'manual', + maxDays: maxDays === null ? undefined : Number(maxDays), + dryRun: url.searchParams.get('dryRun') === 'true', + }); + + return json(result, result.ok ? 202 : result.busy ? 409 : 400); + } + + return json({ ok: false, error: 'Not found' }, 404); + }, + + async scheduled(controller, env) { + let result = await compressor(env).start({ trigger: 'scheduled' }); + + if (!result.ok) { + console.warn(JSON.stringify({ event: 'compress-skipped', ...result })); + } + }, +}; diff --git a/workers/compressor/src/schedule.mjs b/workers/compressor/src/schedule.mjs new file mode 100644 index 0000000..b095d51 --- /dev/null +++ b/workers/compressor/src/schedule.mjs @@ -0,0 +1,18 @@ +// When the cron trigger in wrangler.jsonc ("30 0 * * *") starts a run. Keep the two in step. +const HOUR_UTC = 0; +const MINUTE_UTC = 30; + +/** + * @param {number} now Epoch milliseconds + * @returns {number} The next scheduled run after now, in epoch milliseconds + */ +export function nextScheduledRun(now) { + let next = new Date(now); + next.setUTCHours(HOUR_UTC, MINUTE_UTC, 0, 0); + + if (next.getTime() <= now) { + next.setUTCDate(next.getUTCDate() + 1); + } + + return next.getTime(); +} diff --git a/workers/compressor/src/schedule.test.mjs b/workers/compressor/src/schedule.test.mjs new file mode 100644 index 0000000..8f958fc --- /dev/null +++ b/workers/compressor/src/schedule.test.mjs @@ -0,0 +1,12 @@ +import { describe, expect, it } from 'vitest'; +import { nextScheduledRun } from './schedule.mjs'; + +const next = iso => new Date(nextScheduledRun(Date.parse(iso))).toISOString(); + +describe('nextScheduledRun', () => { + it('is 00:30 UTC today until then, and tomorrow from then on', () => { + expect(next('2026-11-12T00:29:59Z')).toBe('2026-11-12T00:30:00.000Z'); + expect(next('2026-11-12T00:30:00Z')).toBe('2026-11-13T00:30:00.000Z'); + expect(next('2026-12-31T23:00:00Z')).toBe('2027-01-01T00:30:00.000Z'); + }); +}); diff --git a/workers/compressor/wrangler.jsonc b/workers/compressor/wrangler.jsonc new file mode 100644 index 0000000..44810ef --- /dev/null +++ b/workers/compressor/wrangler.jsonc @@ -0,0 +1,59 @@ +{ + "$schema": "../../node_modules/wrangler/config-schema.json", + "name": "splatoon3-ink-compressor", + "main": "src/index.mjs", + "compatibility_date": "2026-09-03", + "compatibility_flags": ["nodejs_compat"], + "workers_dev": true, + "preview_urls": false, + "migrations": [{ "tag": "v1", "new_sqlite_classes": ["Compressor"] }], + "observability": { "enabled": true, "logs": { "head_sampling_rate": 1 } }, + + // Select --env dev or --env production explicitly. Deploying builds the container image with + // a local Docker-compatible CLI (Dockerfile, built from the repository root). + // Secrets: RUN_TOKEN, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY (a DigitalOcean Spaces key + // that can read and write the archive bucket), optional SENTRY_DSN. + "env": { + "dev": { + "name": "splatoon3-ink-dev-compressor", + "vars": { + // Dev's own archive Space, the one the dev updater writes. Never the production one. + "AWS_S3_ENDPOINT": "https://nyc3.digitaloceanspaces.com", + "AWS_REGION": "nyc3", + "AWS_S3_ARCHIVE_BUCKET": "splatoon3ink-archive-test" + }, + "triggers": { "crons": ["30 0 * * *"] }, + "containers": [ + { + "class_name": "Compressor", + "image": "./Dockerfile", + "image_build_context": "../..", + // 1/2 vCPU, 4 GiB, 8 GB disk: a day (~430 MB of snapshots) compresses in under a minute. + "instance_type": "standard-1", + "max_instances": 1 + } + ], + "durable_objects": { "bindings": [{ "name": "COMPRESSOR", "class_name": "Compressor" }] } + }, + "production": { + "name": "splatoon3-ink-compressor", + "vars": { + "AWS_S3_ENDPOINT": "https://nyc3.digitaloceanspaces.com", + "AWS_REGION": "nyc3", + "AWS_S3_ARCHIVE_BUCKET": "splatoon3ink-archive" + }, + // Spectre also compresses at 00:30 UTC until cutover; deploy this only once it stops. + "triggers": { "crons": ["30 0 * * *"] }, + "containers": [ + { + "class_name": "Compressor", + "image": "./Dockerfile", + "image_build_context": "../..", + "instance_type": "standard-1", + "max_instances": 1 + } + ], + "durable_objects": { "bindings": [{ "name": "COMPRESSOR", "class_name": "Compressor" }] } + } + } +}