diff --git a/.gitignore b/.gitignore index 04380e8..b4a477c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ logs test-screenshot-* /storage .wrangler +.dev.vars diff --git a/package-lock.json b/package-lock.json index bddbfa9..09a121e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "name": "splatoon2.ink", "dependencies": { "@atproto/api": "^0.20.42", + "@sentry/cloudflare": "^10.73.0", "@sentry/core": "^10.73.0", "@sentry/node": "^10.73.0", "bulma": "^0.9.4", @@ -1851,7 +1852,7 @@ "version": "0.5.0", "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", - "dev": true, + "devOptional": true, "license": "MIT OR Apache-2.0", "engines": { "node": ">=22.0.0" @@ -1861,7 +1862,7 @@ "version": "2.16.1", "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", - "dev": true, + "devOptional": true, "license": "MIT OR Apache-2.0", "peerDependencies": { "unenv": "2.0.0-rc.24", @@ -1998,7 +1999,6 @@ "version": "5.20260903.1", "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-5.20260903.1.tgz", "integrity": "sha512-Dgm28XJqMYj3VCNK14Xwo9UPtSPWL8Izo0emiyeQLZCRXWuhc3GnlKENF4Pf3ot+3NaWosq+yq8OIa531dCr6g==", - "dev": true, "license": "MIT OR Apache-2.0", "optional": true, "peer": true @@ -2007,7 +2007,7 @@ "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "0.3.9" @@ -2020,7 +2020,7 @@ "version": "0.3.9", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", @@ -3265,7 +3265,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=6.0.0" @@ -3286,7 +3286,6 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", - "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { @@ -3842,7 +3841,7 @@ "version": "4.1.6", "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "kleur": "^4.1.5" @@ -3852,7 +3851,7 @@ "version": "0.6.5", "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@poppinss/colors": "^4.1.5", @@ -3864,7 +3863,7 @@ "version": "10.2.2", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=18" @@ -3877,7 +3876,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/@puppeteer/browsers": { @@ -4322,6 +4321,33 @@ "dev": true, "license": "MIT" }, + "node_modules/@sentry/cloudflare": { + "version": "10.73.0", + "resolved": "https://registry.npmjs.org/@sentry/cloudflare/-/cloudflare-10.73.0.tgz", + "integrity": "sha512-6BmvIbstT8GLdA7xDrjd3oR98tdXh7uwDbnm/Hu3HjO3py0HEQlTypeSZIDVdRpR3veYZ9eT0hJ/74Gp+QZZtQ==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@sentry/core": "10.73.0", + "@sentry/server-utils": "10.73.0", + "magic-string": "~0.30.21" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^4.x || ^5.x", + "wrangler": "^4.x" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + }, + "wrangler": { + "optional": true + } + } + }, "node_modules/@sentry/conventions": { "version": "0.16.0", "resolved": "https://registry.npmjs.org/@sentry/conventions/-/conventions-0.16.0.tgz", @@ -4461,7 +4487,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=18" @@ -4500,7 +4526,7 @@ "version": "1.2.24", "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.24.tgz", "integrity": "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==", - "dev": true, + "devOptional": true, "license": "CC0-1.0" }, "node_modules/@standard-schema/spec": { @@ -6200,7 +6226,7 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/bluebird": { @@ -6947,7 +6973,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=18" @@ -7970,7 +7996,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", - "dev": true, + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/antfu" @@ -8016,7 +8042,7 @@ "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", - "dev": true, + "devOptional": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -10216,7 +10242,7 @@ "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=6" @@ -10892,7 +10918,6 @@ "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", - "dev": true, "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" @@ -11128,7 +11153,7 @@ "version": "5.20260831.0-alpha", "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20260831.0-alpha.tgz", "integrity": "sha512-Hwgh1VDUiPCPGQKODQfUmy7hRAje1D55icB+9png3ueiM64rlSM87nSrtqpxAD+DlLWI4ehnYBuECaXV43zGmQ==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", @@ -11663,7 +11688,7 @@ "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, + "devOptional": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -11676,7 +11701,7 @@ "version": "0.35.2", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.2.tgz", "integrity": "sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@img/colour": "^1.1.0", @@ -11721,7 +11746,7 @@ "version": "8.21.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -12697,7 +12722,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/picocolors": { @@ -15553,7 +15578,7 @@ "version": "7.29.0", "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=20.18.1" @@ -15570,7 +15595,7 @@ "version": "2.0.0-rc.24", "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "pathe": "^2.0.3" @@ -16807,7 +16832,7 @@ "version": "1.20260831.1", "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260831.1.tgz", "integrity": "sha512-A2LwrkBel/FnKABPfeBAMiL6v70+rugnunqQRfWsWZjlhsTZoBScWUVunMy/xLCGLjWCQL2zp39AVR6aO0jurQ==", - "dev": true, + "devOptional": true, "hasInstallScript": true, "license": "Apache-2.0", "bin": { @@ -16828,7 +16853,7 @@ "version": "4.128.0", "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.128.0.tgz", "integrity": "sha512-jNXy9e8/pbx8iqTzXPiuflnitKJZoAfEUSUUDLW87bwyeMvJ7kb3yQMSbxEcfNdfHqJW38KRcKaLljOYV4N/4w==", - "dev": true, + "devOptional": true, "license": "MIT OR Apache-2.0", "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", @@ -16864,7 +16889,7 @@ "version": "6.3.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/wrap-ansi": { @@ -16995,7 +17020,7 @@ "version": "4.1.0-beta.10", "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@poppinss/colors": "^4.1.5", @@ -17009,7 +17034,7 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "@poppinss/exception": "^1.2.2", diff --git a/package.json b/package.json index b50efcb..672a580 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "scripts": { "serve": "vue-cli-service serve", "build": "vue-cli-service build --modern --no-clean", - "test": "node --test && npm run worker:test", + "test": "node --test && npm run worker:test && npm run updater:test", "lint": "eslint .", "cron": "node src/app/cron.js", "start": "npm run splatnet && npm run twitter && npm run cron", @@ -23,10 +23,16 @@ "worker:test": "vitest run --config vitest.worker.config.mjs", "worker:deploy:dry-run": "wrangler deploy --dry-run", "worker:deploy": "wrangler deploy", - "lint-fix": "npm run lint -- --fix" + "lint-fix": "npm run lint -- --fix", + "updater:dev": "wrangler dev --config workers/updater/wrangler.jsonc --test-scheduled", + "updater:test": "vitest run --config workers/updater/vitest.config.mjs", + "updater:deploy:dry-run": "wrangler deploy --config workers/updater/wrangler.jsonc --dry-run", + "updater:deploy": "wrangler deploy --config workers/updater/wrangler.jsonc", + "updater:tail": "wrangler tail --config workers/updater/wrangler.jsonc" }, "dependencies": { "@atproto/api": "^0.20.42", + "@sentry/cloudflare": "^10.73.0", "@sentry/core": "^10.73.0", "@sentry/node": "^10.73.0", "bulma": "^0.9.4", diff --git a/src/app/updater/updateAll.js b/src/app/updater/updateAll.js index a485fbf..2d4bf37 100644 --- a/src/app/updater/updateAll.js +++ b/src/app/updater/updateAll.js @@ -20,16 +20,31 @@ export function createUpdaters(storage) { } /** - * Run every updater. `storage` is shared by all of them for the run (see createUpdaters); - * a Worker passes BucketStorage over its R2 bindings, a local run passes FilesystemStorage. + * Run every updater (or the named ones). `storage` is shared by all of them for the run; a + * Worker passes BucketStorage over its R2 bindings, a local run passes FilesystemStorage. + * A failing updater is logged and does not stop the others. + * + * @param {object} storage + * @param {{ only?: string[] }} [options] restrict the run to updaters with these names + * @returns {Promise>} */ -export default async function updateAll(storage) { +export default async function updateAll(storage, { only } = {}) { + let results = []; + for (let updater of createUpdaters(storage)) { + let name = updater.options.name; + if (only && !only.includes(name)) + continue; + + let started = Date.now(); try { await updater.update(); + results.push({ name, ok: true, ms: Date.now() - started }); } catch (e) { console.error(e); + results.push({ name, ok: false, ms: Date.now() - started, error: e instanceof Error ? e.message : String(e) }); } } + return results; } diff --git a/src/common/splatnet.js b/src/common/splatnet.js index 071f759..02fcd1a 100644 --- a/src/common/splatnet.js +++ b/src/common/splatnet.js @@ -1,7 +1,9 @@ // SplatNet2 API -const userAgent = process.env.SPLATNET_USER_AGENT; const splatnetBaseUrl = 'https://app.splatoon2.nintendo.net'; +// Read lazily: in a Worker, process.env is populated from the bindings rather than at startup. +const userAgent = () => process.env.SPLATNET_USER_AGENT; + export default class SplatNet { constructor(region = 'NA', language = 'en-US') { this.region = region; @@ -18,7 +20,7 @@ export default class SplatNet { getHeaders() { return { - ...(userAgent ? { 'User-Agent': userAgent } : {}), + ...(userAgent() ? { 'User-Agent': userAgent() } : {}), 'Cookie': `iksm_session=${this.getSessionId()}`, 'Accept-Language': this.language, }; @@ -110,7 +112,7 @@ export default class SplatNet { /** Download an image (no session cookie needed). Returns the bytes. */ async getImage(imagePath) { let response = await this.request(imagePath, { - headers: userAgent ? { 'User-Agent': userAgent } : {}, + headers: userAgent() ? { 'User-Agent': userAgent() } : {}, }); return new Uint8Array(await response.arrayBuffer()); } diff --git a/workers/updater/README.md b/workers/updater/README.md new file mode 100644 index 0000000..cd2d58d --- /dev/null +++ b/workers/updater/README.md @@ -0,0 +1,78 @@ +# Splatoon 2 updater Worker + +Runs the site's updaters (`src/app/updater`, the same code `npm run splatnet` runs +locally) on Cloudflare, writing to R2 through `BucketStorage` instead of to `dist/` +and `storage/` through `FilesystemStorage`. + +## Scheduling: cron wakes, the object works + +Cron Triggers fire anywhere inside their minute (observed about 55 seconds late) +and execute in whichever colo Cloudflare chooses; placement hints only apply to +fetch handlers. So Cron Triggers never do work here. They call the `Scheduler` +Durable Object, which runs jobs from its own alarm, in place, next to the R2 +buckets (the stub is created with a `wnam` location hint). + +- **Hourly job** (`updaters`): the full updater run from an alarm at :00:10, + re-armed for the next hour after each run. A run with a failed updater is + retried after a minute, up to three times, before falling back to the next + hour. Alarms have measured about 1 ms of drift. +- **Woken jobs**: `wake(job)` asks the object to run a job as soon as possible. + The scheduled handler maps each cron expression in `CRON_ACTIONS` to a call on + the object; this is how a cron-driven job is expressed. +- **Watchdog**: the only cron trigger, at minute 30, calls `ensureArmed()` so a + lost hourly alarm heals within the hour. The alarm handler also restores the + hourly schedule itself if it finds it missing. + +Every run logs a structured summary (`driftMs`, `runMs`, per-updater results, +the colo) under `updater: "alarm"`, visible in the Worker's Observability tab. +`GET /status` returns the last run of each job. + +## Errors + +The shared updater code reports through `@sentry/core`; this Worker wraps its +handlers and the Durable Object with `@sentry/cloudflare`, so those reports go +to Sentry when the `SENTRY_DSN` secret is set and nowhere otherwise. + +## Shadow mode + +`wrangler.jsonc` points the `ASSETS` binding at `splatoon2-ink-dev-assets` while +the container still owns `splatoon2-ink-assets`. Cutover is changing that +`bucket_name`. Compare the two buckets' `data/` with `scripts/compare-data.mjs` +after downloading, e.g. with `wrangler r2 object get --remote`. + +## Secrets + +```sh +npx wrangler secret put NINTENDO_SESSION_ID_NA --config workers/updater/wrangler.jsonc +npx wrangler secret put NINTENDO_SESSION_ID_EU --config workers/updater/wrangler.jsonc +npx wrangler secret put NINTENDO_SESSION_ID_JP --config workers/updater/wrangler.jsonc +npx wrangler secret put SPLATNET_USER_AGENT --config workers/updater/wrangler.jsonc +npx wrangler secret put RUN_TOKEN --config workers/updater/wrangler.jsonc +npx wrangler secret put SENTRY_DSN --config workers/updater/wrangler.jsonc # optional +``` + +The updaters read the SplatNet secrets through `process.env`, which Workers +populate from the bindings. For local development put the same names in +`workers/updater/.dev.vars` (gitignored). + +## Running + +```sh +npm run updater:test # vitest in workerd: the updaters against local R2, the Scheduler +npm run updater:dev # local dev; GET http://localhost:8787/cdn-cgi/handler/scheduled fires the cron +npm run updater:deploy:dry-run +npm run updater:deploy +npm run updater:tail # live logs +``` + +Authenticated operator endpoints, all requiring `Authorization: Bearer $UPDATER_RUN_TOKEN` +(the token is in `.env` locally); without `RUN_TOKEN` set they are off: + +```sh +BASE=https://splatoon2-ink-updater..workers.dev +curl -X POST -H "Authorization: Bearer $UPDATER_RUN_TOKEN" "$BASE/run" # run every updater now +curl -X POST -H "Authorization: Bearer $UPDATER_RUN_TOKEN" "$BASE/run?only=Schedules,Timeline" # or some of them +curl -X POST -H "Authorization: Bearer $UPDATER_RUN_TOKEN" "$BASE/wake?job=updaters" # ask the Scheduler to run now +curl -X POST -H "Authorization: Bearer $UPDATER_RUN_TOKEN" "$BASE/arm" # schedule the hourly job if it is not scheduled +curl -H "Authorization: Bearer $UPDATER_RUN_TOKEN" "$BASE/status" # alarm time, hourly/retry state, last run per job +``` diff --git a/workers/updater/Scheduler.spec.mjs b/workers/updater/Scheduler.spec.mjs new file mode 100644 index 0000000..37463f8 --- /dev/null +++ b/workers/updater/Scheduler.spec.mjs @@ -0,0 +1,162 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createExecutionContext, createScheduledController, env, runDurableObjectAlarm } from 'cloudflare:test'; +import worker from './src/index.mjs'; +import { nextRunAt, GRACE_MS, HOUR_MS } from './src/schedule.mjs'; + +function stub() { + return env.SCHEDULER.get(env.SCHEDULER.idFromName(`test-${crypto.randomUUID()}`)); +} + +// In the test runtime a due alarm may fire on its own (the fake Date is visible to the +// runtime too), so trigger it explicitly and then wait for the outcome either way. +async function runAlarmUntil(scheduler, done) { + await runDurableObjectAlarm(scheduler); + await vi.waitFor(async () => expect(done(await scheduler.status())).toBe(true), { timeout: 5000 }); + return scheduler.status(); +} + +import { fakeSplatNet, setSessionEnvironment } from './fakeSplatNet.mjs'; + +setSessionEnvironment(); + +// The Durable Object runs in the test isolate, so stubbing global fetch (and Date) reaches it. +function splatnetDown() { + vi.stubGlobal('fetch', async () => new Response('down', { status: 503 })); +} + +function splatnetUp() { + vi.stubGlobal('fetch', fakeSplatNet()); +} + +describe('Scheduler', () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + it('schedules the hourly job for the next :00:10 and is idempotent', async () => { + let scheduler = stub(); + let before = Date.now(); + let first = await scheduler.ensureArmed(); + + expect(first.armed).toBe(true); + expect(first.hourlyAt).toBeGreaterThanOrEqual(nextRunAt(before)); + expect((first.hourlyAt - GRACE_MS) % HOUR_MS).toBe(0); + expect(first.alarmAt).toBe(first.hourlyAt); + + let second = await scheduler.ensureArmed(); + expect(second).toEqual({ armed: false, hourlyAt: first.hourlyAt, alarmAt: first.hourlyAt }); + + let status = await scheduler.status(); + expect(status.alarmAt).toBe(first.hourlyAt); + expect(status.pending).toEqual([]); + expect(status.lastRuns).toEqual({}); + }); + + it('runs a woken job immediately and then goes back to the hourly schedule', async () => { + splatnetUp(); + let scheduler = stub(); + let { hourlyAt } = await scheduler.ensureArmed(); + + let woke = await scheduler.wake('updaters'); + expect(woke.accepted).toBe(true); + expect(woke.alarmAt).toBeLessThan(hourlyAt); + expect(woke.alarmAt - Date.now()).toBeLessThan(1000); + await scheduler.wake('updaters'); // duplicate collapses into the same run + + let status = await runAlarmUntil(scheduler, s => s.lastRuns.updaters !== undefined); + expect(status.pending).toEqual([]); + expect(status.lastRuns.updaters.reason).toBe('wake'); + expect(status.lastRuns.updaters.ok).toBe(true); + expect(status.lastRuns.updaters.driftMs).toBeNull(); + expect(status.lastRuns.updaters.result.updaters).toHaveLength(8); + expect(status.hourlyAt).toBe(hourlyAt); // a wake run does not move the hourly schedule + expect(status.alarmAt).toBe(hourlyAt); + }); + + it('restores the hourly schedule if a wake runs before the object was ever armed', async () => { + splatnetUp(); + let scheduler = stub(); + await scheduler.wake('updaters'); + + let status = await runAlarmUntil(scheduler, s => s.lastRuns.updaters !== undefined); + expect(status.lastRuns.updaters.ok).toBe(true); + expect(status.hourlyAt).toBe(nextRunAt(status.lastRuns.updaters.firedAt)); + expect(status.alarmAt).toBe(status.hourlyAt); + }); + + it('rejects unknown jobs without scheduling anything', async () => { + let scheduler = stub(); + expect(await scheduler.wake('nope')).toEqual({ job: 'nope', accepted: false, error: 'Unknown job: nope' }); + expect((await scheduler.status()).alarmAt).toBeNull(); + }); + + it('retries a failed hourly run after a minute, then gives up until the next hour', async () => { + splatnetDown(); + let scheduler = stub(); + let { hourlyAt } = await scheduler.ensureArmed(); + + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(hourlyAt + 5); + let status = await runAlarmUntil(scheduler, s => s.retries === 1); + expect(status.lastRuns.updaters).toMatchObject({ reason: 'hourly', ok: false, retries: 0 }); + expect(status.lastRuns.updaters.driftMs).toBeGreaterThanOrEqual(5); + expect(status.lastRuns.updaters.driftMs).toBeLessThan(1000); // the faked clock still creeps a little + expect(status.lastRuns.updaters.error).toContain('updaters failed'); + expect(status.retries).toBe(1); + expect(status.retryAt - (hourlyAt + 5 + 60 * 1000)).toBeGreaterThanOrEqual(0); + expect(status.retryAt - (hourlyAt + 5 + 60 * 1000)).toBeLessThan(2000); + expect(status.alarmAt).toBe(status.retryAt); + expect(status.hourlyAt).toBe(hourlyAt); + + for (let attempt = 2; attempt <= 3; attempt++) { + vi.setSystemTime(status.retryAt); + status = await runAlarmUntil(scheduler, s => s.retries === attempt); + expect(status.lastRuns.updaters.reason).toBe('retry'); + } + + // Fourth failure exhausts retries: back to the next hour + vi.setSystemTime(status.retryAt); + status = await runAlarmUntil(scheduler, s => s.retryAt === null); + expect(status.retries).toBe(0); + expect(status.hourlyAt).toBe(hourlyAt + HOUR_MS); + expect(status.alarmAt).toBe(hourlyAt + HOUR_MS); + }); + + it('moves to the next hour after a successful hourly run', async () => { + splatnetUp(); + let scheduler = stub(); + let { hourlyAt } = await scheduler.ensureArmed(); + + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(hourlyAt + 1); + let status = await runAlarmUntil(scheduler, s => s.hourlyAt !== hourlyAt); + expect(status.lastRuns.updaters).toMatchObject({ reason: 'hourly', ok: true, colo: 'TEST' }); + expect(status.lastRuns.updaters.driftMs).toBeGreaterThanOrEqual(1); + expect(status.lastRuns.updaters.driftMs).toBeLessThan(1000); + expect(status.hourlyAt).toBe(hourlyAt + HOUR_MS); + expect(status.alarmAt).toBe(hourlyAt + HOUR_MS); + }); +}); + +describe('cron watchdog', () => { + it('arms the scheduler and does nothing else', async () => { + let calls = []; + let fakeEnv = { + SCHEDULER: { + idFromName: name => ({ name }), + get: (id, options) => { + calls.push({ id, options }); + return { ensureArmed: async () => ({ armed: true, hourlyAt: 1, alarmAt: 1 }) }; + }, + }, + }; + await worker.scheduled(createScheduledController({ cron: '30 * * * *', scheduledTime: new Date }), fakeEnv, createExecutionContext()); + expect(calls).toEqual([{ id: { name: 'schedules' }, options: { locationHint: 'wnam' } }]); + }); + + it('ignores cron expressions it has no action for', async () => { + let fakeEnv = { SCHEDULER: { idFromName: () => ({}), get: () => { throw new Error('should not be called'); } } }; + await expect(worker.scheduled(createScheduledController({ cron: '0 0 1 1 *', scheduledTime: new Date }), fakeEnv, createExecutionContext())).resolves.toBeUndefined(); + }); +}); diff --git a/workers/updater/fakeSplatNet.mjs b/workers/updater/fakeSplatNet.mjs new file mode 100644 index 0000000..357b311 --- /dev/null +++ b/workers/updater/fakeSplatNet.mjs @@ -0,0 +1,50 @@ +// A fake SplatNet for the Worker specs: enough of every endpoint for all eight updaters. +export const SESSIONS = { NA: 'na-session', EU: 'eu-session', JP: 'jp-session' }; + +const stage = (id, name) => ({ id, name, image: `/images/stage/${id}.png` }); +const rotation = start => ({ + id: start, start_time: start, end_time: start + 7200, + stage_a: stage('0', 'The Reef'), stage_b: stage('1', 'Musselforge Fitness'), + game_mode: { key: 'regular', name: 'Regular Battle' }, + rule: { key: 'turf_war', name: 'Turf War', multiline_name: 'Turf\nWar' }, +}); +const gear = { kind: 'head', id: '1', name: 'Fake Hat', image: '/images/gear/1.png', brand: { id: '0', name: 'Brand', image: '/images/brand/0.png', frequent_skill: { id: '0', name: 'Skill', image: '/images/skill/0.png' } } }; +const festival = id => ({ festival_id: id, names: { alpha_short: 'A', bravo_short: 'B' }, times: { start: 3600, end: 7200 }, images: { alpha: '/images/festival/a.png', bravo: '/images/festival/b.png', panel: '/images/festival/p.png' }, special_stage: stage('100', 'Shifty Station') }); + +export const ROUTES = { + '/api/schedules': () => ({ regular: [rotation(3600)], gachi: [rotation(3600)], league: [rotation(3600)] }), + '/api/data/stages': () => ({ stages: [stage('0', 'The Reef'), stage('1', 'Musselforge Fitness')] }), + '/api/coop_schedules': () => ({ schedules: [{ start_time: 3600, end_time: 7200 }], details: [{ start_time: 3600, end_time: 7200, stage: { name: 'Spawning Grounds', image: '/images/coop_stage/a.png' }, weapons: [null] }] }), + '/api/timeline': () => ({ coop: { importance: 1, reward_gear: { available_time: 0, gear } }, weapon_availability: null }), + '/api/festivals/active': () => ({ festivals: [festival(2)] }), + '/api/festivals/pasts': () => ({ festivals: [festival(1)], results: [{ festival_id: 1 }] }), + '/api/festivals/1/rankings': () => ({ rankings: [] }), + '/api/onlineshop/merchandises': () => ({ merchandises: [{ end_time: 7200, gear, skill: { id: '0', name: 'Skill', image: '/images/skill/0.png' } }] }), +}; + +/** Install on global fetch (e.g. vi.stubGlobal('fetch', fakeSplatNet())). Records every request. */ +export function fakeSplatNet(routes = ROUTES) { + const requests = []; + const fetch = async (input, init = {}) => { + const url = new URL(input); + const headers = new Headers(init.headers); + requests.push({ path: url.pathname, language: headers.get('Accept-Language'), cookie: headers.get('Cookie') }); + if (url.hostname === 'www.cloudflare.com') + return new Response('colo=TEST\n'); + if (url.pathname.startsWith('/images/')) + return new Response(new Uint8Array([0x89, 0x50, 0x4e, 0x47]), { headers: { 'content-type': 'image/png' } }); + const route = routes[url.pathname]; + if (!route) + return new Response('not found', { status: 404 }); + const result = await route(); + return result instanceof Response ? result : Response.json(result); + }; + fetch.requests = requests; + return fetch; +} + +export function setSessionEnvironment() { + process.env.NINTENDO_SESSION_ID_NA = SESSIONS.NA; + process.env.NINTENDO_SESSION_ID_EU = SESSIONS.EU; + process.env.NINTENDO_SESSION_ID_JP = SESSIONS.JP; +} diff --git a/workers/updater/schedule.spec.mjs b/workers/updater/schedule.spec.mjs new file mode 100644 index 0000000..c8d3e67 --- /dev/null +++ b/workers/updater/schedule.spec.mjs @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; +import { nextRunAt, HOUR_MS, GRACE_MS } from './src/schedule.mjs'; + +const HOUR = Date.UTC(2026, 8, 4, 14, 0, 0); // 14:00:00 UTC +const t = (seconds) => HOUR + seconds * 1000; + +describe('nextRunAt', () => { + it('targets :00:10 of the current hour when that is still ahead', () => { + expect(nextRunAt(t(0))).toBe(HOUR + GRACE_MS); + expect(nextRunAt(t(9))).toBe(HOUR + GRACE_MS); + }); + + it('moves to the next hour once :00:10 has passed', () => { + expect(nextRunAt(t(10))).toBe(HOUR + HOUR_MS + GRACE_MS); + expect(nextRunAt(t(12))).toBe(HOUR + HOUR_MS + GRACE_MS); + expect(nextRunAt(t(59 * 60 + 55))).toBe(HOUR + HOUR_MS + GRACE_MS); + }); +}); diff --git a/workers/updater/src/Scheduler.mjs b/workers/updater/src/Scheduler.mjs new file mode 100644 index 0000000..3011e60 --- /dev/null +++ b/workers/updater/src/Scheduler.mjs @@ -0,0 +1,153 @@ +// Durable Object that owns *when* and *where* updater jobs run. +// +// Why a Durable Object: Cron Triggers fire anywhere inside their minute and execute in +// whichever colo Cloudflare picks (placement hints only apply to fetch handlers). An +// object's alarm fires within milliseconds, and the object stays in the colo it was +// created in, next to the R2 buckets. So the cron trigger only wakes the object; the +// object does the work. +// +// Two ways work gets scheduled: +// - the hourly job re-arms itself for the next :00:10 after every run; +// - wake(job) asks for a job to run as soon as possible (used by cron-driven jobs). +// Both share one alarm: it is always set to the earliest thing that is due. + +import { DurableObject } from 'cloudflare:workers'; +import { runUpdaters } from './updaters.mjs'; +import { nextRunAt } from './schedule.mjs'; +import { createLogger, describeError } from './log.mjs'; +import { currentColo } from './colo.mjs'; + +const RETRY_DELAY_MS = 60 * 1000; +const MAX_RETRIES = 3; + +export const HOURLY_JOB = 'updaters'; + +// Jobs the object can run. The hourly job is the full updater run; more can be added here. +const JOBS = { + updaters: env => runUpdaters(env), +}; + +const EMPTY_STATE = { + hourlyAt: null, // next scheduled run of the hourly job + retryAt: null, // when set, a failed hourly run is retried at this time instead + retries: 0, + pending: [], // jobs requested through wake(), run at the next alarm + lastRuns: {}, // per job: timing and outcome of the most recent run +}; + +export class Scheduler extends DurableObject { + /** Schedule the hourly job if it is not scheduled yet. Safe to call repeatedly (the cron watchdog does). */ + async ensureArmed() { + let state = await this.#state(); + let armed = state.hourlyAt === null; + if (armed) { + state.hourlyAt = nextRunAt(Date.now()); + await this.#save(state); + } + return { armed, hourlyAt: state.hourlyAt, alarmAt: await this.#rearm(state) }; + } + + /** Run a job as soon as possible. Duplicate requests before the run collapse into one. */ + async wake(job) { + if (!Object.hasOwn(JOBS, job)) + return { job, accepted: false, error: `Unknown job: ${job}` }; + let state = await this.#state(); + if (!state.pending.includes(job)) + state.pending.push(job); + await this.#save(state); + return { job, accepted: true, alarmAt: await this.#rearm(state) }; + } + + async status() { + return { alarmAt: await this.ctx.storage.getAlarm(), ...await this.#state() }; + } + + async #state() { + return { ...EMPTY_STATE, ...await this.ctx.storage.get('state') ?? {} }; + } + + async #save(state) { + await this.ctx.storage.put('state', state); + } + + /** Point the single alarm at the earliest due time. */ + async #rearm(state) { + let candidates = [state.retryAt ?? state.hourlyAt, state.pending.length ? Date.now() : null] + .filter(time => time !== null); + if (!candidates.length) + return null; + let alarmAt = Math.min(...candidates); + await this.ctx.storage.setAlarm(alarmAt); + return alarmAt; + } + + async alarm(alarmInfo) { + let firedAt = Date.now(); + let log = createLogger('alarm'); + let colo = await currentColo(); + let state = await this.#state(); + + // Work out what is due. The hourly job is due when its time (or its retry time) has come; + // pending jobs are due now. A pending request for the hourly job merges into the hourly run. + let due = []; + let hourlyScheduledFor = state.retryAt ?? state.hourlyAt; + if (hourlyScheduledFor !== null && firedAt >= hourlyScheduledFor) + due.push({ job: HOURLY_JOB, reason: state.retryAt ? 'retry' : 'hourly', scheduledFor: hourlyScheduledFor }); + for (let job of state.pending) + if (Object.hasOwn(JOBS, job) && !due.some(entry => entry.job === job)) + due.push({ job, reason: 'wake', scheduledFor: null }); + state.pending = []; + + for (let { job, reason, scheduledFor } of due) { + let startedAt = Date.now(); + let run = { + job, + reason, + scheduledFor, + firedAt, + driftMs: scheduledFor === null ? null : firedAt - scheduledFor, + retryCount: alarmInfo?.retryCount ?? 0, + retries: state.retries, + colo, + }; + + // Errors are caught so the alarm is always re-armed; the platform's own alarm retries are + // capped and only cover the latest setAlarm(), so hourly retries are managed here. + try { + let result = await JOBS[job](this.env); + // A job reports partial failure by returning { ok: false } rather than throwing + run = { ...run, ok: result?.ok !== false, runMs: Date.now() - startedAt, result }; + if (!run.ok) + run.error = `${job} failed: ${result.updaters?.filter(u => !u.ok).map(u => u.name).join(', ')}`; + } catch (error) { + run = { ...run, ok: false, runMs: Date.now() - startedAt, ...describeError(error) }; + } + + if (job === HOURLY_JOB && reason !== 'wake') { + if (run.ok || state.retries >= MAX_RETRIES) { + state.hourlyAt = nextRunAt(Date.now()); + state.retryAt = null; + state.retries = 0; + } else { + state.retryAt = Date.now() + RETRY_DELAY_MS; + state.retries += 1; + } + } + + state.lastRuns[job] = run; + log[run.ok ? 'info' : 'error']('Alarm run finished', run); + } + + // This instance always owns the hourly job. If it has somehow been lost (for example the + // alarm was consumed by a wake before ensureArmed() ever ran), restore it here rather than + // waiting for the cron watchdog. + if (state.hourlyAt === null) { + state.hourlyAt = nextRunAt(Date.now()); + log.warn('Hourly schedule was missing; restored', { hourlyAt: state.hourlyAt }); + } + + await this.#save(state); + let alarmAt = await this.#rearm(state); + log.info('Alarm re-armed', { alarmAt, hourlyAt: state.hourlyAt, retryAt: state.retryAt, ran: due.map(entry => entry.job) }); + } +} diff --git a/workers/updater/src/colo.mjs b/workers/updater/src/colo.mjs new file mode 100644 index 0000000..a1af155 --- /dev/null +++ b/workers/updater/src/colo.mjs @@ -0,0 +1,12 @@ +// Which Cloudflare colo this invocation is running in, for comparing scheduling paths. +// A subrequest to a Cloudflare-fronted host is answered from the local colo, so the +// trace endpoint reports where this Worker's code is executing. +export async function currentColo() { + try { + let response = await fetch('https://www.cloudflare.com/cdn-cgi/trace'); + let text = await response.text(); + return text.match(/^colo=(\w+)$/m)?.[1] ?? null; + } catch { + return null; + } +} diff --git a/workers/updater/src/index.mjs b/workers/updater/src/index.mjs new file mode 100644 index 0000000..0959096 --- /dev/null +++ b/workers/updater/src/index.mjs @@ -0,0 +1,92 @@ +import { withSentry, instrumentDurableObjectWithSentry } from '@sentry/cloudflare'; +import { runUpdaters } from './updaters.mjs'; +import { Scheduler as SchedulerClass } from './Scheduler.mjs'; +import { createLogger, describeError } from './log.mjs'; + +// Sentry: the shared updater code reports through @sentry/core, which lands on the client +// these wrappers set up per invocation. With no SENTRY_DSN secret nothing is sent. +const sentryOptions = env => ({ dsn: env.SENTRY_DSN, tracesSampleRate: 0 }); + +export const Scheduler = instrumentDurableObjectWithSentry(sentryOptions, SchedulerClass); + +// The object is created next to the R2 buckets (western North America). Only the first +// get() for an object honors the hint; after that it stays where it is. +function scheduler(env) { + return env.SCHEDULER.get(env.SCHEDULER.idFromName('schedules'), { locationHint: 'wnam' }); +} + +// Cron Triggers never do work themselves: they wake the Scheduler, which runs jobs in +// place. Each entry maps a cron expression from wrangler.jsonc to a call on the object. +const CRON_ACTIONS = { + '30 * * * *': scheduler => scheduler.ensureArmed(), // watchdog: the hourly alarm must always be armed + // Example of a cron-driven job: '15 3 * * *': scheduler => scheduler.wake('some-daily-job'), +}; + +function timingSafeEqual(a, b) { + let encoder = new TextEncoder; + let left = encoder.encode(a); + let right = encoder.encode(b); + if (left.byteLength !== right.byteLength) + return false; + return crypto.subtle.timingSafeEqual(left, right); +} + +function isAuthorized(request, env) { + if (!env.RUN_TOKEN) + return false; + let header = request.headers.get('Authorization') ?? ''; + let [scheme, token] = header.split(' '); + return scheme === 'Bearer' && !!token && timingSafeEqual(token, env.RUN_TOKEN); +} + +export default withSentry(sentryOptions, { + async scheduled(controller, env, ctx) { + let log = createLogger('cron'); + let action = CRON_ACTIONS[controller.cron]; + if (!action) { + log.warn('No action for cron expression', { cron: controller.cron }); + return; + } + + try { + log.info('Cron action finished', { cron: controller.cron, result: await action(scheduler(env)) }); + } catch (error) { + log.error('Cron action failed', { cron: controller.cron, ...describeError(error) }); + throw error; // Mark the invocation as failed in Workers metrics + } + }, + + // Authenticated operator endpoints, "Authorization: Bearer ": + // POST /run[?only=Name,Name] run the updaters in this invocation and return the summary + // POST /wake?job=NAME ask the Scheduler to run a job as soon as possible + // POST /arm schedule the hourly job if it is not scheduled + // GET /status alarm state and the last run of each job + async fetch(request, env, ctx) { + let url = new URL(request.url); + let route = `${request.method} ${url.pathname}`; + if (!['POST /run', 'POST /wake', 'POST /arm', 'GET /status'].includes(route)) + return new Response('Not found', { status: 404 }); + if (!isAuthorized(request, env)) + return new Response('Unauthorized', { status: 401 }); + + try { + switch (route) { + case 'POST /run': { + let only = url.searchParams.get('only')?.split(',').map(name => name.trim()).filter(Boolean); + return Response.json(await runUpdaters(env, { only })); + } + case 'POST /wake': { + let woke = await scheduler(env).wake(url.searchParams.get('job') ?? ''); + return Response.json({ ok: woke.accepted, ...woke }, { status: woke.accepted ? 200 : 400 }); + } + case 'POST /arm': + return Response.json({ ok: true, ...await scheduler(env).ensureArmed() }); + case 'GET /status': + return Response.json({ ok: true, ...await scheduler(env).status() }); + } + } catch (error) { + createLogger('http').error(`${route} failed`, describeError(error)); + return Response.json({ ok: false, ...describeError(error) }, { status: 500 }); + } + }, +}); diff --git a/workers/updater/src/log.mjs b/workers/updater/src/log.mjs new file mode 100644 index 0000000..718884d --- /dev/null +++ b/workers/updater/src/log.mjs @@ -0,0 +1,19 @@ +// Structured logging for Workers Logs. Objects (not JSON strings) are logged so the +// dashboard extracts and indexes each field, e.g. filter on driftMs or updater. +export function createLogger(updater) { + function emit(level, message, fields) { + console[level]({ updater, message, ...fields }); + } + + return { + info: (message, fields = {}) => emit('info', message, fields), + warn: (message, fields = {}) => emit('warn', message, fields), + error: (message, fields = {}) => emit('error', message, fields), + }; +} + +export function describeError(error) { + return error instanceof Error + ? { error: error.message, stack: error.stack } + : { error: String(error) }; +} diff --git a/workers/updater/src/schedule.mjs b/workers/updater/src/schedule.mjs new file mode 100644 index 0000000..0434f3b --- /dev/null +++ b/workers/updater/src/schedule.mjs @@ -0,0 +1,11 @@ +// Time math for the alarm scheduler. The hourly job runs ten seconds past the top of the +// hour, the moment the container's node-cron ran at. + +export const HOUR_MS = 60 * 60 * 1000; +export const GRACE_MS = 10 * 1000; + +/** The :00:10 strictly after `now`. */ +export function nextRunAt(now = Date.now()) { + let candidate = now - (now % HOUR_MS) + GRACE_MS; + return candidate > now ? candidate : candidate + HOUR_MS; +} diff --git a/workers/updater/src/updaters.mjs b/workers/updater/src/updaters.mjs new file mode 100644 index 0000000..1a806f2 --- /dev/null +++ b/workers/updater/src/updaters.mjs @@ -0,0 +1,27 @@ +// Runs the shared updaters (src/app/updater) against this Worker's R2 bindings. +// This is the Worker's counterpart of src/app/local.js. + +import { BucketStorage } from '../../../src/common/storage/index.js'; +import { updateAll } from '../../../src/app/updater/index.js'; +import { createLogger } from './log.mjs'; + +export function bucketStorage(env) { + return { + publicStorage: new BucketStorage(env.ASSETS), + privateStorage: new BucketStorage(env.PRIVATE), + }; +} + +/** + * @param {{ ASSETS: R2Bucket, PRIVATE: R2Bucket }} env + * @param {{ only?: string[] }} [options] restrict to updaters with these names + * @returns {Promise<{ ok: boolean, ms: number, updaters: object[] }>} + */ +export async function runUpdaters(env, { only } = {}) { + let log = createLogger('updaters'); + let started = Date.now(); + let updaters = await updateAll(bucketStorage(env), { only }); + let summary = { ok: updaters.every(u => u.ok), ms: Date.now() - started, updaters }; + log[summary.ok ? 'info' : 'error']('Updaters finished', summary); + return summary; +} diff --git a/workers/updater/updaters.spec.mjs b/workers/updater/updaters.spec.mjs new file mode 100644 index 0000000..98a7a56 --- /dev/null +++ b/workers/updater/updaters.spec.mjs @@ -0,0 +1,55 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { env } from 'cloudflare:test'; +import { runUpdaters } from './src/updaters.mjs'; +import { fakeSplatNet, setSessionEnvironment } from './fakeSplatNet.mjs'; + +// The shared updaters (src/app/updater) running inside workerd against the R2 bindings. +setSessionEnvironment(); + +async function keys(bucket, prefix = '') { + return (await bucket.list({ prefix })).objects.map(o => o.key).sort(); +} + +describe('runUpdaters', () => { + let fetch; + beforeEach(() => { fetch = fakeSplatNet(); vi.stubGlobal('fetch', fetch); }); + afterEach(() => vi.unstubAllGlobals()); + + it('runs every updater against the R2 bindings and publishes the site data', async () => { + let summary = await runUpdaters(env); + + expect(summary.ok).toBe(true); + expect(summary.updaters.map(u => u.name)).toEqual([ + 'Original Gear', 'Schedules', 'Co-op Schedules', 'Timeline', 'Festivals NA', 'Festivals EU', 'Festivals JP', 'Merchandises', + ]); + expect(await keys(env.ASSETS, 'data/')).toEqual(expect.arrayContaining([ + 'data/schedules.json', 'data/coop-schedules.json', 'data/coop-schedules.ics', 'data/timeline.json', + 'data/festivals.json', 'data/festivals-na.ics', 'data/festivals-eu.ics', 'data/festivals-jp.ics', + 'data/festivals/na-1-rankings.json', 'data/merchandises.json', 'data/locale/en.json', 'data/locale/ja.json', + ])); + expect(await keys(env.PRIVATE)).toEqual(['stages.json']); + + let schedules = await (await env.ASSETS.get('data/schedules.json')).json(); + expect(schedules.regular[0].stage_a.name).toBe('The Reef'); + expect((await env.ASSETS.head('data/schedules.json')).httpMetadata.cacheControl).toContain('stale-while-revalidate'); + expect(await keys(env.ASSETS, 'assets/splatnet/images/stage/')).toEqual(['assets/splatnet/images/stage/0.png', 'assets/splatnet/images/stage/1.png', 'assets/splatnet/images/stage/100.png']); + + let festivals = await (await env.ASSETS.get('data/festivals.json')).json(); + expect(Object.keys(festivals).sort()).toEqual(['eu', 'jp', 'na']); + expect(fetch.requests.find(r => r.path === '/api/schedules').cookie).toBe('iksm_session=na-session'); + }); + + it('can run a subset of updaters, and reports a failing one without stopping the rest', async () => { + await env.ASSETS.delete(['data/timeline.json', 'data/merchandises.json']); + vi.stubGlobal('fetch', fakeSplatNet({ '/api/timeline': () => new Response('down', { status: 503 }), '/api/onlineshop/merchandises': () => ({ merchandises: [] }) })); + let summary = await runUpdaters(env, { only: ['Timeline', 'Merchandises'] }); + + expect(summary.ok).toBe(false); + expect(summary.updaters).toMatchObject([ + { name: 'Timeline', ok: false, error: expect.stringContaining('status 503') }, + { name: 'Merchandises', ok: true }, + ]); + expect(await env.ASSETS.get('data/timeline.json')).toBeNull(); + expect(await env.ASSETS.get('data/merchandises.json')).not.toBeNull(); + }); +}); diff --git a/workers/updater/vitest.config.mjs b/workers/updater/vitest.config.mjs new file mode 100644 index 0000000..ef011dc --- /dev/null +++ b/workers/updater/vitest.config.mjs @@ -0,0 +1,23 @@ +import { fileURLToPath } from 'node:url'; +import { cloudflareTest } from '@cloudflare/vitest-plugin'; +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + plugins: [ + cloudflareTest({ + wrangler: { + configPath: fileURLToPath(new URL('./wrangler.jsonc', import.meta.url)), + }, + }), + ], + test: { + include: ['workers/updater/**/*.spec.mjs'], + deps: { + optimizer: { + // ics -> yup -> property-expr is CommonJS; pre-bundle it so its named exports resolve + // in vitest's module runner the way they do in wrangler's bundle. + ssr: { enabled: true, include: ['ics'] }, + }, + }, + }, +}); diff --git a/workers/updater/wrangler.jsonc b/workers/updater/wrangler.jsonc new file mode 100644 index 0000000..ad72b3c --- /dev/null +++ b/workers/updater/wrangler.jsonc @@ -0,0 +1,43 @@ +{ + "$schema": "../../node_modules/wrangler/config-schema.json", + "name": "splatoon2-ink-updater", + "main": "src/index.mjs", + "compatibility_date": "2026-09-03", + "compatibility_flags": ["nodejs_compat"], + // The workers.dev URL only exposes the authenticated manual trigger (see README). + "workers_dev": true, + "preview_urls": false, + "triggers": { + // Cron Triggers only wake the Scheduler Durable Object (see src/index.mjs CRON_ACTIONS). + // The hourly update runs from the object's own alarm at :00:10; this cron is the watchdog + // that re-arms that alarm if it is ever lost. Minute 30 keeps it clear of the hourly run. + "crons": ["30 * * * *"] + }, + "durable_objects": { + "bindings": [ + { "name": "SCHEDULER", "class_name": "Scheduler" } + ] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["Scheduler"] } + ], + "r2_buckets": [ + { + // Shadow mode: writes go to the dev bucket while the container still owns production. + // Cutover = point this at "splatoon2-ink-assets". + "binding": "ASSETS", + "bucket_name": "splatoon2-ink-dev-assets" + }, + { + // Mutable updater state that is not public (the old storage/ directory). + "binding": "PRIVATE", + "bucket_name": "splatoon2-ink-private" + } + ], + "observability": { + "enabled": true, + "logs": { + "head_sampling_rate": 1 + } + } +}