From 9971afe2e86e0da363089f2855e751d8be9fee49 Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Mon, 7 Sep 2026 17:52:36 -0700 Subject: [PATCH] Improve spacing and visual grouping throughout runtime code --- scripts/compare-data.mjs | 37 +++++++- src/app/log.js | 7 ++ src/app/node.js | 8 +- src/app/screenshots/BrowserRunClient.js | 39 ++++++-- src/app/screenshots/PuppeteerRenderer.js | 2 +- src/app/screenshots/ScreenshotGenerator.js | 5 ++ src/app/screenshots/cli.js | 32 ++++--- src/app/screenshots/convert.node.js | 4 +- src/app/screenshots/convert.worker.js | 4 +- src/app/screenshots/node.js | 28 ++++-- src/app/social/posts/GearPost.js | 12 ++- src/app/social/posts/NewStagePost.js | 16 ++-- src/app/social/posts/SalmonRunGearPost.js | 13 ++- src/app/social/posts/SocialPostBase.js | 38 +++++--- src/app/social/social.js | 5 +- src/app/updater/LocalizationProcessor.js | 10 ++- src/app/updater/updateAll.js | 12 ++- src/app/updater/updaters/FestivalsUpdater.js | 89 ++++++++++-------- src/app/updater/updaters/Updater.js | 28 ++++-- src/common/cdn.node.js | 6 +- src/common/cdn.worker.js | 1 + src/common/cli.js | 2 + src/common/dataRefresh.js | 2 + src/common/fetch.js | 1 + src/common/screenshot.js | 4 + src/common/splatnet.js | 19 ++-- src/common/storage/BucketStorage.js | 25 ++++-- src/common/storage/FilesystemStorage.js | 6 ++ src/common/storage/MemoryBucket.js | 36 ++++++-- src/common/storage/index.js | 1 + src/web/components/Screenshots.vue | 2 + src/web/store/splatoon/data.js | 36 +++++--- workers/updater/preview/server.mjs | 44 ++++++--- workers/updater/src/Scheduler.mjs | 95 ++++++++++++++++---- workers/updater/src/admin/access.mjs | 15 +++- workers/updater/src/admin/page.html | 69 +++++++++++--- workers/updater/src/admin/routes.mjs | 18 +++- workers/updater/src/index.mjs | 55 ++++++++++-- workers/updater/src/log.mjs | 5 +- workers/updater/src/posters.mjs | 30 +++++-- workers/updater/src/schedule.mjs | 1 + workers/updater/src/updaters.mjs | 2 + 42 files changed, 671 insertions(+), 193 deletions(-) diff --git a/scripts/compare-data.mjs b/scripts/compare-data.mjs index bf9b7ae..9462afc 100644 --- a/scripts/compare-data.mjs +++ b/scripts/compare-data.mjs @@ -11,6 +11,7 @@ import path from 'node:path'; import stringify from 'json-stable-stringify'; const [base, current] = process.argv.slice(2); + if (!base || !current) { console.error('Usage: node scripts/compare-data.mjs '); process.exit(2); @@ -20,54 +21,81 @@ function firstDifference(a, b, at = '$') { if (Array.isArray(a) && Array.isArray(b)) { if (a.length !== b.length) return `${at}.length ${a.length} vs ${b.length}`; + // Festival result lists come back in random order if (a.length && a[0] && typeof a[0] === 'object' && 'festival_id' in a[0]) { let leftItems = a.map(item => stringify(item)).sort(); let rightItems = b.map(item => stringify(item)).sort(); - return leftItems.every((item, index) => item === rightItems[index]) ? null : `${at}: set of items differs`; + + return leftItems.every((item, index) => item === rightItems[index]) + ? null + : `${at}: set of items differs`; } + for (let i = 0; i < a.length; i++) { let difference = firstDifference(a[i], b[i], `${at}[${i}]`); + if (difference) return difference; } + return null; } + if (a && b && typeof a === 'object' && typeof b === 'object') { for (let key of new Set([...Object.keys(a), ...Object.keys(b)])) { let difference = firstDifference(a[key], b[key], `${at}.${key}`); + if (difference) return difference; } + return null; } + return a === b ? null : `${at}: ${JSON.stringify(a)?.slice(0, 60)} vs ${JSON.stringify(b)?.slice(0, 60)}`; } -const walk = dir => fs.readdirSync(dir, { withFileTypes: true }) - .flatMap(entry => entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]); +const walk = dir => + fs + .readdirSync(dir, { withFileTypes: true }) + .flatMap(entry => + entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)], + ); let problems = 0; + for (let file of walk(base)) { let relative = path.relative(base, file); let other = path.join(current, relative); + if (!fs.existsSync(other)) { console.log(`${relative}: missing in ${current}`); problems++; continue; } + let a = fs.readFileSync(file); let b = fs.readFileSync(other); + if (a.equals(b)) continue; + if (relative.endsWith('.json')) { let difference = firstDifference(JSON.parse(a), JSON.parse(b)); + if (difference) { console.log(`${relative}: ${difference}`); problems++; } } else if (relative.endsWith('.ics')) { - let strip = s => s.toString().split(/\r?\n/).filter(line => !line.startsWith('DTSTAMP')).join('\n'); + let strip = s => + s + .toString() + .split(/\r?\n/) + .filter(line => !line.startsWith('DTSTAMP')) + .join('\n'); + if (strip(a) !== strip(b)) { console.log(`${relative}: calendar content differs`); problems++; @@ -77,5 +105,6 @@ for (let file of walk(base)) { problems++; } } + console.log(problems ? `${problems} real difference(s)` : 'semantically identical'); process.exit(problems ? 1 : 0); diff --git a/src/app/log.js b/src/app/log.js index b3976f0..5f7b1af 100644 --- a/src/app/log.js +++ b/src/app/log.js @@ -23,11 +23,13 @@ export class RunLog { append(level, message, fields) { let line = { at: Date.now(), level, text: this.formatStoredMessage(message, fields) }; + this.snapshot.lines.push(line); this.storedBytes += this.storedLineBytes(line); while (this.snapshot.lines.length > MAX_STORED_LINES || this.storedBytes > MAX_STORED_BYTES) { let removed = this.snapshot.lines.shift(); + this.storedBytes -= this.storedLineBytes(removed); this.snapshot.omitted++; } @@ -39,17 +41,21 @@ export class RunLog { // Keep readable progress messages; the full structured result has its own JSON view. if (fields?.updater) text = `[${fields.updater}] ${text}`; + if (fields?.error) text += `: ${fields.error}`; + if (fields?.attempt) text += ` (retry ${fields.attempt})`; for (let secret of this.secrets) text = text.replaceAll(secret, '[redacted]'); + text = text.replace(/Bearer\s+[^\s,;]+/gi, 'Bearer [redacted]'); if (text.length > MAX_STORED_LINE_LENGTH) text = text.slice(0, MAX_STORED_LINE_LENGTH) + '…'; + return text; } @@ -68,6 +74,7 @@ export function logMessage(level, message, fields) { } let runLog = currentRunLog.getStore(); + if (runLog) runLog.append(level, message, fields); } diff --git a/src/app/node.js b/src/app/node.js index 5a2ac5f..5b19e08 100644 --- a/src/app/node.js +++ b/src/app/node.js @@ -15,16 +15,22 @@ export function filesystemStorage() { export async function updateAll() { let updaters = await updater.updateAll(filesystemStorage()); + if (updaters.some(updater => !updater.ok)) throw new Error('One or more updaters failed; social posting skipped.'); + return 'Done'; } export async function sendStatuses() { let clients = social.createClients(); - let result = await withScreenshots(screenshots => social.sendStatuses(filesystemStorage(), clients, screenshots)); + let result = await withScreenshots(screenshots => + social.sendStatuses(filesystemStorage(), clients, screenshots), + ); + if (!result.ok) throw new Error('One or more social posts failed.'); + return result; } diff --git a/src/app/screenshots/BrowserRunClient.js b/src/app/screenshots/BrowserRunClient.js index 24252c4..ddbbadd 100644 --- a/src/app/screenshots/BrowserRunClient.js +++ b/src/app/screenshots/BrowserRunClient.js @@ -5,7 +5,10 @@ async function errorMessage(response) { let body = await response.text(); try { - let messages = JSON.parse(body).errors?.map(error => error.message).filter(Boolean); + let messages = JSON.parse(body) + .errors?.map(error => error.message) + .filter(Boolean); + if (messages?.length) return messages.join('; '); } catch { @@ -24,20 +27,26 @@ export default class BrowserRunClient { async capture({ url, viewport, readySelector }) { let missing = []; + if (!this.accountId) missing.push('CLOUDFLARE_ACCOUNT_ID'); + if (!this.apiToken) missing.push('CLOUDFLARE_BROWSER_RUN_API_TOKEN'); + if (missing.length) throw new Error(`Missing screenshot configuration: ${missing.join(', ')}`); - let endpoint = new URL(`/client/v4/accounts/${this.accountId}/browser-rendering/screenshot`, 'https://api.cloudflare.com'); + let endpoint = new URL( + `/client/v4/accounts/${this.accountId}/browser-rendering/screenshot`, + 'https://api.cloudflare.com', + ); endpoint.searchParams.set('cacheTTL', '0'); let request = { method: 'POST', headers: { - 'Authorization': `Bearer ${this.apiToken}`, + Authorization: `Bearer ${this.apiToken}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ @@ -51,29 +60,45 @@ export default class BrowserRunClient { }), }; let image; + for (let attempt = 0; attempt <= 3; attempt++) { try { // Three 10s browser phases plus transport overhead; covers response body too. let response = await fetchWithTimeout(endpoint, request, 40_000); + if (!response.ok) { let message = await errorMessage(response); - let error = new Error(`Browser Rendering screenshot failed (${response.status}): ${message}`); + let error = new Error( + `Browser Rendering screenshot failed (${response.status}): ${message}`, + ); + // Browser Run reports navigation/selector/action timeouts as 422 errors. - error.retryable = response.status === 408 || response.status >= 500 - || (response.status === 422 && /timeout|timed out/i.test(message)); + error.retryable = + response.status === 408 || + response.status >= 500 || + (response.status === 422 && /timeout|timed out/i.test(message)); throw error; } + image = new Uint8Array(await response.arrayBuffer()); break; } catch (error) { let retryable = error.retryable ?? ['TimeoutError', 'TypeError'].includes(error.name); + if (!retryable || attempt === 3) throw error; + let delayMs = 500 * 2 ** attempt; - logMessage('warn', 'Retrying Browser Run screenshot', { attempt: attempt + 1, delayMs, error: error.message }); + + logMessage('warn', 'Retrying Browser Run screenshot', { + attempt: attempt + 1, + delayMs, + error: error.message, + }); await new Promise(resolve => setTimeout(resolve, delayMs)); } } + return image; } } diff --git a/src/app/screenshots/PuppeteerRenderer.js b/src/app/screenshots/PuppeteerRenderer.js index ff24eff..85ae5a6 100644 --- a/src/app/screenshots/PuppeteerRenderer.js +++ b/src/app/screenshots/PuppeteerRenderer.js @@ -10,7 +10,6 @@ export default class PuppeteerRenderer { try { let page = await browser.newPage(); - page.setDefaultTimeout(10_000); await page.setViewport(viewport); await page.setCacheEnabled(false); @@ -26,6 +25,7 @@ export default class PuppeteerRenderer { await page.waitForSelector(readySelector); let image = await page.screenshot({ type: 'png' }); + return new Uint8Array(image); } finally { await browser.close(); diff --git a/src/app/screenshots/ScreenshotGenerator.js b/src/app/screenshots/ScreenshotGenerator.js index bd91671..89d1b45 100644 --- a/src/app/screenshots/ScreenshotGenerator.js +++ b/src/app/screenshots/ScreenshotGenerator.js @@ -19,15 +19,18 @@ export default class ScreenshotGenerator { if (!url) { if (!this.siteUrl) throw new Error('SITE_URL is required to capture a screenshot route.'); + url = new URL('/screenshots.html', this.siteUrl); url.hash = hash; } + let captureViewport = { ...defaultViewport, ...viewport }; let image = await this.renderer.capture({ url: new URL(url).toString(), viewport: captureViewport, readySelector: screenshotReadySelector, }); + return { image, type: 'image/png', @@ -67,6 +70,7 @@ export default class ScreenshotGenerator { const rows = Math.ceil(weaponCount / 4); // Determine the image height based on the number of rows let height = rows * 320; + // Add some extra height for the bottom banner height += 60; // Set a minimum overall image height @@ -77,6 +81,7 @@ export default class ScreenshotGenerator { captureSplatfestScreenshot(region, now, regions) { regions = regions.join(','); + let hash = `/splatfest/${region}/${now}?regions=${regions}`; return this.capture({ hash }); diff --git a/src/app/screenshots/cli.js b/src/app/screenshots/cli.js index 90af181..3a7581e 100644 --- a/src/app/screenshots/cli.js +++ b/src/app/screenshots/cli.js @@ -9,22 +9,34 @@ if (existsSync('.env')) process.loadEnvFile(); try { - let { values } = parseArgs({ options: { - provider: { type: 'string' }, - url: { type: 'string' }, - hash: { type: 'string' }, - output: { type: 'string', default: 'dist/test-screenshots/capture.png' }, - help: { type: 'boolean' }, - } }); + let { values } = parseArgs({ + options: { + provider: { type: 'string' }, + url: { type: 'string' }, + hash: { type: 'string' }, + output: { type: 'string', default: 'dist/test-screenshots/capture.png' }, + help: { type: 'boolean' }, + }, + }); + if (values.help) { - console.log('Usage: npm run screenshot -- [--provider puppeteer|cloudflare] (--url | --hash ) [--output ]'); - console.log('Uses SCREENSHOT_PROVIDER and SITE_URL from the environment or .env. Without SITE_URL, Puppeteer serves dist/ temporarily.'); + console.log( + 'Usage: npm run screenshot -- [--provider puppeteer|cloudflare] (--url | --hash ) [--output ]', + ); + console.log( + 'Uses SCREENSHOT_PROVIDER and SITE_URL from the environment or .env. Without SITE_URL, Puppeteer serves dist/ temporarily.', + ); } else { if (!!values.url === !!values.hash) - throw new Error('Specify exactly one of --url or --hash. Use --help for examples of the options.'); + throw new Error( + 'Specify exactly one of --url or --hash. Use --help for examples of the options.', + ); + let result = await withScreenshots(screenshots => screenshots.capture(values), values); + await mkdir(dirname(values.output), { recursive: true }); await writeFile(values.output, result.image); + console.log(`Saved ${result.width}×${result.height} PNG to ${values.output}`); } } catch (error) { diff --git a/src/app/screenshots/convert.node.js b/src/app/screenshots/convert.node.js index 42030b3..9ab53e9 100644 --- a/src/app/screenshots/convert.node.js +++ b/src/app/screenshots/convert.node.js @@ -2,5 +2,7 @@ import sharp from 'sharp'; // Node-only adapter; package imports keep native sharp out of the Worker bundle. export async function convertToJpeg(png) { - return new Uint8Array(await sharp(png).flatten({ background: '#ffffff' }).jpeg({ quality: 90 }).toBuffer()); + return new Uint8Array( + await sharp(png).flatten({ background: '#ffffff' }).jpeg({ quality: 90 }).toBuffer(), + ); } diff --git a/src/app/screenshots/convert.worker.js b/src/app/screenshots/convert.worker.js index cbd7943..51d1c93 100644 --- a/src/app/screenshots/convert.worker.js +++ b/src/app/screenshots/convert.worker.js @@ -1,8 +1,10 @@ import { env } from 'cloudflare:workers'; export async function convertToJpeg(png, images = env.IMAGES) { - let result = await images.input(new Response(png).body) + let result = await images + .input(new Response(png).body) .transform({ background: '#ffffff' }) .output({ format: 'image/jpeg', quality: 90 }); + return new Uint8Array(await result.response().arrayBuffer()); } diff --git a/src/app/screenshots/node.js b/src/app/screenshots/node.js index 0a4d60f..9bc07d1 100644 --- a/src/app/screenshots/node.js +++ b/src/app/screenshots/node.js @@ -7,43 +7,55 @@ import ScreenshotGenerator from './ScreenshotGenerator.js'; // Provider selection and the temporary file server belong to the Node command. // The Worker constructs its BrowserRunClient directly. -export async function withScreenshots(callback, { - provider = process.env.SCREENSHOT_PROVIDER, - siteUrl = process.env.SITE_URL, - url, -} = {}) { +export async function withScreenshots( + callback, + { provider = process.env.SCREENSHOT_PROVIDER, siteUrl = process.env.SITE_URL, url } = {}, +) { let renderer; + if (provider === 'cloudflare') { renderer = new BrowserRunClient({ accountId: process.env.CLOUDFLARE_ACCOUNT_ID, apiToken: process.env.CLOUDFLARE_BROWSER_RUN_API_TOKEN, }); + if (!siteUrl && !url) throw new Error('SITE_URL or --url is required for Cloudflare screenshots.'); } else if (provider === 'puppeteer') { - renderer = new PuppeteerRenderer; + renderer = new PuppeteerRenderer(); } else { throw new Error('SCREENSHOT_PROVIDER must be "puppeteer" or "cloudflare" (or pass --provider).'); } let server; + try { if (!siteUrl && !url) { await access('dist/screenshots.html').catch(() => { - throw new Error('Missing dist/screenshots.html. Run npm run build, or set SITE_URL to your dev server.'); + throw new Error( + 'Missing dist/screenshots.html. Run npm run build, or set SITE_URL to your dev server.', + ); }); + server = createServer((request, response) => { - handler(request, response, { public: 'dist', cleanUrls: false, directoryListing: false }).catch(error => { + handler(request, response, { + public: 'dist', + cleanUrls: false, + directoryListing: false, + }).catch(error => { console.error(error); response.destroy(); }); }); + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); }); + siteUrl = `http://127.0.0.1:${server.address().port}`; } + return await callback(new ScreenshotGenerator(renderer, siteUrl)); } finally { if (server) { diff --git a/src/app/social/posts/GearPost.js b/src/app/social/posts/GearPost.js index 0379e91..6cb8dd1 100644 --- a/src/app/social/posts/GearPost.js +++ b/src/app/social/posts/GearPost.js @@ -2,8 +2,12 @@ import SocialPostBase from './SocialPostBase.js'; import { getTopOfCurrentHour } from '../../../common/time.js'; export default class GearPost extends SocialPostBase { - getKey() { return 'gear'; } - getName() { return 'Gear'; } + getKey() { + return 'gear'; + } + getName() { + return 'Gear'; + } async getMerchandises() { return (await this.readData('merchandises.json')).merchandises; @@ -13,21 +17,25 @@ export default class GearPost extends SocialPostBase { // We only have end_times for merchandise items, so we need to track the latest end_time let endTimes = (await this.getMerchandises()).map(m => m.end_time); let lastEndTime = Math.max(...endTimes); + return lastEndTime; } async getData() { let time = await this.getDataTime(); + return (await this.getMerchandises()).find(m => m.end_time == time); } async getTestData() { let merchandises = await this.getMerchandises(); + return merchandises[merchandises.length - 1]; } getImage(data) { let now = getTopOfCurrentHour(); + return this.screenshots.captureGearScreenshot(now); } diff --git a/src/app/social/posts/NewStagePost.js b/src/app/social/posts/NewStagePost.js index a6369cb..24e689f 100644 --- a/src/app/social/posts/NewStagePost.js +++ b/src/app/social/posts/NewStagePost.js @@ -1,15 +1,20 @@ import SocialPostBase from './SocialPostBase.js'; export default class NewStagePost extends SocialPostBase { - getKey() { return 'newstage'; } - getName() { return 'New Stage'; } + getKey() { + return 'newstage'; + } + getName() { + return 'New Stage'; + } async getStages() { - return await this.readState('stages.json') ?? []; + return (await this.readState('stages.json')) ?? []; } async getData() { let time = await this.getDataTime(); + return (await this.getStages()).find(s => s.first_seen == time); } @@ -18,8 +23,9 @@ export default class NewStagePost extends SocialPostBase { } async getText(data) { - let hours = (data.first_available - await this.getDataTime()) / 60 / 60; - let duration = (hours == 1) ? '1 hour' : `${hours} hours`; + let hours = (data.first_available - (await this.getDataTime())) / 60 / 60; + let duration = hours == 1 ? '1 hour' : `${hours} hours`; + return `NEW STAGE: The first schedules for ${data.name} have been posted! Start playing the new stage when this post is ${duration} old. #splatoon2`; } } diff --git a/src/app/social/posts/SalmonRunGearPost.js b/src/app/social/posts/SalmonRunGearPost.js index 1a87207..48bb766 100644 --- a/src/app/social/posts/SalmonRunGearPost.js +++ b/src/app/social/posts/SalmonRunGearPost.js @@ -2,17 +2,23 @@ import SocialPostBase from './SocialPostBase.js'; import moment from 'moment-timezone'; export default class SalmonRunGearPost extends SocialPostBase { - getKey() { return 'salmonrungear'; } - getName() { return 'Salmon Run Gear'; } + getKey() { + return 'salmonrungear'; + } + getName() { + return 'Salmon Run Gear'; + } async getRewardGear() { let timeline = await this.readData('timeline.json'); + return timeline.coop && timeline.coop.reward_gear; } async getData() { let rewardGear = await this.getRewardGear(); - if (rewardGear && rewardGear.available_time == await this.getDataTime()) + + if (rewardGear && rewardGear.available_time == (await this.getDataTime())) return rewardGear; } @@ -26,6 +32,7 @@ export default class SalmonRunGearPost extends SocialPostBase { getText(data) { let monthName = moment.unix(data.available_time).tz('UTC').format('MMMM'); + return `New Salmon Run reward gear has been posted! ${monthName}'s gear is the ${data.gear.name}. #salmonrun #splatoon2`; } } diff --git a/src/app/social/posts/SocialPostBase.js b/src/app/social/posts/SocialPostBase.js index 3de8bf7..48538e9 100644 --- a/src/app/social/posts/SocialPostBase.js +++ b/src/app/social/posts/SocialPostBase.js @@ -22,22 +22,26 @@ export default class SocialPostBase { try { // Read once: Salmon Run's getData also remembers the current shift. let data = await this.getData(); + if (!data) return false; let due = []; + for (let client of this.clients) - if (await client.canSend() && await this.shouldPostForCurrentTime(client)) + if ((await client.canSend()) && (await this.shouldPostForCurrentTime(client))) due.push(client); // Without credentials, retain the public-image-only shadow mode. let enabled = await this.canPost(); + if (!due.length && (enabled || !this.getPublicImageFilename())) return false; return await this.post(data, due); } catch (error) { this.error(`Could not prepare post: ${error.message}`); + return { ok: false, error: error.message, clients: [] }; } } @@ -46,6 +50,7 @@ export default class SocialPostBase { for (let client of this.clients) if (await client.canSend()) return true; + return false; } @@ -54,12 +59,15 @@ export default class SocialPostBase { let text = await this.getText(data); let time = await this.getDataTime(); let image = await this.getMedia(data); + await this.maybeSavePublicImage(data, image.file); + let media = { 'image/png': image }; for (let client of clients) { try { let mediaType = client.mediaType ?? 'image/png'; + media[mediaType] ??= await this.convertMedia(image, mediaType); await client.send({ status: text, media: [media[mediaType]] }); await this.updateLastPostTime(client, time); @@ -70,11 +78,13 @@ export default class SocialPostBase { this.error(`Could not post to ${client.name}: ${error.message}`); } } + return { ok: results.every(result => result.ok), clients: results }; } async maybeSavePublicImage(data, image) { let filename = this.getPublicImageFilename(); + if (filename) { // Keep the established public URLs; removing X does not require breaking image links. await this.publicStorage.writeBytes(`twitter-images/${filename}`, image); @@ -85,8 +95,10 @@ export default class SocialPostBase { async saveTestScreenshot() { try { let data = await this.getTestData(); + if (!data) { this.info('No data available'); + return; } @@ -95,8 +107,7 @@ export default class SocialPostBase { await this.publicStorage.writeBytes(key, file); this.info(`Saved screenshot as ${key}`); - } - catch (e) { + } catch (e) { this.error('Couldn\'t save screenshot'); throw e; } @@ -129,18 +140,18 @@ export default class SocialPostBase { } async getLastPostTimes(client) { - return await this.readState(this.getLastPostTimesKey(client)) ?? {}; + return (await this.readState(this.getLastPostTimesKey(client))) ?? {}; } async getLastPostTime(client) { let key = this.getKey(); + return (await this.getLastPostTimes(client))[key] || 0; } async updateLastPostTime(client, time) { let key = this.getKey(); let lastPostTimes = await this.getLastPostTimes(client); - lastPostTimes[key] = time; await this.writeState(this.getLastPostTimesKey(client), lastPostTimes); @@ -150,6 +161,7 @@ export default class SocialPostBase { // Check whether the current data time has already been posted let time = await this.getDataTime(); let lastPostTime = await this.getLastPostTime(client); + return lastPostTime < time; } @@ -159,6 +171,7 @@ export default class SocialPostBase { formatLogMessage(message) { let name = this.getName(); + return `[Social] [${name}] ${message}`; } @@ -179,10 +192,10 @@ export default class SocialPostBase { */ // The unique key for this Post (used for storing the last time this Post was posted) - getKey() { } + getKey() {} // The friendly name for this Post (used for console log messages) - getName() { } + getName() {} // The time which the current Post is based off of (usually the top of the current hour) async getDataTime() { @@ -190,7 +203,7 @@ export default class SocialPostBase { } // The current data item the Post is based on (used by getImage and getText) - async getData() { } + async getData() {} // Data for test screenshots async getTestData() { @@ -199,7 +212,7 @@ export default class SocialPostBase { // The image to post with the Post, as a screenshot result ({ image, type, width, height }) // or raw PNG bytes. - async getImage(data) { } + async getImage(data) {} // The image as a media attachment: { file, type, width?, height? } async getMedia(data) { @@ -207,6 +220,7 @@ export default class SocialPostBase { if (result instanceof Uint8Array) { let size = pngSize(result) ?? {}; + return { file: result, type: 'image/png', ...size }; } @@ -216,18 +230,20 @@ export default class SocialPostBase { async convertMedia(image, mediaType) { if (mediaType !== 'image/jpeg') throw new Error(`Unsupported social image type: ${mediaType}`); + return { ...image, file: await convertToJpeg(image.file), type: mediaType }; } // The filename to store the image as (optional) - getPublicImageFilename() { } + getPublicImageFilename() {} // The text body of the Post - async getText(data) { } + async getText(data) {} // The key for test screenshots in public storage getTestScreenshotKey() { let key = this.getKey(); + return `test-screenshots/${key}.png`; } diff --git a/src/app/social/social.js b/src/app/social/social.js index 61659dc..b9e5ad8 100644 --- a/src/app/social/social.js +++ b/src/app/social/social.js @@ -2,7 +2,7 @@ import { createPosts } from './posts/index.js'; import BlueskyClient from './clients/BlueskyClient.js'; export function createClients() { - return [new BlueskyClient]; + return [new BlueskyClient()]; } /** @@ -13,10 +13,13 @@ export function createClients() { */ export async function sendStatuses(storage, clients = createClients(), screenshots) { let posts = []; + for (let post of createPosts(storage, clients, screenshots)) { let result = await post.maybePost(); + posts.push({ key: post.getKey(), ...(result || { ok: true, skipped: true }) }); } + return { ok: posts.every(post => post.ok), posts }; } diff --git a/src/app/updater/LocalizationProcessor.js b/src/app/updater/LocalizationProcessor.js index 3d81935..4ea6bad 100644 --- a/src/app/updater/LocalizationProcessor.js +++ b/src/app/updater/LocalizationProcessor.js @@ -14,10 +14,12 @@ export default class LocalizationProcessor { this.storage = storage; let entities = this.ruleset.entities; - this.entityExpressions = (Array.isArray(entities)) ? entities : [entities]; + + this.entityExpressions = Array.isArray(entities) ? entities : [entities]; let values = this.ruleset.values; - this.valueExpressions = (Array.isArray(values)) ? values : [values]; + + this.valueExpressions = Array.isArray(values) ? values : [values]; } getKey() { @@ -25,7 +27,7 @@ export default class LocalizationProcessor { } async readData() { - return await this.storage.readJson(this.getKey()) ?? {}; + return (await this.storage.readJson(this.getKey())) ?? {}; } writeData(data) { @@ -57,6 +59,7 @@ export default class LocalizationProcessor { for (let entity of this.entities(data)) { let ids = this.getIdValues(entity); + for (let valueKey of this.valueExpressions) _.setWith(localizations, this.getExpression(ids, valueKey), _.get(entity, valueKey), Object); } @@ -69,6 +72,7 @@ export default class LocalizationProcessor { for (let entity of this.entities(data)) { let ids = this.getIdValues(entity); + for (let valueKey of this.valueExpressions) if (_.get(localizations, this.getExpression(ids, valueKey)) === undefined) return false; diff --git a/src/app/updater/updateAll.js b/src/app/updater/updateAll.js index 635a20b..a23d818 100644 --- a/src/app/updater/updateAll.js +++ b/src/app/updater/updateAll.js @@ -33,23 +33,33 @@ export default async function updateAll(storage, { only } = {}) { let results = []; let updaters = createUpdaters(storage); + if (only) { let unknown = only.filter(name => !updaters.some(updater => updater.options.name === name)); + if (!only.length || unknown.length) throw new Error(`Unknown or empty updater selection: ${unknown.join(', ')}`); } + for (let updater of updaters) { let name = updater.options.name; + if (only && !only.includes(name)) continue; let started = Date.now(); + try { await updater.update(); results.push({ name, ok: true, ms: Date.now() - started }); } catch (e) { logMessage('error', e); - results.push({ name, ok: false, ms: Date.now() - started, error: e instanceof Error ? e.message : String(e) }); + results.push({ + name, + ok: false, + ms: Date.now() - started, + error: e instanceof Error ? e.message : String(e), + }); } } diff --git a/src/app/updater/updaters/FestivalsUpdater.js b/src/app/updater/updaters/FestivalsUpdater.js index 7313970..8d1744e 100644 --- a/src/app/updater/updaters/FestivalsUpdater.js +++ b/src/app/updater/updaters/FestivalsUpdater.js @@ -7,32 +7,35 @@ import SplatNet from '../../../common/splatnet.js'; export default class FestivalsUpdater extends Updater { constructor(region, storage) { - super({ - name: `Festivals ${region}`, - filename: 'festivals.json', - calendarFilename: `festivals-${region.toLowerCase()}.ics`, - request: (splatnet) => splatnet.getCombinedFestivals(), - imagePaths: [ - '$..images.alpha', - '$..images.bravo', - '$..images.panel', - '$..special_stage.image', - ], - localization: [ - { - name: 'festivals', - entities: '$.festivals[*]', - id: 'festival_id', - values: 'names', - }, - { - name: 'stages', - entities: '$..special_stage', - id: 'id', - values: 'name', - }, - ], - }, storage); + super( + { + name: `Festivals ${region}`, + filename: 'festivals.json', + calendarFilename: `festivals-${region.toLowerCase()}.ics`, + request: splatnet => splatnet.getCombinedFestivals(), + imagePaths: [ + '$..images.alpha', + '$..images.bravo', + '$..images.panel', + '$..special_stage.image', + ], + localization: [ + { + name: 'festivals', + entities: '$.festivals[*]', + id: 'festival_id', + values: 'names', + }, + { + name: 'stages', + entities: '$..special_stage', + id: 'id', + values: 'name', + }, + ], + }, + storage, + ); this.region = region; } @@ -41,38 +44,46 @@ export default class FestivalsUpdater extends Updater { // Fix alpha/bravo images for the Chicken vs. Egg Splatfest. // For some reason these got swapped out with images that have an opaque background // even though they started out with transparent images. - jsonpath.apply(regionData, '$..images.alpha', value => value.replace( - '/images/festival/a070cc6b405b4fb335992d824097acd8.png', - '/images/festival/06b3b0b7773d9e6c4ac0a5cc5371fc32.png', - )); - jsonpath.apply(regionData, '$..images.bravo', value => value.replace( - '/images/festival/00e4c5fdccd3720d07127084fc1f4152.png', - '/images/festival/d93df77468714c6211e9377f39a559f4.png', - )); + jsonpath.apply(regionData, '$..images.alpha', value => + value.replace( + '/images/festival/a070cc6b405b4fb335992d824097acd8.png', + '/images/festival/06b3b0b7773d9e6c4ac0a5cc5371fc32.png', + ), + ); + jsonpath.apply(regionData, '$..images.bravo', value => + value.replace( + '/images/festival/00e4c5fdccd3720d07127084fc1f4152.png', + '/images/festival/d93df77468714c6211e9377f39a559f4.png', + ), + ); // Download result ranking data let festivalIds = jsonpath.query(regionData, '$.results..festival_id'); + for (let id of festivalIds) { let key = `data/festivals/${this.region.toLowerCase()}-${id}-rankings.json`; // Have we already downloaded these rankings? - if (!await this.publicStorage.exists(key)) { + if (!(await this.publicStorage.exists(key))) { let splatnet = new SplatNet(this.region); + this.info(`Retrieving rankings for festival ID ${id}`); + try { - let rankings = await this.handleRequest(splatnet.getFestivalRankings(id)); + let rankings = await this.handleRequest(splatnet.getFestivalRankings(id)); + await this.publicStorage.writeJson(key, rankings, { cacheControl: DATA_CACHE_CONTROL }); - } - catch { + } catch { // Do nothing } } } // Load existing data since we only need to modify this region's data - let data = await this.publicStorage.readJson(this.getKey()) ?? {}; + let data = (await this.publicStorage.readJson(this.getKey())) ?? {}; let region = this.region.toLowerCase(); + data[region] = regionData; return data; diff --git a/src/app/updater/updaters/Updater.js b/src/app/updater/updaters/Updater.js index da2875d..d9134de 100644 --- a/src/app/updater/updaters/Updater.js +++ b/src/app/updater/updaters/Updater.js @@ -63,6 +63,7 @@ export default class Updater { getData({ region, language }) { let splatnet = new SplatNet(region, language); + return this.options.request(splatnet); } @@ -73,8 +74,7 @@ export default class Updater { async handleRequest(request) { try { return await request; - } - catch (e) { + } catch (e) { // Send the error to Sentry Sentry.captureException(e); @@ -92,7 +92,8 @@ export default class Updater { for (let key of this.options.rootKeys) { let value = data[key]; - result[key] = (this.shouldIncludeRootValue(value)) ? value : null; + + result[key] = this.shouldIncludeRootValue(value) ? value : null; } return result; @@ -108,7 +109,9 @@ export default class Updater { } getProcessors(languageInfo) { - return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage)); + return this.options.localization.map( + ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage), + ); } async updateLocalizations(data, initialLanguageInfo) { @@ -119,9 +122,10 @@ export default class Updater { // Do we need to retrieve data for any other languages? let missingLanguages = []; + for (let languageInfo of this.getLanguages()) { for (let processor of this.getProcessors(languageInfo)) { - if (!await processor.hasLocalizations(data)) { + if (!(await processor.hasLocalizations(data))) { missingLanguages.push(languageInfo); break; } @@ -130,9 +134,13 @@ export default class Updater { // Retrieve data for missing languages for (let missingLanguageInfo of missingLanguages) { - this.info(`Retrieving localized data for region: ${missingLanguageInfo.region}, language: ${missingLanguageInfo.language}`); + this.info( + `Retrieving localized data for region: ${missingLanguageInfo.region}, language: ${missingLanguageInfo.language}`, + ); + let localData = await this.handleRequest(this.getData(missingLanguageInfo)); localData = this.filterRootKeys(localData); + for (let processor of this.getProcessors(missingLanguageInfo)) await processor.updateLocalizations(localData); } @@ -156,6 +164,7 @@ export default class Updater { if (this.options.imagePaths) { for (let expression of this.options.imagePaths) { let splatnetImages = jsonpath.query(data, expression); + for (let splatnetImage of splatnetImages) await this.maybeDownloadImage(splatnetImage); } @@ -174,6 +183,7 @@ export default class Updater { // Certain images are not available on the CDN anymore let backup = await cdnBackup(imagePath); + if (backup) { this.info(`Using CDN backup: ${imagePath}`); await this.publicStorage.writeBytes(key, backup); @@ -183,8 +193,10 @@ export default class Updater { // Download the image this.info(`Downloading image: ${imagePath}`); - let splatnet = new SplatNet; + + let splatnet = new SplatNet(); let image = await this.handleRequest(splatnet.getImage(imagePath)); + await this.publicStorage.writeBytes(key, image); } @@ -194,11 +206,13 @@ export default class Updater { async updateCalendarEvents(data) { let key = this.getCalendarKey(); + if (!key) return; let events = this.getCalendarEntries(data); let ical = this.getiCalData(events); + await this.publicStorage.writeText(key, ical, { cacheControl: DATA_CACHE_CONTROL }); } diff --git a/src/common/cdn.node.js b/src/common/cdn.node.js index 052b52f..63cc3a4 100644 --- a/src/common/cdn.node.js +++ b/src/common/cdn.node.js @@ -5,11 +5,15 @@ import { readFile } from 'node:fs/promises'; const images = { - '/images/skill/53c62995f9d2dc4a60f3850c5dbdd2323f1eef87.png': new URL('./cdn/images/skill/53c62995f9d2dc4a60f3850c5dbdd2323f1eef87.png', import.meta.url), + '/images/skill/53c62995f9d2dc4a60f3850c5dbdd2323f1eef87.png': new URL( + './cdn/images/skill/53c62995f9d2dc4a60f3850c5dbdd2323f1eef87.png', + import.meta.url, + ), }; /** @returns {Promise} the image bytes, or null if SplatNet still serves this path */ export async function cdnBackup(imagePath) { let file = images[imagePath]; + return file ? new Uint8Array(await readFile(file)) : null; } diff --git a/src/common/cdn.worker.js b/src/common/cdn.worker.js index 5f6ab67..eff3d56 100644 --- a/src/common/cdn.worker.js +++ b/src/common/cdn.worker.js @@ -11,5 +11,6 @@ const images = { /** @returns {Promise} the image bytes, or null if SplatNet still serves this path */ export async function cdnBackup(imagePath) { let data = images[imagePath]; + return data ? new Uint8Array(data) : null; } diff --git a/src/common/cli.js b/src/common/cli.js index f350d5a..3cfe27d 100644 --- a/src/common/cli.js +++ b/src/common/cli.js @@ -3,10 +3,12 @@ export async function runAction(actions, [name, ...args]) { if (!name || !Object.hasOwn(actions, name)) { console.error(`Usage: [args...]\nAvailable actions: ${Object.keys(actions).join(', ')}`); process.exitCode = 1; + return; } let result = await actions[name](...args); + if (result !== undefined) console.log(result); } diff --git a/src/common/dataRefresh.js b/src/common/dataRefresh.js index 6188546..8024b9a 100644 --- a/src/common/dataRefresh.js +++ b/src/common/dataRefresh.js @@ -3,7 +3,9 @@ export function nextDataRefreshAt(now = Date.now(), jitter = Math.floor(Math.random() * 35)) { let candidate = new Date(now); candidate.setSeconds(25 + jitter, 0); + while (candidate.getTime() <= now || (candidate.getMinutes() > 5 && candidate.getMinutes() % 5 !== 0)) candidate.setMinutes(candidate.getMinutes() + 1); + return candidate.getTime(); } diff --git a/src/common/fetch.js b/src/common/fetch.js index 6c00308..98a6bed 100644 --- a/src/common/fetch.js +++ b/src/common/fetch.js @@ -3,5 +3,6 @@ export function fetchWithTimeout(input, init = {}, timeoutMs = 30_000) { let timeout = AbortSignal.timeout(timeoutMs); let existing = init.signal ?? (input instanceof Request ? input.signal : null); let signal = existing ? AbortSignal.any([existing, timeout]) : timeout; + return fetch(input, { ...init, signal }); } diff --git a/src/common/screenshot.js b/src/common/screenshot.js index 30dd6ba..e8742cb 100644 --- a/src/common/screenshot.js +++ b/src/common/screenshot.js @@ -11,14 +11,18 @@ export async function markScreenshotReady({ requestAnimationFrame = globalThis.requestAnimationFrame, }) { document.documentElement.removeAttribute(screenshotReadyAttribute); + let results = await loadData(); + if (results.some(result => result.status === 'rejected')) throw new Error('Screenshot data failed to load'); + await nextTick(); await document.fonts?.ready; await Promise.allSettled([...document.images].map(image => image.decode())); await new Promise(resolve => requestAnimationFrame(resolve)); await new Promise(resolve => requestAnimationFrame(resolve)); + if (isCurrent()) document.documentElement.setAttribute(screenshotReadyAttribute, 'true'); } diff --git a/src/common/splatnet.js b/src/common/splatnet.js index 511de4e..6ac98fe 100644 --- a/src/common/splatnet.js +++ b/src/common/splatnet.js @@ -14,16 +14,19 @@ export default class SplatNet { getSessionId() { switch (this.region) { - case 'NA': return process.env.NINTENDO_SESSION_ID_NA; - case 'EU': return process.env.NINTENDO_SESSION_ID_EU; - case 'JP': return process.env.NINTENDO_SESSION_ID_JP; + case 'NA': + return process.env.NINTENDO_SESSION_ID_NA; + case 'EU': + return process.env.NINTENDO_SESSION_ID_EU; + case 'JP': + return process.env.NINTENDO_SESSION_ID_JP; } } getHeaders() { return { ...(userAgent() ? { 'User-Agent': userAgent() } : {}), - 'Cookie': `iksm_session=${this.getSessionId()}`, + Cookie: `iksm_session=${this.getSessionId()}`, 'Accept-Language': this.language, }; } @@ -32,18 +35,22 @@ export default class SplatNet { async request(path, { headers = this.getHeaders() } = {}) { let url = new URL(path, splatnetBaseUrl); let response = await fetchWithTimeout(url, { headers }); + if (!response.ok) throw new Error(`SplatNet request failed with status ${response.status}: ${url.pathname}`); + return response; } async getResponse(path) { let response = await this.request(`/api/${path}`); + return response.json(); } async getText(path) { let response = await this.request(path); + return response.text(); } @@ -107,7 +114,8 @@ export default class SplatNet { } getResults(id = null, region = 'NA') { - let url = (id) ? `results/${id}` : 'results'; + let url = id ? `results/${id}` : 'results'; + return this.getResponse(url); } @@ -116,6 +124,7 @@ export default class SplatNet { let response = await this.request(imagePath, { headers: userAgent() ? { 'User-Agent': userAgent() } : {}, }); + return new Uint8Array(await response.arrayBuffer()); } } diff --git a/src/common/storage/BucketStorage.js b/src/common/storage/BucketStorage.js index 30f188c..1b0d8da 100644 --- a/src/common/storage/BucketStorage.js +++ b/src/common/storage/BucketStorage.js @@ -15,8 +15,8 @@ function directoryOf(key) { export default class BucketStorage { #bucket; - #listings = new Map; // directory prefix -> Set of keys under it - #documents = new Map; // key -> { data, serialized } for JSON documents (data may be null) + #listings = new Map(); // directory prefix -> Set of keys under it + #documents = new Map(); // key -> { data, serialized } for JSON documents (data may be null) constructor(bucket) { this.#bucket = bucket; @@ -24,16 +24,21 @@ export default class BucketStorage { async #listing(directory) { if (!this.#listings.has(directory)) { - let keys = new Set; + let keys = new Set(); let cursor; + do { let page = await this.#bucket.list({ prefix: directory, cursor, limit: 1000 }); + for (let object of page.objects) keys.add(object.key); + cursor = page.truncated ? page.cursor : undefined; } while (cursor); + this.#listings.set(directory, keys); } + return this.#listings.get(directory); } @@ -50,26 +55,36 @@ export default class BucketStorage { if (!this.#documents.has(key)) { let object = await this.#bucket.get(key); let serialized = object ? await object.text() : null; - this.#documents.set(key, { data: serialized === null ? null : JSON.parse(serialized), serialized }); + + this.#documents.set(key, { + data: serialized === null ? null : JSON.parse(serialized), + serialized, + }); } + return this.#documents.get(key).data; } /** @returns {Promise} whether anything was written */ async writeJson(key, data, { cacheControl } = {}) { let serialized = JSON.stringify(data); + if (this.#documents.get(key)?.serialized === serialized) return false; - await this.#bucket.put(key, serialized, { httpMetadata: { contentType: 'application/json', cacheControl } }); + await this.#bucket.put(key, serialized, { + httpMetadata: { contentType: 'application/json', cacheControl }, + }); this.#documents.set(key, { data, serialized }); this.#noteWritten(key); + return true; } /** @returns {Promise} */ async readBytes(key) { let object = await this.#bucket.get(key); + return object ? new Uint8Array(await object.arrayBuffer()) : null; } diff --git a/src/common/storage/FilesystemStorage.js b/src/common/storage/FilesystemStorage.js index 817c4ad..7f214a3 100644 --- a/src/common/storage/FilesystemStorage.js +++ b/src/common/storage/FilesystemStorage.js @@ -20,6 +20,7 @@ export default class FilesystemStorage { } catch (error) { if (error.code === 'ENOENT') return false; + throw error; } } @@ -30,12 +31,14 @@ export default class FilesystemStorage { } catch (error) { if (error.code === 'ENOENT' || error.code === 'EISDIR') return null; + throw error; } } async #write(key, body) { let filename = this.pathFor(key); + await fs.mkdir(path.dirname(filename), { recursive: true }); await fs.writeFile(filename, body); } @@ -43,18 +46,21 @@ export default class FilesystemStorage { /** The parsed document, or null if it does not exist. */ async readJson(key) { let buffer = await this.#read(key); + return buffer === null ? null : JSON.parse(buffer.toString('utf8')); } /** @returns {Promise} always true; the filesystem write is cheap enough not to dedupe */ async writeJson(key, data) { await this.#write(key, JSON.stringify(data)); + return true; } /** @returns {Promise} */ async readBytes(key) { let buffer = await this.#read(key); + return buffer === null ? null : new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); } diff --git a/src/common/storage/MemoryBucket.js b/src/common/storage/MemoryBucket.js index dd9c42e..bad489f 100644 --- a/src/common/storage/MemoryBucket.js +++ b/src/common/storage/MemoryBucket.js @@ -3,26 +3,40 @@ function toBytes(value) { if (typeof value === 'string') return new TextEncoder().encode(value); + if (value instanceof ArrayBuffer) return new Uint8Array(value); + if (ArrayBuffer.isView(value)) return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + throw new TypeError('MemoryBucket.put: unsupported value type'); } export default class MemoryBucket { - objects = new Map; + objects = new Map(); async head(key) { let object = this.objects.get(key); - return object ? { key, size: object.bytes.byteLength, uploaded: object.uploaded, httpMetadata: object.httpMetadata } : null; + + return object + ? { + key, + size: object.bytes.byteLength, + uploaded: object.uploaded, + httpMetadata: object.httpMetadata, + } + : null; } async get(key) { let object = this.objects.get(key); + if (!object) return null; + let bytes = object.bytes; + return { key, size: bytes.byteLength, @@ -35,10 +49,13 @@ export default class MemoryBucket { } async put(key, value, options = {}) { - let bytes = value instanceof ReadableStream - ? new Uint8Array(await new Response(value).arrayBuffer()) - : toBytes(value); - this.objects.set(key, { bytes, uploaded: new Date, httpMetadata: options.httpMetadata ?? {} }); + let bytes = + value instanceof ReadableStream + ? new Uint8Array(await new Response(value).arrayBuffer()) + : toBytes(value); + + this.objects.set(key, { bytes, uploaded: new Date(), httpMetadata: options.httpMetadata ?? {} }); + return { key }; } @@ -52,8 +69,13 @@ export default class MemoryBucket { let start = cursor ? Number(cursor) : 0; let page = keys.slice(start, start + limit); let truncated = start + limit < keys.length; + return { - objects: page.map(key => ({ key, size: this.objects.get(key).bytes.byteLength, uploaded: this.objects.get(key).uploaded })), + objects: page.map(key => ({ + key, + size: this.objects.get(key).bytes.byteLength, + uploaded: this.objects.get(key).uploaded, + })), truncated, cursor: truncated ? String(start + limit) : undefined, }; diff --git a/src/common/storage/index.js b/src/common/storage/index.js index d8c7844..37cee00 100644 --- a/src/common/storage/index.js +++ b/src/common/storage/index.js @@ -19,5 +19,6 @@ const CONTENT_TYPES = { export function contentTypeFor(key) { let extension = key.split('.').pop().toLowerCase(); + return CONTENT_TYPES[extension]; } diff --git a/src/web/components/Screenshots.vue b/src/web/components/Screenshots.vue index dca619b..4ef509f 100644 --- a/src/web/components/Screenshots.vue +++ b/src/web/components/Screenshots.vue @@ -27,7 +27,9 @@ export default { ...mapActions('splatoon/data', ['updateAll']), async loadData() { let version = ++this.readinessVersion; + this.setNow({ now: this.$route.params.now || 0 }); + try { await markScreenshotReady({ loadData: () => this.updateAll(), diff --git a/src/web/store/splatoon/data.js b/src/web/store/splatoon/data.js index eadad73..0a2ae9d 100644 --- a/src/web/store/splatoon/data.js +++ b/src/web/store/splatoon/data.js @@ -42,17 +42,23 @@ for (let source of dataSources) { source.mutationName = 'UPDATE_' + source.name.toUpperCase(); } -export const state = { }; +export const state = {}; export const actions = { updateLanguage({ dispatch, rootGetters }) { let language = rootGetters['splatoon/languages/selectedLanguage']; + if (language) { - return fetchJson(`/data/locale/${language.language}.json`) - .then(data => dispatch('i18n/addLocale', { - locale: language.language, - translations: { splatnet: data }, - }, { root: true })); + return fetchJson(`/data/locale/${language.language}.json`).then(data => + dispatch( + 'i18n/addLocale', + { + locale: language.language, + translations: { splatnet: data }, + }, + { root: true }, + ), + ); } }, updateAll({ dispatch }) { @@ -64,7 +70,9 @@ export const actions = { startUpdatingData({ dispatch }) { if (updatingData) return; + updatingData = true; + let generation = ++refreshGeneration; let refreshing = false; let lastRefresh = -Infinity; @@ -72,27 +80,33 @@ export const actions = { async function refresh() { if (!updatingData || generation !== refreshGeneration || refreshing) return; + refreshing = true; lastRefresh = Date.now(); clearTimeout(updateDataTimer); updateDataTimer = null; + try { await dispatch('updateAll'); } catch (error) { console.error('Could not refresh site data', error); } finally { refreshing = false; + if (updatingData && generation === refreshGeneration) updateDataTimer = setTimeout(refresh, nextDataRefreshAt() - Date.now()); } } + function onVisibilityChange() { // Match Splatoon3's activation cooldown, without delaying the hourly retries. if (document.visibilityState === 'visible' && Date.now() - lastRefresh >= 60_000) void refresh(); } + document.addEventListener('visibilitychange', onVisibilityChange); removeVisibilityListener = () => document.removeEventListener('visibilitychange', onVisibilityChange); + return refresh(); }, stopUpdatingData() { @@ -104,7 +118,7 @@ export const actions = { }, }; -export const mutations = { }; +export const mutations = {}; for (let source of dataSources) { // State @@ -112,8 +126,7 @@ for (let source of dataSources) { // Actions actions[source.actionName] = async ({ commit }) => { - return fetchJson(source.url) - .then(data => commit(source.mutationName, { data })); + return fetchJson(source.url).then(data => commit(source.mutationName, { data })); }; // Mutations @@ -125,12 +138,15 @@ for (let source of dataSources) { async function fetchJson(url) { // Use AbortController rather than newer AbortSignal helpers in the older frontend. // Keep the deadline active through body parsing so a stalled response cannot stop polling. - let controller = new AbortController; + let controller = new AbortController(); let timeout = setTimeout(() => controller.abort(), 30_000); + try { let response = await fetch(url, { signal: controller.signal }); + if (!response.ok) throw new Error(`Data request failed: ${response.status}`); + return await response.json(); } finally { clearTimeout(timeout); diff --git a/workers/updater/preview/server.mjs b/workers/updater/preview/server.mjs index 48934aa..004d764 100644 --- a/workers/updater/preview/server.mjs +++ b/workers/updater/preview/server.mjs @@ -42,16 +42,21 @@ const state = { social: { ok: true }, }, }; + createServer(async (request, response) => { response.setHeader('Cache-Control', 'no-store'); + const url = new URL(request.url, `http://127.0.0.1:${port}`); + function json(body, status = 200) { response.writeHead(status, { 'Content-Type': 'application/json' }); response.end(JSON.stringify(body)); } + try { if (request.method === 'GET' && ['/', '/admin', '/admin/'].includes(url.pathname)) { response.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + return response.end( (await readFile(new URL('../src/admin/page.html', import.meta.url), 'utf8')).replaceAll( '__NONCE__', @@ -59,25 +64,41 @@ createServer(async (request, response) => { ), ); } - if (request.method === 'GET' && url.pathname === '/admin/api/status') return json(state); + + if (request.method === 'GET' && url.pathname === '/admin/api/status') + return json(state); + if (request.method === 'POST' && url.pathname === '/admin/api/run') { if (request.headers.origin !== `http://${request.headers.host}`) return json({ error: 'Invalid origin.' }, 403); - if (state.busy) return json({ error: 'A run is already active.' }, 409); + + if (state.busy) + return json({ error: 'A run is already active.' }, 409); + let body = ''; + for await (const chunk of request) { body += chunk; - if (body.length > 1024) return json({ error: 'Request too large.' }, 413); + + if (body.length > 1024) + return json({ error: 'Request too large.' }, 413); } + const { mode } = JSON.parse(body); - if (!['data', 'social', 'both'].includes(mode)) return json({ error: 'Unknown mode.' }, 400); + + if (!['data', 'social', 'both'].includes(mode)) + return json({ error: 'Unknown mode.' }, 400); + const run = { id: randomUUID(), mode, status: 'queued', requestedAt: Date.now() }; + state.busy = true; state.pendingManual = run; + const logs = { lines: [{ at: Date.now(), level: 'info', text: `Starting ${mode} run` }], omitted: 0, }; + state.activeRun = { mode, startedAt: Date.now(), logs }; setTimeout( () => @@ -97,9 +118,7 @@ createServer(async (request, response) => { at: Date.now(), level: 'info', text: - mode === 'social' - ? '[Social] Preparing a due Bluesky post…' - : '[Updater] [Schedules] Done.', + mode === 'social' ? '[Social] Preparing a due Bluesky post…' : '[Updater] [Schedules] Done.', }), 3200, ); @@ -128,11 +147,12 @@ createServer(async (request, response) => { mode === 'social' ? { ok: true, skipped: true } : { + ok: true, + updaters: ['Schedules', 'Timeline', 'CoopSchedules', 'Merchandises'].map(name => ({ + name, ok: true, - updaters: ['Schedules', 'Timeline', 'CoopSchedules', 'Merchandises'].map( - (name) => ({ name, ok: true }), - ), - }, + })), + }, social: mode === 'data' ? { ok: true, skipped: true } @@ -142,8 +162,10 @@ createServer(async (request, response) => { state.pendingManual = null; state.activeRun = null; }, 6000); + return json({ ok: true, run }, 202); } + json({ error: 'Not found.' }, 404); } catch { json({ error: 'Invalid preview request.' }, 400); diff --git a/workers/updater/src/Scheduler.mjs b/workers/updater/src/Scheduler.mjs index 944ee0c..1c5b6cf 100644 --- a/workers/updater/src/Scheduler.mjs +++ b/workers/updater/src/Scheduler.mjs @@ -1,6 +1,7 @@ -import { RunLog, logMessage } from '../../../src/app/log.js'; // One owner for the hourly update → social pipeline and authenticated manual runs. // The alarm targets :00:10; the cron watchdog repairs a missing alarm. Alarms can be late. + +import { RunLog, logMessage } from '../../../src/app/log.js'; import { DurableObject } from 'cloudflare:workers'; import { runUpdaters } from './updaters.mjs'; import { runPosters } from './posters.mjs'; @@ -19,7 +20,8 @@ export class Scheduler extends DurableObject { #activeRun = null; async #state() { - let saved = await this.ctx.storage.get('state') ?? {}; + let saved = (await this.ctx.storage.get('state')) ?? {}; + // Retain the existing hourly/retry schedule across deployment. Any old pending wake // becomes one immediate full run, then the generic queue is retired. return { @@ -32,48 +34,60 @@ export class Scheduler extends DurableObject { } async #pendingManual() { - return await this.ctx.storage.get('pendingManual') ?? null; + return (await this.ctx.storage.get('pendingManual')) ?? null; } async ensureArmed() { if (this.#running) return { armed: false, busy: true }; + let state = await this.#state(); + if (state.paused) return { armed: false, paused: true }; - let armed = await this.ctx.storage.getAlarm() === null; + + let armed = (await this.ctx.storage.getAlarm()) === null; + state.hourlyAt ??= nextRunAt(); await this.ctx.storage.put('state', state); - let alarmAt = await this.#pendingManual() ? Date.now() : state.retryAt ?? state.hourlyAt; + + let alarmAt = (await this.#pendingManual()) ? Date.now() : (state.retryAt ?? state.hourlyAt); + await this.ctx.storage.setAlarm(alarmAt); + return { armed, hourlyAt: state.hourlyAt, alarmAt }; } async pause() { - if (this.#running || await this.#pendingManual()) + if (this.#running || (await this.#pendingManual())) return { ok: false, busy: true, error: 'Wait for the current run to finish before pausing.' }; + let state = await this.#state(); state.paused = true; await this.ctx.storage.put('state', state); await this.ctx.storage.deleteAlarm(); + return { ok: true, paused: true }; } async resume() { if (this.#running) return { ok: false, busy: true }; + let state = await this.#state(); state.paused = false; await this.ctx.storage.put('state', state); - return { ok: true, ...await this.ensureArmed() }; + + return { ok: true, ...(await this.ensureArmed()) }; } async status() { let pendingManual = await this.#pendingManual(); + return { - ...await this.#state(), + ...(await this.#state()), alarmAt: await this.ctx.storage.getAlarm(), - lastManualRun: await this.ctx.storage.get('lastManualRun') ?? null, + lastManualRun: (await this.ctx.storage.get('lastManualRun')) ?? null, pendingManual, activeRun: this.#activeRun, busy: this.#running || !!pendingManual, @@ -84,19 +98,26 @@ export class Scheduler extends DurableObject { async startManual(mode) { if (!MANUAL_MODES.includes(mode)) return { ok: false, error: 'Unknown run mode.' }; + if (this.#running) return { ok: false, busy: true, error: 'A run is already active.' }; + this.#running = true; + try { if (await this.#pendingManual()) return { ok: false, busy: true, error: 'A manual run is already active.' }; + if ((await this.#state()).paused) return { ok: false, paused: true, error: 'Scheduler is paused.' }; + let run = { id: crypto.randomUUID(), mode, status: 'queued', requestedAt: Date.now() }; + await this.ctx.storage.transaction(async txn => { await txn.put('pendingManual', run); await txn.setAlarm(Date.now()); }); + return { ok: true, run }; } finally { this.#running = false; @@ -108,14 +129,20 @@ export class Scheduler extends DurableObject { async run({ only } = {}) { if (this.#running) return { ok: false, busy: true, error: 'An update is already running; retry later.' }; + this.#running = true; + try { if (await this.#pendingManual()) return { ok: false, busy: true, error: 'A manual run is already active.' }; + if ((await this.#state()).paused) return { ok: false, paused: true, error: 'Scheduler is paused; use /arm to resume.' }; + let result = await this.#execute(only); + await this.ctx.storage.put('lastManualRun', result); + return result; } finally { this.#running = false; @@ -128,11 +155,18 @@ export class Scheduler extends DurableObject { .filter(([key]) => /TOKEN|PASSWORD|SESSION|ACCOUNT_ID|SECRET/.test(key)) .map(([, value]) => value); const capture = new RunLog(secrets); + this.#activeRun = { mode, startedAt: Date.now(), logs: capture.snapshot }; + try { return await capture.run(async () => { - logMessage('info', `Starting ${mode === 'both' ? 'full cycle' : mode === 'data' ? 'data update' : 'social cycle'}`); + logMessage( + 'info', + `Starting ${mode === 'both' ? 'full cycle' : mode === 'data' ? 'data update' : 'social cycle'}`, + ); + const result = await this.#executeWithLogs(only, mode); + return { ...result, logs: capture.snapshot }; }); } finally { @@ -143,12 +177,12 @@ export class Scheduler extends DurableObject { async #executeWithLogs(only, mode) { let startedAt = Date.now(); let result; + try { - let updaters = mode === 'social' - ? { ok: true, skipped: true } - : await runUpdaters(this.env, { only }); + let updaters = mode === 'social' ? { ok: true, skipped: true } : await runUpdaters(this.env, { only }); // A targeted repair does not publish social posts from a partially refreshed dataset. let social; + if (mode === 'data') { social = { ok: true, skipped: true, reason: 'data-only' }; } else if (only) { @@ -158,12 +192,15 @@ export class Scheduler extends DurableObject { } else { social = await runPosters(this.env); } + result = { ok: updaters.ok && social.ok, updaters, social }; } catch (error) { result = { ok: false, ...describeError(error) }; } + result = { ...result, mode, startedAt, runMs: Date.now() - startedAt }; createLogger('pipeline')[result.ok ? 'info' : 'error']('Run finished', result); + return result; } @@ -171,42 +208,67 @@ export class Scheduler extends DurableObject { if (this.#running) { // A manual run must not make us skip this hour. Leave the original due time intact. await this.ctx.storage.setAlarm(Date.now() + RETRY_DELAY_MS); + return; } + this.#running = true; + try { let state = await this.#state(); + if (state.paused) { await this.ctx.storage.deleteAlarm(); + return; } + let manual = await this.#pendingManual(); + if (manual) { let result; + if (manual.status === 'running') { // After an isolate interruption, do not automatically replay an uncertain social // send. Show the interruption and let the operator retry with normal checkpoints. - result = { ok: false, error: 'Run interrupted. Review the result before retrying.', startedAt: manual.startedAt, runMs: Date.now() - manual.startedAt }; + result = { + ok: false, + error: 'Run interrupted. Review the result before retrying.', + startedAt: manual.startedAt, + runMs: Date.now() - manual.startedAt, + }; } else { manual = { ...manual, status: 'running', startedAt: Date.now() }; await this.ctx.storage.put('pendingManual', manual); result = await this.#execute(undefined, manual.mode); } + await this.ctx.storage.transaction(async txn => { - await txn.put('lastManualRun', { ...manual, ...result, status: result.ok ? 'succeeded' : 'failed', finishedAt: Date.now() }); + await txn.put('lastManualRun', { + ...manual, + ...result, + status: result.ok ? 'succeeded' : 'failed', + finishedAt: Date.now(), + }); await txn.delete('pendingManual'); }); } + state.hourlyAt ??= nextRunAt(); + let scheduledFor = state.retryAt ?? state.hourlyAt; + if (Date.now() >= scheduledFor) { let result = await this.#execute(); + state.lastRun = { - ...result, scheduledFor, + ...result, + scheduledFor, driftMs: result.startedAt - scheduledFor, retries: state.retries, retryCount: alarmInfo?.retryCount ?? 0, }; + if (result.ok || state.retries >= MAX_RETRIES) { state.hourlyAt = nextRunAt(); state.retryAt = null; @@ -216,6 +278,7 @@ export class Scheduler extends DurableObject { state.retries++; } } + // Storage failures escape so the platform retries. Schedule and state are saved // together without external I/O in between. await this.ctx.storage.put('state', state); diff --git a/workers/updater/src/admin/access.mjs b/workers/updater/src/admin/access.mjs index d5fc84f..b58177b 100644 --- a/workers/updater/src/admin/access.mjs +++ b/workers/updater/src/admin/access.mjs @@ -4,27 +4,38 @@ let keySets = new Map(); export async function verifyAccess(request, env, resolveKey) { let domain = env.ACCESS_TEAM_DOMAIN; + if (!domain || !env.ACCESS_AUD || new URL(request.url).hostname !== env.ADMIN_HOSTNAME) return null; + let token = request.headers.get('Cf-Access-Jwt-Assertion'); - if (!token) return null; + + if (!token) + return null; + try { let issuer = new URL(`https://${domain}`); - if (issuer.hostname !== domain || !domain.endsWith('.cloudflareaccess.com')) return null; + + if (issuer.hostname !== domain || !domain.endsWith('.cloudflareaccess.com')) + return null; + if (!resolveKey) { if (!keySets.has(domain)) keySets.set( domain, createRemoteJWKSet(new URL('/cdn-cgi/access/certs', issuer), { timeoutDuration: 5000 }), ); + resolveKey = keySets.get(domain); } + let { payload } = await jwtVerify(token, resolveKey, { issuer: issuer.origin, audience: env.ACCESS_AUD, algorithms: ['RS256'], requiredClaims: ['exp', 'sub'], }); + return { email: payload.email ?? 'Authenticated administrator' }; } catch { return null; diff --git a/workers/updater/src/admin/page.html b/workers/updater/src/admin/page.html index 9835e07..db82e61 100644 --- a/workers/updater/src/admin/page.html +++ b/workers/updater/src/admin/page.html @@ -517,11 +517,11 @@ function formatDate(value) { return value ? new Date(value).toLocaleString([], { - month: 'short', - day: 'numeric', - hour: 'numeric', - minute: '2-digit', - }) + month: 'short', + day: 'numeric', + hour: 'numeric', + minute: '2-digit', + }) : '—'; } @@ -533,50 +533,61 @@ function renderLogLines(logs) { const box = document.createElement('div'); box.className = 'log-view'; + if (!logs?.lines?.length) { box.textContent = 'No log lines recorded for this run.'; + return box; } + if (logs.omitted) { const note = document.createElement('div'); note.className = 'log-note'; note.textContent = `${logs.omitted} earlier lines omitted. Showing the most recent log lines.`; box.append(note); } + for (const line of logs.lines) { const row = document.createElement('div'); row.className = 'log-line ' + line.level; + const time = document.createElement('span'); time.className = 'log-time'; time.textContent = new Date(line.at).toLocaleTimeString() + ' '; row.append(time, document.createTextNode(line.text)); box.append(row); } + return box; } function renderRunRecord(label, run) { const row = document.createElement('div'); row.className = 'record'; + const top = document.createElement('div'); top.className = 'record-top'; + const title = document.createElement('span'); title.className = 'record-name'; title.textContent = label; + const badge = document.createElement('span'); badge.className = 'badge' + (!run ? ' neutral' : run.ok ? '' : ' failed'); badge.textContent = !run ? 'No runs yet' : run.ok ? 'Completed' : 'Needs attention'; top.append(title, badge); row.append(top); + const detail = document.createElement('div'); detail.className = 'detail'; detail.textContent = run ? `${runNames[run.mode] || 'Full cycle'} · ${formatDate(run.startedAt)} · ${Math.max( - 1, - Math.round((run.runMs || 0) / 1000), - )}s` + 1, + Math.round((run.runMs || 0) / 1000), + )}s` : 'Results will appear here after a run.'; row.append(detail); + if (run) { const messages = [ run.error, @@ -587,32 +598,38 @@ .filter((item) => item && !item.ok) .map((item) => item.error || 'A social post failed'), ].filter(Boolean); + if (messages.length) { const error = document.createElement('div'); error.className = 'error'; error.textContent = messages.join(' · '); row.append(error); } + const logDetails = document.createElement('details'); const logSummary = document.createElement('summary'); logSummary.textContent = 'View log lines'; logDetails.append(logSummary, renderLogLines(run.logs)); row.append(logDetails); + const details = document.createElement('details'); const summary = document.createElement('summary'); const pre = document.createElement('pre'); summary.textContent = 'View run details'; + const summaryResult = { ...run }; delete summaryResult.logs; pre.textContent = JSON.stringify(summaryResult, null, 2); details.append(summary, pre); row.append(details); } + return row; } function render() { element('preview').style.display = state.preview ? 'block' : 'none'; + if (state.busy) { element('state').textContent = 'Run in progress'; element('dot').className = 'dot busy'; @@ -626,13 +643,16 @@ element('dot').className = 'dot'; element('state-detail').textContent = 'The hourly schedule is running automatically.'; } + if (state.pendingManual) { const name = runNames[state.pendingManual.mode]; const progress = state.pendingManual.status === 'queued' ? 'waiting to start' : 'running'; element('state-detail').textContent = `${name} · ${progress}`; } + element('next').textContent = state.paused ? 'Paused' : formatDate(state.retryAt || state.hourlyAt); element('live-run').hidden = !state.activeRun; + if (state.activeRun) { const old = element('live-log').firstElementChild; const follow = !old || old.scrollHeight - old.scrollTop - old.clientHeight < 30; @@ -641,8 +661,10 @@ element('live-log').replaceChildren(box); box.scrollTop = follow ? box.scrollHeight : position; } + // Keep expanded details open during polling when the records have not changed. const signature = JSON.stringify([state.lastManualRun, state.lastRun]); + if (element('history').dataset.signature !== signature) { element('history').replaceChildren( renderRunRecord('Last manual run', state.lastManualRun), @@ -650,12 +672,14 @@ ); element('history').dataset.signature = signature; } + if (lastRunId && state.lastManualRun?.id === lastRunId) { element('notice').textContent = state.lastManualRun.ok ? 'Your run finished successfully.' : 'Your run needs attention. See the result below before retrying.'; lastRunId = null; } + element('identity').textContent = state.preview ? 'Preview session · Simulated results' : `Signed in as ${state.user.email}`; @@ -673,21 +697,31 @@ redirect: 'error', signal: AbortSignal.timeout(15000), }); + if (response.status === 401 || response.status === 403) { element('sign-in').style.display = 'inline-block'; throw new Error('Your session may have expired. Sign in again to continue.'); } + if (!response.headers.get('content-type')?.includes('application/json')) throw new Error('Could not read status. Reload to sign in again.'); + const result = await response.json(); - if (!response.ok) throw new Error(result.error || 'The request failed.'); + + if (!response.ok) + throw new Error(result.error || 'The request failed.'); + return result; } async function refreshStatus() { clearTimeout(timer); - if (polling) return; + + if (polling) + return; + polling = true; + try { state = await requestAdmin('status'); render(); @@ -700,14 +734,18 @@ element('notice').textContent = error.message; } finally { polling = false; - if (!document.hidden) timer = setTimeout(refreshStatus, state?.busy ? 2000 : 15000); + + if (!document.hidden) + timer = setTimeout(refreshStatus, state?.busy ? 2000 : 15000); } } + for (const button of buttons) button.addEventListener('click', async () => { sending = true; updateControls(); element('notice').textContent = 'Starting your run…'; + try { const result = await requestAdmin('run', { method: 'POST', @@ -715,11 +753,13 @@ body: JSON.stringify({ mode: button.dataset.mode }), }); lastRunId = result.run.id; + if (state) { state.busy = true; state.pendingManual = result.run; render(); } + element('notice').textContent = 'Run accepted. You can close this page and check back later.'; } catch (error) { element( @@ -731,10 +771,13 @@ updateControls(); } }); + element('refresh').addEventListener('click', refreshStatus); document.addEventListener('visibilitychange', () => { - if (document.hidden) clearTimeout(timer); - else refreshStatus(); + if (document.hidden) + clearTimeout(timer); + else + refreshStatus(); }); refreshStatus(); diff --git a/workers/updater/src/admin/routes.mjs b/workers/updater/src/admin/routes.mjs index beb689e..404d54c 100644 --- a/workers/updater/src/admin/routes.mjs +++ b/workers/updater/src/admin/routes.mjs @@ -7,10 +7,15 @@ const json = (body, status = 200) => Response.json(body, { status, headers }); export async function adminRequest(request, env, getScheduler) { let user = await verifyAccess(request, env); - if (!user) return json({ error: 'Sign in through Cloudflare Access to continue.' }, 401); + + if (!user) + return json({ error: 'Sign in through Cloudflare Access to continue.' }, 401); + let url = new URL(request.url); + if (request.method === 'GET' && ['/', '/admin', '/admin/'].includes(url.pathname)) { let nonce = crypto.randomUUID(); + return new Response(page.replaceAll('__NONCE__', nonce), { headers: { ...headers, @@ -20,8 +25,10 @@ export async function adminRequest(request, env, getScheduler) { }, }); } + if (request.method === 'GET' && url.pathname === '/admin/api/status') return json({ ...(await getScheduler().status()), user, preview: false }); + if (request.method === 'POST' && url.pathname === '/admin/api/run') { // Access cookies authenticate the user; require a same-origin JSON request as well. if ( @@ -29,15 +36,22 @@ export async function adminRequest(request, env, getScheduler) { request.headers.get('Content-Type') !== 'application/json' ) return json({ error: 'A same-origin JSON request is required.' }, 403); + let mode; + try { ({ mode } = await request.json()); } catch { return json({ error: 'Invalid request.' }, 400); } - if (!MANUAL_MODES.includes(mode)) return json({ error: 'Unknown run mode.' }, 400); + + if (!MANUAL_MODES.includes(mode)) + return json({ error: 'Unknown run mode.' }, 400); + let result = await getScheduler().startManual(mode); + return json(result, result.ok ? 202 : result.busy || result.paused ? 409 : 400); } + return json({ error: 'Not found.' }, 404); } diff --git a/workers/updater/src/index.mjs b/workers/updater/src/index.mjs index e9ed035..de65e04 100644 --- a/workers/updater/src/index.mjs +++ b/workers/updater/src/index.mjs @@ -15,25 +15,30 @@ function scheduler(env) { } function timingSafeEqual(a, b) { - let encoder = new TextEncoder; + let encoder = new TextEncoder(); let left = encoder.encode(a); let right = encoder.encode(b); + if (left.byteLength !== right.byteLength) return false; + return crypto.subtle.timingSafeEqual(left, right); } function isAuthorized(request, env) { if (!env.RUN_TOKEN) return false; + let header = request.headers.get('Authorization') ?? ''; let [scheme, token] = header.split(' '); + return scheme === 'Bearer' && !!token && timingSafeEqual(token, env.RUN_TOKEN); } export default withSentry(sentryOptions, { async scheduled(controller, env, ctx) { let log = createLogger('cron'); + try { log.info('Cron action finished', { cron: controller.cron, result: await scheduler(env).ensureArmed() }); } catch (error) { @@ -45,45 +50,79 @@ export default withSentry(sentryOptions, { // Manual runs use the same owner as alarms. Targeted runs refresh data only. async fetch(request, env, ctx) { let url = new URL(request.url); - if ((url.hostname === env.ADMIN_HOSTNAME && url.pathname === '/') || - url.pathname === '/admin' || url.pathname.startsWith('/admin/')) { + + if ( + (url.hostname === env.ADMIN_HOSTNAME && url.pathname === '/') || + url.pathname === '/admin' || + url.pathname.startsWith('/admin/') + ) { try { return await adminRequest(request, env, () => scheduler(env)); } catch (error) { createLogger('admin').error('Admin request failed', describeError(error)); - return Response.json({ error: 'The request failed. Refresh status before retrying.' }, { status: 500, headers: { 'Cache-Control': 'no-store' } }); + + return Response.json( + { error: 'The request failed. Refresh status before retrying.' }, + { status: 500, headers: { 'Cache-Control': 'no-store' } }, + ); } } + let route = `${request.method} ${url.pathname}`; + if (!['POST /run', 'POST /arm', 'POST /pause', 'GET /status', 'GET /list'].includes(route)) return new Response('Not found', { status: 404 }); + if (!isAuthorized(request, env)) return new Response('Unauthorized', { status: 401 }); try { switch (route) { case 'POST /run': { - let only = url.searchParams.get('only')?.split(',').map(name => name.trim()).filter(Boolean); + let only = url.searchParams + .get('only') + ?.split(',') + .map(name => name.trim()) + .filter(Boolean); let result = await scheduler(env).run({ only }); - return Response.json(result, { status: result.busy || result.paused ? 409 : result.ok ? 200 : 502 }); + + return Response.json(result, { + status: result.busy || result.paused ? 409 : result.ok ? 200 : 502, + }); } + case 'POST /arm': { let result = await scheduler(env).resume(); + return Response.json(result, { status: result.busy ? 409 : 200 }); } + case 'POST /pause': { let result = await scheduler(env).pause(); + return Response.json(result, { status: result.busy ? 409 : 200 }); } + case 'GET /status': - return Response.json({ ok: true, ...await scheduler(env).status() }); + return Response.json({ ok: true, ...(await scheduler(env).status()) }); + case 'GET /list': { let listing = await env.ASSETS.list({ prefix: url.searchParams.get('prefix') ?? '', limit: 1000 }); - return Response.json({ ok: true, truncated: listing.truncated, keys: listing.objects.map(object => ({ key: object.key, size: object.size, uploaded: object.uploaded })) }); + + return Response.json({ + ok: true, + truncated: listing.truncated, + keys: listing.objects.map(object => ({ + key: object.key, + size: object.size, + uploaded: object.uploaded, + })), + }); } } } catch (error) { createLogger('http').error(`${route} failed`, describeError(error)); + return Response.json({ ok: false, ...describeError(error) }, { status: 500 }); } }, diff --git a/workers/updater/src/log.mjs b/workers/updater/src/log.mjs index a4f4fa4..c264819 100644 --- a/workers/updater/src/log.mjs +++ b/workers/updater/src/log.mjs @@ -1,4 +1,5 @@ import { logMessage } from '../../../src/app/log.js'; + // Structured logging for Workers Logs. Objects (not JSON strings) are logged so the // dashboard extracts and indexes each field, e.g. filter on driftMs or updater. export function createLogger(updater) { @@ -14,7 +15,5 @@ export function createLogger(updater) { } export function describeError(error) { - return error instanceof Error - ? { error: error.message, stack: error.stack } - : { error: String(error) }; + return error instanceof Error ? { error: error.message, stack: error.stack } : { error: String(error) }; } diff --git a/workers/updater/src/posters.mjs b/workers/updater/src/posters.mjs index 01d0870..54dc74f 100644 --- a/workers/updater/src/posters.mjs +++ b/workers/updater/src/posters.mjs @@ -12,14 +12,25 @@ import { createLogger } from './log.mjs'; export async function verifyPublishedData(publicStorage) { if (!process.env.SITE_URL) throw new Error('SITE_URL must point to a site serving this Worker\'s published data.'); - for (let filename of ['schedules.json', 'coop-schedules.json', 'timeline.json', 'festivals.json', 'merchandises.json', 'locale/en.json']) { + + for (let filename of [ + 'schedules.json', + 'coop-schedules.json', + 'timeline.json', + 'festivals.json', + 'merchandises.json', + 'locale/en.json', + ]) { let key = `data/${filename}`; let expected = await publicStorage.readJson(key); + if (!expected) throw new Error(`Missing published data: ${key}`); + let response = await fetchWithTimeout(new URL(`/${key}`, process.env.SITE_URL), { headers: { 'Cache-Control': 'no-cache' }, }); + if (!response.ok || stringify(await response.json()) !== stringify(expected)) throw new Error(`Rendering site data does not match this Worker: ${key}`); } @@ -28,18 +39,27 @@ export async function verifyPublishedData(publicStorage) { export async function runPosters(env) { let started = Date.now(); let storage = bucketStorage(env); + await verifyPublishedData(storage.publicStorage); + let clients = createClients(); let enabled = []; + for (let client of clients) if (await client.canSend()) enabled.push(client.key); - let screenshots = new ScreenshotGenerator(new BrowserRunClient({ - accountId: process.env.CLOUDFLARE_ACCOUNT_ID, - apiToken: process.env.CLOUDFLARE_BROWSER_RUN_API_TOKEN, - }), process.env.SITE_URL); + + let screenshots = new ScreenshotGenerator( + new BrowserRunClient({ + accountId: process.env.CLOUDFLARE_ACCOUNT_ID, + apiToken: process.env.CLOUDFLARE_BROWSER_RUN_API_TOKEN, + }), + process.env.SITE_URL, + ); let result = await sendStatuses(storage, clients, screenshots); let summary = { ...result, ms: Date.now() - started, clients: enabled }; + createLogger('social')[summary.ok ? 'info' : 'error']('Social run finished', summary); + return summary; } diff --git a/workers/updater/src/schedule.mjs b/workers/updater/src/schedule.mjs index 0434f3b..3a01ae1 100644 --- a/workers/updater/src/schedule.mjs +++ b/workers/updater/src/schedule.mjs @@ -7,5 +7,6 @@ export const GRACE_MS = 10 * 1000; /** The :00:10 strictly after `now`. */ export function nextRunAt(now = Date.now()) { let candidate = now - (now % HOUR_MS) + GRACE_MS; + return candidate > now ? candidate : candidate + HOUR_MS; } diff --git a/workers/updater/src/updaters.mjs b/workers/updater/src/updaters.mjs index 5a8f2ec..38e2120 100644 --- a/workers/updater/src/updaters.mjs +++ b/workers/updater/src/updaters.mjs @@ -22,6 +22,8 @@ export async function runUpdaters(env, { only } = {}) { let started = Date.now(); let updaters = await updateAll(bucketStorage(env), { only }); let summary = { ok: updaters.every(u => u.ok), ms: Date.now() - started, updaters }; + log[summary.ok ? 'info' : 'error']('Updaters finished', summary); + return summary; }