From 49ee02731ce1adb102e5708b99276ac60a4dfa75 Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sat, 5 Sep 2026 18:04:55 -0700 Subject: [PATCH] Return a run summary from each updater Updater.update() now returns what happened: per-phase timings (fetch, localize, process, publish, images), which languages had to be fetched, which locale documents changed, and how many images were downloaded. Subclasses add their own details (stages seeded and new stages, festival rankings fetched). updateAll() carries each summary in its results, so the Worker's "Updaters finished" log entry contains the whole picture per updater. FilesystemStorage.writeJson now skips unchanged content like BucketStorage does, so "locale documents written" means the same locally. Co-Authored-By: Claude Fable 5.1 --- src/app/updater/LocalizationProcessor.js | 11 +++-- src/app/updater/updateAll.js | 8 ++-- src/app/updater/updaters/FestivalsUpdater.js | 2 + .../updaters/OriginalGearImageUpdater.js | 5 +++ src/app/updater/updaters/SchedulesUpdater.js | 3 ++ src/app/updater/updaters/Updater.js | 43 ++++++++++++++++++- src/common/storage/FilesystemStorage.js | 8 +++- test/storage/storage-contract.test.js | 5 ++- test/updater/Updater.test.js | 13 +++++- test/updater/updaters.test.js | 10 +++-- workers/updater/updaters.spec.mjs | 2 + 11 files changed, 94 insertions(+), 16 deletions(-) diff --git a/src/app/updater/LocalizationProcessor.js b/src/app/updater/LocalizationProcessor.js index 3d81935..783a295 100644 --- a/src/app/updater/LocalizationProcessor.js +++ b/src/app/updater/LocalizationProcessor.js @@ -7,11 +7,13 @@ export default class LocalizationProcessor { * @param {object} ruleset { name, entities, id, values } * @param {{ region: string, language: string }} languageInfo * @param {object} storage the public storage (BucketStorage or FilesystemStorage) + * @param {object} [summary] the updater's run summary; written languages are recorded in it */ - constructor(ruleset, languageInfo, storage) { + constructor(ruleset, languageInfo, storage, summary = null) { this.ruleset = ruleset; this.languageInfo = languageInfo; this.storage = storage; + this.summary = summary; let entities = this.ruleset.entities; this.entityExpressions = (Array.isArray(entities)) ? entities : [entities]; @@ -28,8 +30,11 @@ export default class LocalizationProcessor { return await this.storage.readJson(this.getKey()) ?? {}; } - writeData(data) { - return this.storage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL }); + async writeData(data) { + let written = await this.storage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL }); + if (written && this.summary && !this.summary.localesWritten.includes(this.languageInfo.language)) + this.summary.localesWritten.push(this.languageInfo.language); + return written; } getExpression(ids, valueKey) { diff --git a/src/app/updater/updateAll.js b/src/app/updater/updateAll.js index 2d4bf37..c5f50a9 100644 --- a/src/app/updater/updateAll.js +++ b/src/app/updater/updateAll.js @@ -26,7 +26,9 @@ export function createUpdaters(storage) { * * @param {object} storage * @param {{ only?: string[] }} [options] restrict the run to updaters with these names - * @returns {Promise>} + * @returns {Promise>} + * `summary` is what the updater reports about its run (phase timings, localized fetches, + * locale documents written, images downloaded, and updater-specific details). */ export default async function updateAll(storage, { only } = {}) { let results = []; @@ -38,8 +40,8 @@ export default async function updateAll(storage, { only } = {}) { let started = Date.now(); try { - await updater.update(); - results.push({ name, ok: true, ms: Date.now() - started }); + let summary = await updater.update(); + results.push({ name, ok: true, ms: Date.now() - started, summary }); } catch (e) { console.error(e); results.push({ name, ok: false, ms: Date.now() - started, error: e instanceof Error ? e.message : String(e) }); diff --git a/src/app/updater/updaters/FestivalsUpdater.js b/src/app/updater/updaters/FestivalsUpdater.js index 7313970..6dc29f6 100644 --- a/src/app/updater/updaters/FestivalsUpdater.js +++ b/src/app/updater/updaters/FestivalsUpdater.js @@ -52,6 +52,7 @@ export default class FestivalsUpdater extends Updater { // Download result ranking data let festivalIds = jsonpath.query(regionData, '$.results..festival_id'); + this.summary.rankingsFetched = 0; for (let id of festivalIds) { let key = `data/festivals/${this.region.toLowerCase()}-${id}-rankings.json`; @@ -62,6 +63,7 @@ export default class FestivalsUpdater extends Updater { try { let rankings = await this.handleRequest(splatnet.getFestivalRankings(id)); await this.publicStorage.writeJson(key, rankings, { cacheControl: DATA_CACHE_CONTROL }); + this.summary.rankingsFetched++; } catch { // Do nothing diff --git a/src/app/updater/updaters/OriginalGearImageUpdater.js b/src/app/updater/updaters/OriginalGearImageUpdater.js index 1538825..58409f2 100644 --- a/src/app/updater/updaters/OriginalGearImageUpdater.js +++ b/src/app/updater/updaters/OriginalGearImageUpdater.js @@ -10,10 +10,15 @@ export default class OriginalGearImageUpdater extends Updater { } async update() { + this.startSummary(); + // Get the list of skills let data = skills; // Retrieve skill images await this.downloadImages(data); + this.lap('images'); + + return this.finishSummary(); } } diff --git a/src/app/updater/updaters/SchedulesUpdater.js b/src/app/updater/updaters/SchedulesUpdater.js index 4145d93..833097a 100644 --- a/src/app/updater/updaters/SchedulesUpdater.js +++ b/src/app/updater/updaters/SchedulesUpdater.js @@ -45,6 +45,8 @@ export default class SchedulesUpdater extends Updater { // Load known stages, or create the list from the current list of stages let stages = await this.privateStorage.readJson(STAGES_KEY); + this.summary.stagesSeeded = !stages; + this.summary.newStages = []; if (!stages) { let splatnet = new SplatNet; let stageData = await splatnet.getStages(); @@ -65,6 +67,7 @@ export default class SchedulesUpdater extends Updater { first_seen: getTopOfCurrentHour(), first_available: schedule.start_time, })); + this.summary.newStages.push(stage.name); } } } diff --git a/src/app/updater/updaters/Updater.js b/src/app/updater/updaters/Updater.js index 5662d11..4629e5a 100644 --- a/src/app/updater/updaters/Updater.js +++ b/src/app/updater/updaters/Updater.js @@ -20,8 +20,15 @@ export default class Updater { this.privateStorage = storage.privateStorage; } + /** + * Run the update. + * @returns {Promise} a summary of what happened: per-phase timings in ms, which + * languages had to be fetched, which locale documents changed, how many images were + * downloaded, plus anything a subclass adds to this.summary. + */ async update() { this.info('Updating data...'); + this.startSummary(); // Use the first language as the default let languageInfo = this.getLanguages()[0]; @@ -31,23 +38,54 @@ export default class Updater { // Filter the root keys if necessary data = this.filterRootKeys(data); + this.lap('fetch'); // Update localizations data = await this.updateLocalizations(data, languageInfo); + this.lap('localize'); // Apply any other processing data = await this.processData(data); + this.lap('process'); // Write the data await this.publicStorage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL }); // Update calendar events await this.updateCalendarEvents(data); + this.lap('publish'); // Download images if necessary await this.downloadImages(data); + this.lap('images'); this.info('Done.'); + return this.finishSummary(); + } + + /** + * Summary of the current run + */ + + startSummary() { + this.summary = { + timings: {}, + localizedFetches: [], + localesWritten: [], + imagesDownloaded: 0, + }; + this.summaryStarted = this.lapStarted = Date.now(); + } + + lap(name) { + let now = Date.now(); + this.summary.timings[name] = now - this.lapStarted; + this.lapStarted = now; + } + + finishSummary() { + this.summary.totalMs = Date.now() - this.summaryStarted; + return this.summary; } /** Key of this updater's output in the public storage */ @@ -107,7 +145,7 @@ export default class Updater { } getProcessors(languageInfo) { - return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage)); + return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage, this.summary)); } async updateLocalizations(data, initialLanguageInfo) { @@ -130,6 +168,7 @@ export default class Updater { // Retrieve data for missing languages for (let missingLanguageInfo of missingLanguages) { this.info(`Retrieving localized data for region: ${missingLanguageInfo.region}, language: ${missingLanguageInfo.language}`); + this.summary.localizedFetches.push(missingLanguageInfo.language); let localData = await this.handleRequest(this.getData(missingLanguageInfo)); localData = this.filterRootKeys(localData); for (let processor of this.getProcessors(missingLanguageInfo)) @@ -176,6 +215,7 @@ export default class Updater { if (backup) { this.info(`Using CDN backup: ${imagePath}`); await this.publicStorage.writeBytes(key, backup); + this.summary.imagesDownloaded++; return; } @@ -185,6 +225,7 @@ export default class Updater { let splatnet = new SplatNet; let image = await this.handleRequest(splatnet.getImage(imagePath)); await this.publicStorage.writeBytes(key, image); + this.summary.imagesDownloaded++; } /** diff --git a/src/common/storage/FilesystemStorage.js b/src/common/storage/FilesystemStorage.js index 817c4ad..009885c 100644 --- a/src/common/storage/FilesystemStorage.js +++ b/src/common/storage/FilesystemStorage.js @@ -46,9 +46,13 @@ export default class FilesystemStorage { return buffer === null ? null : JSON.parse(buffer.toString('utf8')); } - /** @returns {Promise} always true; the filesystem write is cheap enough not to dedupe */ + /** @returns {Promise} whether anything was written (unchanged content is skipped, as BucketStorage does) */ async writeJson(key, data) { - await this.#write(key, JSON.stringify(data)); + let serialized = JSON.stringify(data); + let existing = await this.#read(key); + if (existing !== null && existing.toString('utf8') === serialized) + return false; + await this.#write(key, serialized); return true; } diff --git a/test/storage/storage-contract.test.js b/test/storage/storage-contract.test.js index 80540b4..c0e3171 100644 --- a/test/storage/storage-contract.test.js +++ b/test/storage/storage-contract.test.js @@ -31,11 +31,12 @@ for (const [name, { create, destroy }] of Object.entries(implementations)) { }); test('json round-trips and becomes visible to exists', async () => { - await storage.writeJson('data/nested/a.json', { hello: 'world' }); + assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'world' }), true); assert.equal(await storage.exists('data/nested/a.json'), true); assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'world' }); - await storage.writeJson('data/nested/a.json', { hello: 'again' }); + assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'world' }), false); // unchanged + assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'again' }), true); assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'again' }); }); diff --git a/test/updater/Updater.test.js b/test/updater/Updater.test.js index 51f3b12..d6a632e 100644 --- a/test/updater/Updater.test.js +++ b/test/updater/Updater.test.js @@ -39,7 +39,13 @@ afterEach(() => mock.restoreAll()); test('publishes data, locale document, and calendar with the right metadata', async () => { const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) }); - await new ThingsUpdater(b).update(); + const summary = await new ThingsUpdater(b).update(); + + assert.deepEqual(Object.keys(summary.timings), ['fetch', 'localize', 'process', 'publish', 'images']); + assert.deepEqual(summary.localizedFetches, ['es', 'es-MX', 'fr', 'fr-CA', 'de', 'nl', 'it', 'ru', 'ja']); + assert.deepEqual(summary.localesWritten, ['en', 'es', 'es-MX', 'fr', 'fr-CA', 'de', 'nl', 'it', 'ru', 'ja']); + assert.equal(summary.imagesDownloaded, 2); + assert.equal(typeof summary.totalMs, 'number'); assert.deepEqual(await json(b.publicBucket, 'data/things.json'), thingsFor('en')); assert.deepEqual((await b.publicBucket.head('data/things.json')).httpMetadata, { @@ -69,8 +75,11 @@ test('fetches each language with missing strings using that region\'s session, t const second = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) }); const puts = mock.method(b.publicBucket, 'put'); - await new ThingsUpdater(b).update(); + const summary = await new ThingsUpdater(b).update(); assert.deepEqual(second.api.map(r => r.language), ['en']); + assert.deepEqual(summary.localizedFetches, []); + assert.deepEqual(summary.localesWritten, []); + assert.equal(summary.imagesDownloaded, 0); assert.deepEqual(puts.mock.calls.map(c => c.arguments[0]).filter(k => k.startsWith('data/locale/')), []); }); diff --git a/test/updater/updaters.test.js b/test/updater/updaters.test.js index 587a3b7..6abd11d 100644 --- a/test/updater/updaters.test.js +++ b/test/updater/updaters.test.js @@ -41,7 +41,9 @@ test('schedules: seeds the known stages from SplatNet and records new stages onc '/api/schedules': () => schedules, '/api/data/stages': () => ({ stages: [reef, fitness] }), }); - await new SchedulesUpdater(b).update(); + const summary = await new SchedulesUpdater(b).update(); + assert.equal(summary.stagesSeeded, true); + assert.deepEqual(summary.newStages, ['Skipper Pavilion']); assert.deepEqual(await json(b.publicBucket, 'data/schedules.json'), schedules); assert.deepEqual(await json(b.privateBucket, 'stages.json'), [ @@ -110,7 +112,8 @@ test('festivals: merges one region into the shared file, fetches missing ranking '/api/festivals/pasts': () => ({ festivals: [festival(1)], results: [{ festival_id: 1 }, { festival_id: 2 }] }), '/api/festivals/2/rankings': () => ({ rankings: 'two' }), }); - await new FestivalsUpdater('NA', b).update(); + const summary = await new FestivalsUpdater('NA', b).update(); + assert.equal(summary.rankingsFetched, 1); const data = await json(b.publicBucket, 'data/festivals.json'); assert.deepEqual(Object.keys(data).sort(), ['eu', 'na']); @@ -132,9 +135,10 @@ test('original gear: mirrors skill images from the bundled skills data', async ( const images = Object.values(skills).map(s => s.image).filter(Boolean); await b.publicBucket.put(`assets/splatnet${images[0]}`, 'existing'); const splatnet = fakeSplatNet(); - await new OriginalGearImageUpdater(b).update(); + const summary = await new OriginalGearImageUpdater(b).update(); assert.equal(splatnet.images.length, images.length - 2); // one already present, one from the embedded backup + assert.equal(summary.imagesDownloaded, images.length - 1); // the backup counts as downloaded assert.equal(keys(b.publicBucket).length, images.length); assert.ok(keys(b.publicBucket).every(k => k.startsWith('assets/splatnet/images/skill/'))); }); diff --git a/workers/updater/updaters.spec.mjs b/workers/updater/updaters.spec.mjs index 98a7a56..6fb532d 100644 --- a/workers/updater/updaters.spec.mjs +++ b/workers/updater/updaters.spec.mjs @@ -19,6 +19,8 @@ describe('runUpdaters', () => { let summary = await runUpdaters(env); expect(summary.ok).toBe(true); + expect(summary.updaters[1].summary).toMatchObject({ localizedFetches: expect.any(Array), imagesDownloaded: expect.any(Number), stagesSeeded: true }); + expect(Object.keys(summary.updaters[1].summary.timings)).toEqual(['fetch', 'localize', 'process', 'publish', 'images']); expect(summary.updaters.map(u => u.name)).toEqual([ 'Original Gear', 'Schedules', 'Co-op Schedules', 'Timeline', 'Festivals NA', 'Festivals EU', 'Festivals JP', 'Merchandises', ]);