Return a run summary from each updater

Updater.update() now returns what happened: per-phase timings (fetch,
localize, process, publish, images), which languages had to be fetched,
which locale documents changed, and how many images were downloaded.
Subclasses add their own details (stages seeded and new stages, festival
rankings fetched). updateAll() carries each summary in its results, so the
Worker's "Updaters finished" log entry contains the whole picture per
updater. FilesystemStorage.writeJson now skips unchanged content like
BucketStorage does, so "locale documents written" means the same locally.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

src/app/updater/LocalizationProcessor.js

@@ -7,11 +7,13 @@ export default class LocalizationProcessor {
      * @param {object} ruleset  { name, entities, id, values }
      * @param {{ region: string, language: string }} languageInfo
      * @param {object} storage  the public storage (BucketStorage or FilesystemStorage)
+     * @param {object} [summary]  the updater's run summary; written languages are recorded in it
      */
-    constructor(ruleset, languageInfo, storage) {
+    constructor(ruleset, languageInfo, storage, summary = null) {
         this.ruleset = ruleset;
         this.languageInfo = languageInfo;
         this.storage = storage;
+        this.summary = summary;
 
         let entities = this.ruleset.entities;
         this.entityExpressions = (Array.isArray(entities)) ? entities : [entities];
@@ -28,8 +30,11 @@ export default class LocalizationProcessor {
         return await this.storage.readJson(this.getKey()) ?? {};
     }
 
-    writeData(data) {
-        return this.storage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL });
+    async writeData(data) {
+        let written = await this.storage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL });
+        if (written && this.summary && !this.summary.localesWritten.includes(this.languageInfo.language))
+            this.summary.localesWritten.push(this.languageInfo.language);
+        return written;
     }
 
     getExpression(ids, valueKey) {

src/app/updater/updateAll.js

@@ -26,7 +26,9 @@ export function createUpdaters(storage) {
  *
  * @param {object} storage
  * @param {{ only?: string[] }} [options]  restrict the run to updaters with these names
- * @returns {Promise<Array<{ name: string, ok: boolean, ms: number, error?: string }>>}
+ * @returns {Promise<Array<{ name: string, ok: boolean, ms: number, error?: string, summary?: object }>>}
+ *   `summary` is what the updater reports about its run (phase timings, localized fetches,
+ *   locale documents written, images downloaded, and updater-specific details).
  */
 export default async function updateAll(storage, { only } = {}) {
     let results = [];
@@ -38,8 +40,8 @@ export default async function updateAll(storage, { only } = {}) {
 
         let started = Date.now();
         try {
-            await updater.update();
-            results.push({ name, ok: true, ms: Date.now() - started });
+            let summary = await updater.update();
+            results.push({ name, ok: true, ms: Date.now() - started, summary });
         } catch (e) {
             console.error(e);
             results.push({ name, ok: false, ms: Date.now() - started, error: e instanceof Error ? e.message : String(e) });

src/app/updater/updaters/FestivalsUpdater.js

@@ -52,6 +52,7 @@ export default class FestivalsUpdater extends Updater {
 
         // Download result ranking data
         let festivalIds = jsonpath.query(regionData, '$.results..festival_id');
+        this.summary.rankingsFetched = 0;
         for (let id of festivalIds) {
             let key = `data/festivals/${this.region.toLowerCase()}-${id}-rankings.json`;
 
@@ -62,6 +63,7 @@ export default class FestivalsUpdater extends Updater {
                 try {
                     let rankings  = await this.handleRequest(splatnet.getFestivalRankings(id));
                     await this.publicStorage.writeJson(key, rankings, { cacheControl: DATA_CACHE_CONTROL });
+                    this.summary.rankingsFetched++;
                 }
                 catch {
                     // Do nothing

src/app/updater/updaters/OriginalGearImageUpdater.js

@@ -10,10 +10,15 @@ export default class OriginalGearImageUpdater extends Updater {
     }
 
     async update() {
+        this.startSummary();
+
         // Get the list of skills
         let data = skills;
 
         // Retrieve skill images
         await this.downloadImages(data);
+        this.lap('images');
+
+        return this.finishSummary();
     }
 }

src/app/updater/updaters/SchedulesUpdater.js

@@ -45,6 +45,8 @@ export default class SchedulesUpdater extends Updater {
 
         // Load known stages, or create the list from the current list of stages
         let stages = await this.privateStorage.readJson(STAGES_KEY);
+        this.summary.stagesSeeded = !stages;
+        this.summary.newStages = [];
         if (!stages) {
             let splatnet = new SplatNet;
             let stageData = await splatnet.getStages();
@@ -65,6 +67,7 @@ export default class SchedulesUpdater extends Updater {
                         first_seen: getTopOfCurrentHour(),
                         first_available: schedule.start_time,
                     }));
+                    this.summary.newStages.push(stage.name);
                 }
             }
         }

src/app/updater/updaters/Updater.js

@@ -20,8 +20,15 @@ export default class Updater {
         this.privateStorage = storage.privateStorage;
     }
 
+    /**
+     * Run the update.
+     * @returns {Promise<object>} a summary of what happened: per-phase timings in ms, which
+     *   languages had to be fetched, which locale documents changed, how many images were
+     *   downloaded, plus anything a subclass adds to this.summary.
+     */
     async update() {
         this.info('Updating data...');
+        this.startSummary();
 
         // Use the first language as the default
         let languageInfo = this.getLanguages()[0];
@@ -31,23 +38,54 @@ export default class Updater {
 
         // Filter the root keys if necessary
         data = this.filterRootKeys(data);
+        this.lap('fetch');
 
         // Update localizations
         data = await this.updateLocalizations(data, languageInfo);
+        this.lap('localize');
 
         // Apply any other processing
         data = await this.processData(data);
+        this.lap('process');
 
         // Write the data
         await this.publicStorage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL });
 
         // Update calendar events
         await this.updateCalendarEvents(data);
+        this.lap('publish');
 
         // Download images if necessary
         await this.downloadImages(data);
+        this.lap('images');
 
         this.info('Done.');
+        return this.finishSummary();
+    }
+
+    /**
+     * Summary of the current run
+     */
+
+    startSummary() {
+        this.summary = {
+            timings: {},
+            localizedFetches: [],
+            localesWritten: [],
+            imagesDownloaded: 0,
+        };
+        this.summaryStarted = this.lapStarted = Date.now();
+    }
+
+    lap(name) {
+        let now = Date.now();
+        this.summary.timings[name] = now - this.lapStarted;
+        this.lapStarted = now;
+    }
+
+    finishSummary() {
+        this.summary.totalMs = Date.now() - this.summaryStarted;
+        return this.summary;
     }
 
     /** Key of this updater's output in the public storage */
@@ -107,7 +145,7 @@ export default class Updater {
     }
 
     getProcessors(languageInfo) {
-        return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage));
+        return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage, this.summary));
     }
 
     async updateLocalizations(data, initialLanguageInfo) {
@@ -130,6 +168,7 @@ export default class Updater {
             // Retrieve data for missing languages
             for (let missingLanguageInfo of missingLanguages) {
                 this.info(`Retrieving localized data for region: ${missingLanguageInfo.region}, language: ${missingLanguageInfo.language}`);
+                this.summary.localizedFetches.push(missingLanguageInfo.language);
                 let localData = await this.handleRequest(this.getData(missingLanguageInfo));
                 localData = this.filterRootKeys(localData);
                 for (let processor of this.getProcessors(missingLanguageInfo))
@@ -176,6 +215,7 @@ export default class Updater {
         if (backup) {
             this.info(`Using CDN backup: ${imagePath}`);
             await this.publicStorage.writeBytes(key, backup);
+            this.summary.imagesDownloaded++;
 
             return;
         }
@@ -185,6 +225,7 @@ export default class Updater {
         let splatnet = new SplatNet;
         let image = await this.handleRequest(splatnet.getImage(imagePath));
         await this.publicStorage.writeBytes(key, image);
+        this.summary.imagesDownloaded++;
     }
 
     /**

src/common/storage/FilesystemStorage.js

@@ -46,9 +46,13 @@ export default class FilesystemStorage {
         return buffer === null ? null : JSON.parse(buffer.toString('utf8'));
     }
 
-    /** @returns {Promise<boolean>} always true; the filesystem write is cheap enough not to dedupe */
+    /** @returns {Promise<boolean>} whether anything was written (unchanged content is skipped, as BucketStorage does) */
     async writeJson(key, data) {
-        await this.#write(key, JSON.stringify(data));
+        let serialized = JSON.stringify(data);
+        let existing = await this.#read(key);
+        if (existing !== null && existing.toString('utf8') === serialized)
+            return false;
+        await this.#write(key, serialized);
         return true;
     }
 

test/storage/storage-contract.test.js

@@ -31,11 +31,12 @@ for (const [name, { create, destroy }] of Object.entries(implementations)) {
     });
 
     test('json round-trips and becomes visible to exists', async () => {
-      await storage.writeJson('data/nested/a.json', { hello: 'world' });
+      assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'world' }), true);
       assert.equal(await storage.exists('data/nested/a.json'), true);
       assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'world' });
 
-      await storage.writeJson('data/nested/a.json', { hello: 'again' });
+      assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'world' }), false); // unchanged
+      assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'again' }), true);
       assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'again' });
     });
 

test/updater/Updater.test.js

@@ -39,7 +39,13 @@ afterEach(() => mock.restoreAll());
 
 test('publishes data, locale document, and calendar with the right metadata', async () => {
   const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) });
-  await new ThingsUpdater(b).update();
+  const summary = await new ThingsUpdater(b).update();
+
+  assert.deepEqual(Object.keys(summary.timings), ['fetch', 'localize', 'process', 'publish', 'images']);
+  assert.deepEqual(summary.localizedFetches, ['es', 'es-MX', 'fr', 'fr-CA', 'de', 'nl', 'it', 'ru', 'ja']);
+  assert.deepEqual(summary.localesWritten, ['en', 'es', 'es-MX', 'fr', 'fr-CA', 'de', 'nl', 'it', 'ru', 'ja']);
+  assert.equal(summary.imagesDownloaded, 2);
+  assert.equal(typeof summary.totalMs, 'number');
 
   assert.deepEqual(await json(b.publicBucket, 'data/things.json'), thingsFor('en'));
   assert.deepEqual((await b.publicBucket.head('data/things.json')).httpMetadata, {
@@ -69,8 +75,11 @@ test('fetches each language with missing strings using that region\'s session, t
 
   const second = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) });
   const puts = mock.method(b.publicBucket, 'put');
-  await new ThingsUpdater(b).update();
+  const summary = await new ThingsUpdater(b).update();
   assert.deepEqual(second.api.map(r => r.language), ['en']);
+  assert.deepEqual(summary.localizedFetches, []);
+  assert.deepEqual(summary.localesWritten, []);
+  assert.equal(summary.imagesDownloaded, 0);
   assert.deepEqual(puts.mock.calls.map(c => c.arguments[0]).filter(k => k.startsWith('data/locale/')), []);
 });
 

test/updater/updaters.test.js

@@ -41,7 +41,9 @@ test('schedules: seeds the known stages from SplatNet and records new stages onc
     '/api/schedules': () => schedules,
     '/api/data/stages': () => ({ stages: [reef, fitness] }),
   });
-  await new SchedulesUpdater(b).update();
+  const summary = await new SchedulesUpdater(b).update();
+  assert.equal(summary.stagesSeeded, true);
+  assert.deepEqual(summary.newStages, ['Skipper Pavilion']);
 
   assert.deepEqual(await json(b.publicBucket, 'data/schedules.json'), schedules);
   assert.deepEqual(await json(b.privateBucket, 'stages.json'), [
@@ -110,7 +112,8 @@ test('festivals: merges one region into the shared file, fetches missing ranking
     '/api/festivals/pasts': () => ({ festivals: [festival(1)], results: [{ festival_id: 1 }, { festival_id: 2 }] }),
     '/api/festivals/2/rankings': () => ({ rankings: 'two' }),
   });
-  await new FestivalsUpdater('NA', b).update();
+  const summary = await new FestivalsUpdater('NA', b).update();
+  assert.equal(summary.rankingsFetched, 1);
 
   const data = await json(b.publicBucket, 'data/festivals.json');
   assert.deepEqual(Object.keys(data).sort(), ['eu', 'na']);
@@ -132,9 +135,10 @@ test('original gear: mirrors skill images from the bundled skills data', async (
   const images = Object.values(skills).map(s => s.image).filter(Boolean);
   await b.publicBucket.put(`assets/splatnet${images[0]}`, 'existing');
   const splatnet = fakeSplatNet();
-  await new OriginalGearImageUpdater(b).update();
+  const summary = await new OriginalGearImageUpdater(b).update();
 
   assert.equal(splatnet.images.length, images.length - 2); // one already present, one from the embedded backup
+  assert.equal(summary.imagesDownloaded, images.length - 1); // the backup counts as downloaded
   assert.equal(keys(b.publicBucket).length, images.length);
   assert.ok(keys(b.publicBucket).every(k => k.startsWith('assets/splatnet/images/skill/')));
 });

workers/updater/updaters.spec.mjs

@@ -19,6 +19,8 @@ describe('runUpdaters', () => {
     let summary = await runUpdaters(env);
 
     expect(summary.ok).toBe(true);
+    expect(summary.updaters[1].summary).toMatchObject({ localizedFetches: expect.any(Array), imagesDownloaded: expect.any(Number), stagesSeeded: true });
+    expect(Object.keys(summary.updaters[1].summary.timings)).toEqual(['fetch', 'localize', 'process', 'publish', 'images']);
     expect(summary.updaters.map(u => u.name)).toEqual([
       'Original Gear', 'Schedules', 'Co-op Schedules', 'Timeline', 'Festivals NA', 'Festivals EU', 'Festivals JP', 'Merchandises',
     ]);