diff --git a/src/app/updater/LocalizationProcessor.js b/src/app/updater/LocalizationProcessor.js index 3d81935..783a295 100644 --- a/src/app/updater/LocalizationProcessor.js +++ b/src/app/updater/LocalizationProcessor.js @@ -7,11 +7,13 @@ export default class LocalizationProcessor { * @param {object} ruleset { name, entities, id, values } * @param {{ region: string, language: string }} languageInfo * @param {object} storage the public storage (BucketStorage or FilesystemStorage) + * @param {object} [summary] the updater's run summary; written languages are recorded in it */ - constructor(ruleset, languageInfo, storage) { + constructor(ruleset, languageInfo, storage, summary = null) { this.ruleset = ruleset; this.languageInfo = languageInfo; this.storage = storage; + this.summary = summary; let entities = this.ruleset.entities; this.entityExpressions = (Array.isArray(entities)) ? entities : [entities]; @@ -28,8 +30,11 @@ export default class LocalizationProcessor { return await this.storage.readJson(this.getKey()) ?? {}; } - writeData(data) { - return this.storage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL }); + async writeData(data) { + let written = await this.storage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL }); + if (written && this.summary && !this.summary.localesWritten.includes(this.languageInfo.language)) + this.summary.localesWritten.push(this.languageInfo.language); + return written; } getExpression(ids, valueKey) { diff --git a/src/app/updater/updateAll.js b/src/app/updater/updateAll.js index 2d4bf37..c5f50a9 100644 --- a/src/app/updater/updateAll.js +++ b/src/app/updater/updateAll.js @@ -26,7 +26,9 @@ export function createUpdaters(storage) { * * @param {object} storage * @param {{ only?: string[] }} [options] restrict the run to updaters with these names - * @returns {Promise>} + * @returns {Promise>} + * `summary` is what the updater reports about its run (phase timings, localized fetches, + * locale documents written, images downloaded, and updater-specific details). */ export default async function updateAll(storage, { only } = {}) { let results = []; @@ -38,8 +40,8 @@ export default async function updateAll(storage, { only } = {}) { let started = Date.now(); try { - await updater.update(); - results.push({ name, ok: true, ms: Date.now() - started }); + let summary = await updater.update(); + results.push({ name, ok: true, ms: Date.now() - started, summary }); } catch (e) { console.error(e); results.push({ name, ok: false, ms: Date.now() - started, error: e instanceof Error ? e.message : String(e) }); diff --git a/src/app/updater/updaters/FestivalsUpdater.js b/src/app/updater/updaters/FestivalsUpdater.js index 7313970..6dc29f6 100644 --- a/src/app/updater/updaters/FestivalsUpdater.js +++ b/src/app/updater/updaters/FestivalsUpdater.js @@ -52,6 +52,7 @@ export default class FestivalsUpdater extends Updater { // Download result ranking data let festivalIds = jsonpath.query(regionData, '$.results..festival_id'); + this.summary.rankingsFetched = 0; for (let id of festivalIds) { let key = `data/festivals/${this.region.toLowerCase()}-${id}-rankings.json`; @@ -62,6 +63,7 @@ export default class FestivalsUpdater extends Updater { try { let rankings = await this.handleRequest(splatnet.getFestivalRankings(id)); await this.publicStorage.writeJson(key, rankings, { cacheControl: DATA_CACHE_CONTROL }); + this.summary.rankingsFetched++; } catch { // Do nothing diff --git a/src/app/updater/updaters/OriginalGearImageUpdater.js b/src/app/updater/updaters/OriginalGearImageUpdater.js index 1538825..58409f2 100644 --- a/src/app/updater/updaters/OriginalGearImageUpdater.js +++ b/src/app/updater/updaters/OriginalGearImageUpdater.js @@ -10,10 +10,15 @@ export default class OriginalGearImageUpdater extends Updater { } async update() { + this.startSummary(); + // Get the list of skills let data = skills; // Retrieve skill images await this.downloadImages(data); + this.lap('images'); + + return this.finishSummary(); } } diff --git a/src/app/updater/updaters/SchedulesUpdater.js b/src/app/updater/updaters/SchedulesUpdater.js index 4145d93..833097a 100644 --- a/src/app/updater/updaters/SchedulesUpdater.js +++ b/src/app/updater/updaters/SchedulesUpdater.js @@ -45,6 +45,8 @@ export default class SchedulesUpdater extends Updater { // Load known stages, or create the list from the current list of stages let stages = await this.privateStorage.readJson(STAGES_KEY); + this.summary.stagesSeeded = !stages; + this.summary.newStages = []; if (!stages) { let splatnet = new SplatNet; let stageData = await splatnet.getStages(); @@ -65,6 +67,7 @@ export default class SchedulesUpdater extends Updater { first_seen: getTopOfCurrentHour(), first_available: schedule.start_time, })); + this.summary.newStages.push(stage.name); } } } diff --git a/src/app/updater/updaters/Updater.js b/src/app/updater/updaters/Updater.js index 5662d11..4629e5a 100644 --- a/src/app/updater/updaters/Updater.js +++ b/src/app/updater/updaters/Updater.js @@ -20,8 +20,15 @@ export default class Updater { this.privateStorage = storage.privateStorage; } + /** + * Run the update. + * @returns {Promise} a summary of what happened: per-phase timings in ms, which + * languages had to be fetched, which locale documents changed, how many images were + * downloaded, plus anything a subclass adds to this.summary. + */ async update() { this.info('Updating data...'); + this.startSummary(); // Use the first language as the default let languageInfo = this.getLanguages()[0]; @@ -31,23 +38,54 @@ export default class Updater { // Filter the root keys if necessary data = this.filterRootKeys(data); + this.lap('fetch'); // Update localizations data = await this.updateLocalizations(data, languageInfo); + this.lap('localize'); // Apply any other processing data = await this.processData(data); + this.lap('process'); // Write the data await this.publicStorage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL }); // Update calendar events await this.updateCalendarEvents(data); + this.lap('publish'); // Download images if necessary await this.downloadImages(data); + this.lap('images'); this.info('Done.'); + return this.finishSummary(); + } + + /** + * Summary of the current run + */ + + startSummary() { + this.summary = { + timings: {}, + localizedFetches: [], + localesWritten: [], + imagesDownloaded: 0, + }; + this.summaryStarted = this.lapStarted = Date.now(); + } + + lap(name) { + let now = Date.now(); + this.summary.timings[name] = now - this.lapStarted; + this.lapStarted = now; + } + + finishSummary() { + this.summary.totalMs = Date.now() - this.summaryStarted; + return this.summary; } /** Key of this updater's output in the public storage */ @@ -107,7 +145,7 @@ export default class Updater { } getProcessors(languageInfo) { - return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage)); + return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage, this.summary)); } async updateLocalizations(data, initialLanguageInfo) { @@ -130,6 +168,7 @@ export default class Updater { // Retrieve data for missing languages for (let missingLanguageInfo of missingLanguages) { this.info(`Retrieving localized data for region: ${missingLanguageInfo.region}, language: ${missingLanguageInfo.language}`); + this.summary.localizedFetches.push(missingLanguageInfo.language); let localData = await this.handleRequest(this.getData(missingLanguageInfo)); localData = this.filterRootKeys(localData); for (let processor of this.getProcessors(missingLanguageInfo)) @@ -176,6 +215,7 @@ export default class Updater { if (backup) { this.info(`Using CDN backup: ${imagePath}`); await this.publicStorage.writeBytes(key, backup); + this.summary.imagesDownloaded++; return; } @@ -185,6 +225,7 @@ export default class Updater { let splatnet = new SplatNet; let image = await this.handleRequest(splatnet.getImage(imagePath)); await this.publicStorage.writeBytes(key, image); + this.summary.imagesDownloaded++; } /** diff --git a/src/common/storage/FilesystemStorage.js b/src/common/storage/FilesystemStorage.js index 817c4ad..009885c 100644 --- a/src/common/storage/FilesystemStorage.js +++ b/src/common/storage/FilesystemStorage.js @@ -46,9 +46,13 @@ export default class FilesystemStorage { return buffer === null ? null : JSON.parse(buffer.toString('utf8')); } - /** @returns {Promise} always true; the filesystem write is cheap enough not to dedupe */ + /** @returns {Promise} whether anything was written (unchanged content is skipped, as BucketStorage does) */ async writeJson(key, data) { - await this.#write(key, JSON.stringify(data)); + let serialized = JSON.stringify(data); + let existing = await this.#read(key); + if (existing !== null && existing.toString('utf8') === serialized) + return false; + await this.#write(key, serialized); return true; } diff --git a/test/storage/storage-contract.test.js b/test/storage/storage-contract.test.js index 80540b4..c0e3171 100644 --- a/test/storage/storage-contract.test.js +++ b/test/storage/storage-contract.test.js @@ -31,11 +31,12 @@ for (const [name, { create, destroy }] of Object.entries(implementations)) { }); test('json round-trips and becomes visible to exists', async () => { - await storage.writeJson('data/nested/a.json', { hello: 'world' }); + assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'world' }), true); assert.equal(await storage.exists('data/nested/a.json'), true); assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'world' }); - await storage.writeJson('data/nested/a.json', { hello: 'again' }); + assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'world' }), false); // unchanged + assert.equal(await storage.writeJson('data/nested/a.json', { hello: 'again' }), true); assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'again' }); }); diff --git a/test/updater/Updater.test.js b/test/updater/Updater.test.js index 51f3b12..d6a632e 100644 --- a/test/updater/Updater.test.js +++ b/test/updater/Updater.test.js @@ -39,7 +39,13 @@ afterEach(() => mock.restoreAll()); test('publishes data, locale document, and calendar with the right metadata', async () => { const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) }); - await new ThingsUpdater(b).update(); + const summary = await new ThingsUpdater(b).update(); + + assert.deepEqual(Object.keys(summary.timings), ['fetch', 'localize', 'process', 'publish', 'images']); + assert.deepEqual(summary.localizedFetches, ['es', 'es-MX', 'fr', 'fr-CA', 'de', 'nl', 'it', 'ru', 'ja']); + assert.deepEqual(summary.localesWritten, ['en', 'es', 'es-MX', 'fr', 'fr-CA', 'de', 'nl', 'it', 'ru', 'ja']); + assert.equal(summary.imagesDownloaded, 2); + assert.equal(typeof summary.totalMs, 'number'); assert.deepEqual(await json(b.publicBucket, 'data/things.json'), thingsFor('en')); assert.deepEqual((await b.publicBucket.head('data/things.json')).httpMetadata, { @@ -69,8 +75,11 @@ test('fetches each language with missing strings using that region\'s session, t const second = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) }); const puts = mock.method(b.publicBucket, 'put'); - await new ThingsUpdater(b).update(); + const summary = await new ThingsUpdater(b).update(); assert.deepEqual(second.api.map(r => r.language), ['en']); + assert.deepEqual(summary.localizedFetches, []); + assert.deepEqual(summary.localesWritten, []); + assert.equal(summary.imagesDownloaded, 0); assert.deepEqual(puts.mock.calls.map(c => c.arguments[0]).filter(k => k.startsWith('data/locale/')), []); }); diff --git a/test/updater/updaters.test.js b/test/updater/updaters.test.js index 587a3b7..6abd11d 100644 --- a/test/updater/updaters.test.js +++ b/test/updater/updaters.test.js @@ -41,7 +41,9 @@ test('schedules: seeds the known stages from SplatNet and records new stages onc '/api/schedules': () => schedules, '/api/data/stages': () => ({ stages: [reef, fitness] }), }); - await new SchedulesUpdater(b).update(); + const summary = await new SchedulesUpdater(b).update(); + assert.equal(summary.stagesSeeded, true); + assert.deepEqual(summary.newStages, ['Skipper Pavilion']); assert.deepEqual(await json(b.publicBucket, 'data/schedules.json'), schedules); assert.deepEqual(await json(b.privateBucket, 'stages.json'), [ @@ -110,7 +112,8 @@ test('festivals: merges one region into the shared file, fetches missing ranking '/api/festivals/pasts': () => ({ festivals: [festival(1)], results: [{ festival_id: 1 }, { festival_id: 2 }] }), '/api/festivals/2/rankings': () => ({ rankings: 'two' }), }); - await new FestivalsUpdater('NA', b).update(); + const summary = await new FestivalsUpdater('NA', b).update(); + assert.equal(summary.rankingsFetched, 1); const data = await json(b.publicBucket, 'data/festivals.json'); assert.deepEqual(Object.keys(data).sort(), ['eu', 'na']); @@ -132,9 +135,10 @@ test('original gear: mirrors skill images from the bundled skills data', async ( const images = Object.values(skills).map(s => s.image).filter(Boolean); await b.publicBucket.put(`assets/splatnet${images[0]}`, 'existing'); const splatnet = fakeSplatNet(); - await new OriginalGearImageUpdater(b).update(); + const summary = await new OriginalGearImageUpdater(b).update(); assert.equal(splatnet.images.length, images.length - 2); // one already present, one from the embedded backup + assert.equal(summary.imagesDownloaded, images.length - 1); // the backup counts as downloaded assert.equal(keys(b.publicBucket).length, images.length); assert.ok(keys(b.publicBucket).every(k => k.startsWith('assets/splatnet/images/skill/'))); }); diff --git a/workers/updater/updaters.spec.mjs b/workers/updater/updaters.spec.mjs index 98a7a56..6fb532d 100644 --- a/workers/updater/updaters.spec.mjs +++ b/workers/updater/updaters.spec.mjs @@ -19,6 +19,8 @@ describe('runUpdaters', () => { let summary = await runUpdaters(env); expect(summary.ok).toBe(true); + expect(summary.updaters[1].summary).toMatchObject({ localizedFetches: expect.any(Array), imagesDownloaded: expect.any(Number), stagesSeeded: true }); + expect(Object.keys(summary.updaters[1].summary.timings)).toEqual(['fetch', 'localize', 'process', 'publish', 'images']); expect(summary.updaters.map(u => u.name)).toEqual([ 'Original Gear', 'Schedules', 'Co-op Schedules', 'Timeline', 'Festivals NA', 'Festivals EU', 'Festivals JP', 'Merchandises', ]);