diff --git a/readme.md b/readme.md index 5e21a0c..662715d 100644 --- a/readme.md +++ b/readme.md @@ -35,6 +35,11 @@ npm run serve # Start the webpack dev server Data retrieved from Splatnet is stored in the `dist/data` directory. +One image, `src/common/cdn/images/skill/53c62995f9d2dc4a60f3850c5dbdd2323f1eef87.png`, +is no longer served by SplatNet's CDN. It is already in production storage; if you ever +start from an empty `dist/`, copy it to `dist/assets/splatnet/images/skill/` by hand +(or run `npm run sync:download`). + By default, the dev server will run on port 8080. When running `npm run serve` you can access the site by going to http://localhost:8080. diff --git a/scripts/compare-data.mjs b/scripts/compare-data.mjs new file mode 100644 index 0000000..5c91cd2 --- /dev/null +++ b/scripts/compare-data.mjs @@ -0,0 +1,77 @@ +// Compare two data directories (e.g. a saved baseline and dist/data) the way the site +// cares about: SplatNet returns object keys and some arrays in random order per request, +// and ICS files carry a per-run DTSTAMP, so byte diffs are noise. Usage: +// +// node scripts/compare-data.mjs +// +// Exits non-zero when a real difference is found. + +import fs from 'node:fs'; +import path from 'node:path'; + +const [base, current] = process.argv.slice(2); +if (!base || !current) { + console.error('Usage: node scripts/compare-data.mjs '); + process.exit(2); +} + +const canon = value => Array.isArray(value) + ? value.map(canon) + : (value && typeof value === 'object') + ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canon(value[key])])) + : value; + +function firstDifference(a, b, at = '$') { + if (Array.isArray(a) && Array.isArray(b)) { + if (a.length !== b.length) + return `${at}.length ${a.length} vs ${b.length}`; + // Festival result lists come back in random order + if (a.length && a[0] && typeof a[0] === 'object' && 'festival_id' in a[0]) { + let key = item => JSON.stringify(canon(item)); + let sa = a.map(key).sort(), sb = b.map(key).sort(); + return sa.every((v, i) => v === sb[i]) ? null : `${at}: set of items differs`; + } + for (let i = 0; i < a.length; i++) { + let d = firstDifference(a[i], b[i], `${at}[${i}]`); + if (d) return d; + } + return null; + } + if (a && b && typeof a === 'object' && typeof b === 'object') { + for (let key of new Set([...Object.keys(a), ...Object.keys(b)])) { + let d = firstDifference(a[key], b[key], `${at}.${key}`); + if (d) return d; + } + return null; + } + return a === b ? null : `${at}: ${JSON.stringify(a)?.slice(0, 60)} vs ${JSON.stringify(b)?.slice(0, 60)}`; +} + +const walk = dir => fs.readdirSync(dir, { withFileTypes: true }) + .flatMap(entry => entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]); + +let problems = 0; +for (let file of walk(base)) { + let relative = path.relative(base, file); + let other = path.join(current, relative); + if (!fs.existsSync(other)) { + console.log(`${relative}: missing in ${current}`); + problems++; + continue; + } + let a = fs.readFileSync(file), b = fs.readFileSync(other); + if (a.equals(b)) + continue; + if (relative.endsWith('.json')) { + let d = firstDifference(JSON.parse(a), JSON.parse(b)); + if (d) { console.log(`${relative}: ${d}`); problems++; } + } else if (relative.endsWith('.ics')) { + let strip = s => s.toString().split(/\r?\n/).filter(line => !line.startsWith('DTSTAMP')).join('\n'); + if (strip(a) !== strip(b)) { console.log(`${relative}: calendar content differs`); problems++; } + } else { + console.log(`${relative}: binary content differs`); + problems++; + } +} +console.log(problems ? `${problems} real difference(s)` : 'semantically identical'); +process.exit(problems ? 1 : 0); diff --git a/src/app/cron.js b/src/app/cron.js index 7c9f46b..d552013 100644 --- a/src/app/cron.js +++ b/src/app/cron.js @@ -1,12 +1,13 @@ import '../common/bootstrap.js'; import { CronJob } from 'cron'; import { updateAll } from './updater/index.js'; +import { filesystemStorage } from './storage.js'; import { maybePostTweets } from './twitter/index.js'; console.info('Starting periodic tasks...'); // Run every hour at 10sec after the hour new CronJob('10 0 * * * *', async () => { - await updateAll(); + await updateAll(filesystemStorage()); await maybePostTweets(); }, null, true); diff --git a/src/app/index.js b/src/app/index.js index 0014016..4f2688e 100644 --- a/src/app/index.js +++ b/src/app/index.js @@ -1,11 +1,12 @@ import '../common/bootstrap.js'; import { runAction } from '../common/cli.js'; import { updateAll } from './updater/index.js'; +import { filesystemStorage } from './storage.js'; import { maybePostTweets, testScreenshots } from './twitter/index.js'; import { sync, syncUpload, syncDownload } from './sync/index.js'; await runAction({ - splatnet: updateAll, + splatnet: () => updateAll(filesystemStorage()), twitter: maybePostTweets, twitterTest: testScreenshots, sync, diff --git a/src/app/storage.js b/src/app/storage.js new file mode 100644 index 0000000..7615fac --- /dev/null +++ b/src/app/storage.js @@ -0,0 +1,12 @@ +import { FilesystemStorage } from '../common/storage/index.js'; + +/** + * The storage a local (Node) run uses: the same directories the site has always been + * built from, so `npm run splatnet` and the S3 sync keep working unchanged. + */ +export function filesystemStorage() { + return { + publicStorage: new FilesystemStorage('dist'), + privateStorage: new FilesystemStorage('storage'), + }; +} diff --git a/src/app/updater/LocalizationProcessor.js b/src/app/updater/LocalizationProcessor.js index afacea6..3d81935 100644 --- a/src/app/updater/LocalizationProcessor.js +++ b/src/app/updater/LocalizationProcessor.js @@ -1,54 +1,41 @@ -import path from 'node:path'; -import fs from 'node:fs'; -import { mkdirpSync as mkdirp } from 'mkdirp'; import jsonpath from '../../common/jsonpath.js'; import _ from 'lodash'; -import { readJson, writeJson } from '../../common/utilities.js'; - -const localizationsPath = path.resolve('dist/data/locale'); +import { DATA_CACHE_CONTROL } from '../../common/storage/index.js'; export default class LocalizationProcessor { - constructor(ruleset, languageInfo) { + /** + * @param {object} ruleset { name, entities, id, values } + * @param {{ region: string, language: string }} languageInfo + * @param {object} storage the public storage (BucketStorage or FilesystemStorage) + */ + constructor(ruleset, languageInfo, storage) { this.ruleset = ruleset; this.languageInfo = languageInfo; + this.storage = storage; let entities = this.ruleset.entities; this.entityExpressions = (Array.isArray(entities)) ? entities : [entities]; let values = this.ruleset.values; this.valueExpressions = (Array.isArray(values)) ? values : [values]; - - this.readData(); } - getFilename() { - return `${localizationsPath}/${this.languageInfo.language}.json`; + getKey() { + return `data/locale/${this.languageInfo.language}.json`; } - readData() { - if (fs.existsSync(this.getFilename())) - this.data = readJson(this.getFilename()); - else - this.data = {}; + async readData() { + return await this.storage.readJson(this.getKey()) ?? {}; } - writeData() { - mkdirp(path.dirname(this.getFilename())); - writeJson(this.getFilename(), this.data); + writeData(data) { + return this.storage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL }); } getExpression(ids, valueKey) { return [this.ruleset.name, ...ids, valueKey]; } - readValue(ids, valueKey) { - return _.get(this.data, this.getExpression(ids, valueKey)); - } - - writeValue(ids, valueKey, newValue) { - return _.setWith(this.data, this.getExpression(ids, valueKey), newValue, Object); - } - getIdValues(entity) { let idParts = this.ruleset.id; @@ -60,44 +47,31 @@ export default class LocalizationProcessor { return idParts.map(id => _.get(entity, id)); } - eachEntity(data, callback) { - for (let expression of this.entityExpressions) { - let entities = jsonpath.query(data, expression); + *entities(data) { + for (let expression of this.entityExpressions) + yield* jsonpath.query(data, expression); + } - for (let entity of entities) { - let result = callback(entity); - if (result === false) + async updateLocalizations(data) { + let localizations = await this.readData(); + + for (let entity of this.entities(data)) { + let ids = this.getIdValues(entity); + for (let valueKey of this.valueExpressions) + _.setWith(localizations, this.getExpression(ids, valueKey), _.get(entity, valueKey), Object); + } + + await this.writeData(localizations); + } + + async hasLocalizations(data) { + let localizations = await this.readData(); + + for (let entity of this.entities(data)) { + let ids = this.getIdValues(entity); + for (let valueKey of this.valueExpressions) + if (_.get(localizations, this.getExpression(ids, valueKey)) === undefined) return false; - } - } - } - - updateLocalizations(data) { - this.eachEntity(data, entity => this.updateLocalizationsForEntity(entity)); - } - - updateLocalizationsForEntity(entity) { - for (let valueKey of this.valueExpressions) { - let ids = this.getIdValues(entity); - let value = _.get(entity, valueKey); - - this.writeValue(ids, valueKey, value); - } - - this.writeData(); - } - - hasLocalizations(data) { - let result = this.eachEntity(data, entity => this.hasLocalizationsForEntity(entity)); - return (result !== false); - } - - hasLocalizationsForEntity(entity) { - for (let valueKey of this.valueExpressions) { - let ids = this.getIdValues(entity); - - if (this.readValue(ids, valueKey) === undefined) - return false; } return true; diff --git a/src/app/updater/updateAll.js b/src/app/updater/updateAll.js index 3bacaeb..62aa5a3 100644 --- a/src/app/updater/updateAll.js +++ b/src/app/updater/updateAll.js @@ -7,21 +7,28 @@ import MerchandisesUpdater from './updaters/MerchandisesUpdater.js'; import S3Syncer from '../sync/S3Syncer.js'; import { canSync } from '../sync/index.js'; -const updaters = [ - new OriginalGearImageUpdater, - new SchedulesUpdater, - new CoopSchedulesUpdater, - new TimelineUpdater, - new FestivalsUpdater('NA'), - new FestivalsUpdater('EU'), - new FestivalsUpdater('JP'), - new MerchandisesUpdater, -]; +/** @param {{ publicStorage: object, privateStorage: object }} storage */ +export function createUpdaters(storage) { + return [ + new OriginalGearImageUpdater(storage), + new SchedulesUpdater(storage), + new CoopSchedulesUpdater(storage), + new TimelineUpdater(storage), + new FestivalsUpdater('NA', storage), + new FestivalsUpdater('EU', storage), + new FestivalsUpdater('JP', storage), + new MerchandisesUpdater(storage), + ]; +} -export default async function updateAll() { +/** + * Run every updater. `storage` is shared by all of them for the run (see createUpdaters); + * a Worker passes BucketStorage over its R2 bindings, a local run passes FilesystemStorage. + */ +export default async function updateAll(storage) { const syncer = canSync() ? new S3Syncer() : null; - for (let updater of updaters) { + for (let updater of createUpdaters(storage)) { try { await updater.update(); } catch (e) { diff --git a/src/app/updater/updaters/CoopSchedulesUpdater.js b/src/app/updater/updaters/CoopSchedulesUpdater.js index 0f90690..77437a1 100644 --- a/src/app/updater/updaters/CoopSchedulesUpdater.js +++ b/src/app/updater/updaters/CoopSchedulesUpdater.js @@ -1,7 +1,7 @@ import Updater from './Updater.js'; export default class CoopSchedulesUpdater extends Updater { - constructor() { + constructor(storage) { super({ name: 'Co-op Schedules', filename: 'coop-schedules.json', @@ -44,7 +44,7 @@ export default class CoopSchedulesUpdater extends Updater { values: 'name', }, ], - }); + }, storage); } getCalendarEntries({ details, schedules }) { diff --git a/src/app/updater/updaters/FestivalsUpdater.js b/src/app/updater/updaters/FestivalsUpdater.js index 65f4d47..abba4aa 100644 --- a/src/app/updater/updaters/FestivalsUpdater.js +++ b/src/app/updater/updaters/FestivalsUpdater.js @@ -1,13 +1,12 @@ import Updater from './Updater.js'; -import fs from 'node:fs'; import _ from 'lodash'; import jsonpath from '../../../common/jsonpath.js'; -import { readJson, writeJson } from '../../../common/utilities.js'; +import { DATA_CACHE_CONTROL } from '../../../common/storage/index.js'; import { languages } from '../../../common/regions.js'; import SplatNet from '../../../common/splatnet.js'; export default class FestivalsUpdater extends Updater { - constructor(region) { + constructor(region, storage) { super({ name: `Festivals ${region}`, filename: 'festivals.json', @@ -33,7 +32,7 @@ export default class FestivalsUpdater extends Updater { values: 'name', }, ], - }); + }, storage); this.region = region; } @@ -54,15 +53,15 @@ export default class FestivalsUpdater extends Updater { // Download result ranking data let festivalIds = jsonpath.query(regionData, '$.results..festival_id'); for (let id of festivalIds) { - let filename = `${this.getOutputPath()}/festivals/${this.region.toLowerCase()}-${id}-rankings.json`; + let key = `data/festivals/${this.region.toLowerCase()}-${id}-rankings.json`; // Have we already downloaded these rankings? - if (!fs.existsSync(filename)) { + if (!await this.publicStorage.exists(key)) { let splatnet = new SplatNet(this.region); this.info(`Retrieving rankings for festival ID ${id}`); try { let rankings = await this.handleRequest(splatnet.getFestivalRankings(id)); - writeJson(filename, rankings); + await this.publicStorage.writeJson(key, rankings, { cacheControl: DATA_CACHE_CONTROL }); } catch (e) { // Do nothing @@ -71,10 +70,7 @@ export default class FestivalsUpdater extends Updater { } // Load existing data since we only need to modify this region's data - let data = {}; - let filename = this.getFilename(); - if (fs.existsSync(filename)) - data = readJson(filename); + let data = await this.publicStorage.readJson(this.getKey()) ?? {}; let region = this.region.toLowerCase(); data[region] = regionData; diff --git a/src/app/updater/updaters/MerchandisesUpdater.js b/src/app/updater/updaters/MerchandisesUpdater.js index c374e2d..624ba61 100644 --- a/src/app/updater/updaters/MerchandisesUpdater.js +++ b/src/app/updater/updaters/MerchandisesUpdater.js @@ -2,7 +2,7 @@ import Updater from './Updater.js'; import { getOriginalGear } from '../../../common/splatoon.js'; export default class MerchandisesUpdater extends Updater { - constructor() { + constructor(storage) { super({ name: 'Merchandises', filename: 'merchandises.json', @@ -40,7 +40,7 @@ export default class MerchandisesUpdater extends Updater { values: 'name', }, ], - }); + }, storage); } processData(data) { diff --git a/src/app/updater/updaters/OriginalGearImageUpdater.js b/src/app/updater/updaters/OriginalGearImageUpdater.js index 8c02df8..5f3cbf9 100644 --- a/src/app/updater/updaters/OriginalGearImageUpdater.js +++ b/src/app/updater/updaters/OriginalGearImageUpdater.js @@ -1,19 +1,17 @@ import Updater from './Updater.js'; -import path from 'node:path'; -import fs from 'node:fs'; -const dataPath = path.resolve('src/common/data'); +import { readJsonFile } from '../../../common/utilities.js'; export default class OriginalGearImageUpdater extends Updater { - constructor() { + constructor(storage) { super({ name: 'Original Gear', imagePaths: ['$..image'], - }); + }, storage); } async update() { // Get the list of skills - let data = JSON.parse(fs.readFileSync(`${dataPath}/skills.json`)); + let data = readJsonFile(new URL('../../../common/data/skills.json', import.meta.url)); // Retrieve skill images await this.downloadImages(data); diff --git a/src/app/updater/updaters/SchedulesUpdater.js b/src/app/updater/updaters/SchedulesUpdater.js index 3712527..735d74f 100644 --- a/src/app/updater/updaters/SchedulesUpdater.js +++ b/src/app/updater/updaters/SchedulesUpdater.js @@ -1,13 +1,11 @@ import Updater from './Updater.js'; import SplatNet from '../../../common/splatnet.js'; -import path from 'node:path'; -import fs from 'node:fs'; -import { getTopOfCurrentHour, readJson } from '../../../common/utilities.js'; +import { getTopOfCurrentHour } from '../../../common/utilities.js'; -const stagesPath = path.resolve('storage/stages.json'); +const STAGES_KEY = 'stages.json'; // private storage export default class SchedulesUpdater extends Updater { - constructor() { + constructor(storage) { super({ name: 'Schedules', filename: 'schedules.json', @@ -39,28 +37,25 @@ export default class SchedulesUpdater extends Updater { values: ['name', 'multiline_name'], }, ], - }); + }, storage); } async processData(data) { // We need to track whether there are any new stages included with this schedule update - // If we don't have a "known stages" file, create it from the current list of stages - if (!fs.existsSync(stagesPath)) { + // Load known stages, or create the list from the current list of stages + let stages = await this.privateStorage.readJson(STAGES_KEY); + if (!stages) { let splatnet = new SplatNet; let stageData = await splatnet.getStages(); - let stages = stageData.stages.map(s => Object.assign(s, { first_seen: -1, first_available: -1 })); - this.writeFile(stagesPath, JSON.stringify(stages)); + stages = stageData.stages.map(s => Object.assign(s, { first_seen: -1, first_available: -1 })); } - // Load known stages - let stages = readJson(stagesPath); - // Look for new stages (in Regular Battle) - let sortedSchedules = data.regular.sort((a, b) => a.start_time - b.start_time); + let sortedSchedules = [...data.regular].sort((a, b) => a.start_time - b.start_time); for (let schedule of sortedSchedules) { for (let stage of [schedule.stage_a, schedule.stage_b]) { - // Have wee seen this stage before? + // Have we seen this stage before? let knownStage = stages.find(s => s.id == stage.id); if (!knownStage) { @@ -75,7 +70,7 @@ export default class SchedulesUpdater extends Updater { } // Update the stages file - this.writeFile(stagesPath, JSON.stringify(stages)); + await this.privateStorage.writeJson(STAGES_KEY, stages); return data; } diff --git a/src/app/updater/updaters/TimelineUpdater.js b/src/app/updater/updaters/TimelineUpdater.js index 9881b16..3eda380 100644 --- a/src/app/updater/updaters/TimelineUpdater.js +++ b/src/app/updater/updaters/TimelineUpdater.js @@ -1,7 +1,7 @@ import Updater from './Updater.js'; export default class TimelineUpdater extends Updater { - constructor() { + constructor(storage) { super({ name: 'Timeline', filename: 'timeline.json', @@ -46,6 +46,6 @@ export default class TimelineUpdater extends Updater { values: 'name', }, ], - }); + }, storage); } } diff --git a/src/app/updater/updaters/Updater.js b/src/app/updater/updaters/Updater.js index fe748ce..fe4a15c 100644 --- a/src/app/updater/updaters/Updater.js +++ b/src/app/updater/updaters/Updater.js @@ -1,6 +1,3 @@ -import path from 'node:path'; -import fs from 'node:fs'; -import { mkdirpSync as mkdirp } from 'mkdirp'; import _ from 'lodash'; import jsonpath from '../../../common/jsonpath.js'; import SplatNet from '../../../common/splatnet.js'; @@ -8,14 +5,18 @@ import { createEvents } from 'ics'; import * as Sentry from '@sentry/node'; import { languages } from '../../../common/regions.js'; import LocalizationProcessor from '../LocalizationProcessor.js'; - -const dataPath = path.resolve('dist/data'); -const splatnetAssetPath = path.resolve('dist/assets/splatnet'); -const cdnAltPath = path.resolve('src/common/cdn'); +import { DATA_CACHE_CONTROL } from '../../../common/storage/index.js'; export default class Updater { - constructor(options = {}) { + /** + * @param {object} options + * @param {{ publicStorage: object, privateStorage: object }} storage + * publicStorage holds what the site serves (data/, assets/); privateStorage holds updater state. + */ + constructor(options = {}, storage = {}) { this.options = options; + this.publicStorage = storage.publicStorage; + this.privateStorage = storage.privateStorage; } async update() { @@ -36,14 +37,11 @@ export default class Updater { // Apply any other processing data = await this.processData(data); - // Convert the data to a JSON string - let dataString = JSON.stringify(data); - - // Write the data to disk - this.writeFile(this.getFilename(), dataString); + // Write the data + await this.publicStorage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL }); // Update calendar events - this.updateCalendarEvents(data); + await this.updateCalendarEvents(data); // Download images if necessary await this.downloadImages(data); @@ -51,17 +49,14 @@ export default class Updater { this.info('Done.'); } - getOutputPath() { - return dataPath; + /** Key of this updater's output in the public storage */ + getKey() { + return `data/${this.options.filename}`; } - getFilename() { - return `${this.getOutputPath()}/${this.options.filename}`; - } - - getCalendarFilename() { + getCalendarKey() { if (this.options.calendarFilename) - return `${this.getOutputPath()}/${this.options.calendarFilename}`; + return `data/${this.options.calendarFilename}`; } getData({ region, language }) { @@ -110,38 +105,34 @@ export default class Updater { return _.uniqBy(languages, 'language'); } - forEachLanguage(callback) { - for (let languageInfo of this.getLanguages()) - this.forEachRuleset(languageInfo, callback); - } - - forEachRuleset(languageInfo, callback) { - for (let ruleset of (this.options.localization)) { - let processor = new LocalizationProcessor(ruleset, languageInfo); - callback(processor, languageInfo); - } + getProcessors(languageInfo) { + return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage)); } async updateLocalizations(data, initialLanguageInfo) { if (this.options.localization) { // Update localization data for the initial language - this.forEachRuleset(initialLanguageInfo, processor => processor.updateLocalizations(data)); + for (let processor of this.getProcessors(initialLanguageInfo)) + await processor.updateLocalizations(data); // Do we need to retrieve data for any other languages? let missingLanguages = []; - this.forEachLanguage((processor, languageInfo) => { - if (missingLanguages.indexOf(languageInfo) === -1) { - if (!processor.hasLocalizations(data)) + for (let languageInfo of this.getLanguages()) { + for (let processor of this.getProcessors(languageInfo)) { + if (!await processor.hasLocalizations(data)) { missingLanguages.push(languageInfo); + break; + } } - }); + } // Retrieve data for missing languages for (let missingLanguageInfo of missingLanguages) { this.info(`Retrieving localized data for region: ${missingLanguageInfo.region}, language: ${missingLanguageInfo.language}`); let localData = await this.handleRequest(this.getData(missingLanguageInfo)); localData = this.filterRootKeys(localData); - this.forEachRuleset(missingLanguageInfo, processor => processor.updateLocalizations(localData)); + for (let processor of this.getProcessors(missingLanguageInfo)) + await processor.updateLocalizations(localData); } } @@ -159,11 +150,6 @@ export default class Updater { return true; } - writeFile(filename, data) { - mkdirp(path.dirname(filename)); - fs.writeFileSync(filename, data); - } - async downloadImages(data) { if (this.options.imagePaths) { for (let expression of this.options.imagePaths) { @@ -178,40 +164,31 @@ export default class Updater { if (!imagePath) return; - let localPath = splatnetAssetPath + imagePath; + let key = `assets/splatnet${imagePath}`; // Check whether the image has already been downloaded - if (fs.existsSync(localPath)) + if (await this.publicStorage.exists(key)) return; - // Certain images are not available on the CDN anymore - if (fs.existsSync(cdnAltPath + imagePath)) { - this.info(`Using CDN backup: ${imagePath}`); - let image = fs.readFileSync(cdnAltPath + imagePath); - this.writeFile(localPath, image); - - return; - } - - // Otherwise, download the image + // Download the image this.info(`Downloading image: ${imagePath}`); let splatnet = new SplatNet; let image = await this.handleRequest(splatnet.getImage(imagePath)); - this.writeFile(localPath, image); + await this.publicStorage.writeBytes(key, image); } /** * Calendar output */ - updateCalendarEvents(data) { - let filename = this.getCalendarFilename(); - if (!filename) + async updateCalendarEvents(data) { + let key = this.getCalendarKey(); + if (!key) return; let events = this.getCalendarEntries(data); let ical = this.getiCalData(events); - this.writeFile(filename, ical); + await this.publicStorage.writeText(key, ical, { cacheControl: DATA_CACHE_CONTROL }); } getCalendarTitle() { diff --git a/src/common/splatnet.js b/src/common/splatnet.js index 44924c2..071f759 100644 --- a/src/common/splatnet.js +++ b/src/common/splatnet.js @@ -1,5 +1,3 @@ -import './bootstrap.js'; - // SplatNet2 API const userAgent = process.env.SPLATNET_USER_AGENT; const splatnetBaseUrl = 'https://app.splatoon2.nintendo.net'; diff --git a/src/common/storage/BucketStorage.js b/src/common/storage/BucketStorage.js new file mode 100644 index 0000000..d704eb1 --- /dev/null +++ b/src/common/storage/BucketStorage.js @@ -0,0 +1,89 @@ +// Storage over an R2 bucket binding (or MemoryBucket in tests), with a thin per-run cache. +// +// Callers keep the shape the filesystem code had (exists / readJson / writeJson) and this +// class keeps the round trips down because the bucket is remote: +// - exists(key) lists the key's directory once and answers from memory after that; +// - readJson(key) parses each document once and hands back the same object each time; +// - writeJson(key) skips the put when the content matches what is already stored. +// Writes go straight through. Create one instance per update run. + +import { contentTypeFor } from './index.js'; + +function directoryOf(key) { + return key.slice(0, key.lastIndexOf('/') + 1); +} + +export default class BucketStorage { + #bucket; + #listings = new Map; // directory prefix -> Set of keys under it + #documents = new Map; // key -> { data, serialized } for JSON documents (data may be null) + + constructor(bucket) { + this.#bucket = bucket; + } + + get bucket() { + return this.#bucket; + } + + async #listing(directory) { + if (!this.#listings.has(directory)) { + let keys = new Set; + let cursor; + do { + let page = await this.#bucket.list({ prefix: directory, cursor, limit: 1000 }); + for (let object of page.objects) + keys.add(object.key); + cursor = page.truncated ? page.cursor : undefined; + } while (cursor); + this.#listings.set(directory, keys); + } + return this.#listings.get(directory); + } + + #noteWritten(key) { + this.#listings.get(directoryOf(key))?.add(key); + } + + async exists(key) { + return (await this.#listing(directoryOf(key))).has(key); + } + + /** The parsed document, or null if it does not exist. The same object is returned on every call. */ + async readJson(key) { + if (!this.#documents.has(key)) { + let object = await this.#bucket.get(key); + let serialized = object ? await object.text() : null; + this.#documents.set(key, { data: serialized === null ? null : JSON.parse(serialized), serialized }); + } + return this.#documents.get(key).data; + } + + /** @returns {Promise} whether anything was written */ + async writeJson(key, data, { cacheControl } = {}) { + let serialized = JSON.stringify(data); + if (this.#documents.get(key)?.serialized === serialized) + return false; + + await this.#bucket.put(key, serialized, { httpMetadata: { contentType: 'application/json', cacheControl } }); + this.#documents.set(key, { data, serialized }); + this.#noteWritten(key); + return true; + } + + /** @returns {Promise} */ + async readBytes(key) { + let object = await this.#bucket.get(key); + return object ? new Uint8Array(await object.arrayBuffer()) : null; + } + + async writeText(key, text, { contentType = contentTypeFor(key), cacheControl } = {}) { + await this.#bucket.put(key, text, { httpMetadata: { contentType, cacheControl } }); + this.#noteWritten(key); + } + + async writeBytes(key, bytes, { contentType = contentTypeFor(key), cacheControl } = {}) { + await this.#bucket.put(key, bytes, { httpMetadata: { contentType, cacheControl } }); + this.#noteWritten(key); + } +} diff --git a/src/common/storage/FilesystemStorage.js b/src/common/storage/FilesystemStorage.js new file mode 100644 index 0000000..817c4ad --- /dev/null +++ b/src/common/storage/FilesystemStorage.js @@ -0,0 +1,68 @@ +// Storage over a local directory. This is what npm run splatnet uses: keys map to paths +// under the root, so dist/ and storage/ end up exactly as they always have. HTTP metadata +// is accepted and ignored. + +import { promises as fs } from 'node:fs'; +import path from 'node:path'; + +export default class FilesystemStorage { + constructor(root) { + this.root = path.resolve(root); + } + + pathFor(key) { + return path.join(this.root, ...key.split('/')); + } + + async exists(key) { + try { + return (await fs.stat(this.pathFor(key))).isFile(); + } catch (error) { + if (error.code === 'ENOENT') + return false; + throw error; + } + } + + async #read(key) { + try { + return await fs.readFile(this.pathFor(key)); + } catch (error) { + if (error.code === 'ENOENT' || error.code === 'EISDIR') + return null; + throw error; + } + } + + async #write(key, body) { + let filename = this.pathFor(key); + await fs.mkdir(path.dirname(filename), { recursive: true }); + await fs.writeFile(filename, body); + } + + /** The parsed document, or null if it does not exist. */ + async readJson(key) { + let buffer = await this.#read(key); + return buffer === null ? null : JSON.parse(buffer.toString('utf8')); + } + + /** @returns {Promise} always true; the filesystem write is cheap enough not to dedupe */ + async writeJson(key, data) { + await this.#write(key, JSON.stringify(data)); + return true; + } + + /** @returns {Promise} */ + async readBytes(key) { + let buffer = await this.#read(key); + return buffer === null ? null : new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + } + + async writeText(key, text) { + await this.#write(key, text); + } + + async writeBytes(key, bytes) { + await this.#write(key, bytes); + } +} diff --git a/src/common/storage/MemoryBucket.js b/src/common/storage/MemoryBucket.js new file mode 100644 index 0000000..dd9c42e --- /dev/null +++ b/src/common/storage/MemoryBucket.js @@ -0,0 +1,61 @@ +// In-memory bucket with the subset of the R2 binding API the app uses. For tests. + +function toBytes(value) { + if (typeof value === 'string') + return new TextEncoder().encode(value); + if (value instanceof ArrayBuffer) + return new Uint8Array(value); + if (ArrayBuffer.isView(value)) + return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + throw new TypeError('MemoryBucket.put: unsupported value type'); +} + +export default class MemoryBucket { + objects = new Map; + + async head(key) { + let object = this.objects.get(key); + return object ? { key, size: object.bytes.byteLength, uploaded: object.uploaded, httpMetadata: object.httpMetadata } : null; + } + + async get(key) { + let object = this.objects.get(key); + if (!object) + return null; + let bytes = object.bytes; + return { + key, + size: bytes.byteLength, + uploaded: object.uploaded, + httpMetadata: object.httpMetadata, + json: async () => JSON.parse(new TextDecoder().decode(bytes)), + text: async () => new TextDecoder().decode(bytes), + arrayBuffer: async () => bytes.slice().buffer, + }; + } + + async put(key, value, options = {}) { + let bytes = value instanceof ReadableStream + ? new Uint8Array(await new Response(value).arrayBuffer()) + : toBytes(value); + this.objects.set(key, { bytes, uploaded: new Date, httpMetadata: options.httpMetadata ?? {} }); + return { key }; + } + + async delete(keys) { + for (let key of [].concat(keys)) + this.objects.delete(key); + } + + async list({ prefix = '', cursor, limit = 1000 } = {}) { + let keys = [...this.objects.keys()].filter(key => key.startsWith(prefix)).sort(); + let start = cursor ? Number(cursor) : 0; + let page = keys.slice(start, start + limit); + let truncated = start + limit < keys.length; + return { + objects: page.map(key => ({ key, size: this.objects.get(key).bytes.byteLength, uploaded: this.objects.get(key).uploaded })), + truncated, + cursor: truncated ? String(start + limit) : undefined, + }; + } +} diff --git a/src/common/storage/index.js b/src/common/storage/index.js new file mode 100644 index 0000000..bcc0669 --- /dev/null +++ b/src/common/storage/index.js @@ -0,0 +1,22 @@ +export { default as MemoryBucket } from './MemoryBucket.js'; +export { default as BucketStorage } from './BucketStorage.js'; +export { default as FilesystemStorage } from './FilesystemStorage.js'; + +// What the S3 sync applied to data/ objects; the R2 objects carry it directly. +export const DATA_CACHE_CONTROL = 'no-cache, stale-while-revalidate=5, stale-if-error=86400'; + +const CONTENT_TYPES = { + json: 'application/json', + ics: 'text/calendar', + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + svg: 'image/svg+xml', + webp: 'image/webp', +}; + +export function contentTypeFor(key) { + let extension = key.split('.').pop().toLowerCase(); + return CONTENT_TYPES[extension]; +} diff --git a/test/storage/BucketStorage.test.js b/test/storage/BucketStorage.test.js new file mode 100644 index 0000000..40f4fce --- /dev/null +++ b/test/storage/BucketStorage.test.js @@ -0,0 +1,70 @@ +import { test, mock } from 'node:test'; +import assert from 'node:assert/strict'; +import { MemoryBucket, BucketStorage } from '../../src/common/storage/index.js'; + +test('exists lists a directory once and remembers what it wrote', async () => { + const bucket = new MemoryBucket; + await bucket.put('assets/splatnet/images/stage/a.png', 'a'); + await bucket.put('assets/splatnet/images/weapon/w.png', 'w'); + const list = mock.method(bucket, 'list'); + const head = mock.method(bucket, 'head'); + const storage = new BucketStorage(bucket); + + assert.equal(await storage.exists('assets/splatnet/images/stage/a.png'), true); + assert.equal(await storage.exists('assets/splatnet/images/stage/b.png'), false); + assert.equal(await storage.exists('assets/splatnet/images/weapon/w.png'), true); + assert.deepEqual(list.mock.calls.map(c => c.arguments[0].prefix), ['assets/splatnet/images/stage/', 'assets/splatnet/images/weapon/']); + assert.equal(head.mock.calls.length, 0); + + await storage.writeBytes('assets/splatnet/images/stage/b.png', new Uint8Array([1])); + assert.equal(await storage.exists('assets/splatnet/images/stage/b.png'), true); + assert.equal(list.mock.calls.length, 2); +}); + +test('readJson parses once and returns the same object, null for missing keys', async () => { + const bucket = new MemoryBucket; + await bucket.put('data/x.json', JSON.stringify({ a: 1 })); + const get = mock.method(bucket, 'get'); + const storage = new BucketStorage(bucket); + + const first = await storage.readJson('data/x.json'); + const second = await storage.readJson('data/x.json'); + assert.deepEqual(first, { a: 1 }); + assert.equal(first, second); + assert.equal(await storage.readJson('data/missing.json'), null); + assert.equal(await storage.readJson('data/missing.json'), null); + assert.equal(get.mock.calls.length, 2); +}); + +test('writeJson skips unchanged content, writes changed content with metadata, and updates the cache', async () => { + const bucket = new MemoryBucket; + await bucket.put('data/x.json', JSON.stringify({ a: 1 })); + const put = mock.method(bucket, 'put'); + const storage = new BucketStorage(bucket); + + const doc = await storage.readJson('data/x.json'); + assert.equal(await storage.writeJson('data/x.json', { a: 1 }), false); + assert.equal(put.mock.calls.length, 0); + + doc.a = 2; // mutate the shared object, as the localization processor does + assert.equal(await storage.writeJson('data/x.json', doc, { cacheControl: 'no-cache' }), true); + assert.equal(put.mock.calls.length, 1); + assert.deepEqual((await bucket.get('data/x.json')).httpMetadata, { contentType: 'application/json', cacheControl: 'no-cache' }); + assert.deepEqual(await (await bucket.get('data/x.json')).json(), { a: 2 }); + assert.equal(await storage.writeJson('data/x.json', { a: 2 }), false); + + assert.equal(await storage.writeJson('data/new.json', { fresh: true }), true); + assert.equal(await storage.exists('data/new.json'), true); + assert.deepEqual(await storage.readJson('data/new.json'), { fresh: true }); +}); + +test('writeText and readBytes go straight through with content types by extension', async () => { + const bucket = new MemoryBucket; + const storage = new BucketStorage(bucket); + await storage.writeText('data/cal.ics', 'BEGIN:VCALENDAR', { cacheControl: 'no-cache' }); + assert.deepEqual((await bucket.get('data/cal.ics')).httpMetadata, { contentType: 'text/calendar', cacheControl: 'no-cache' }); + await storage.writeBytes('assets/a.png', new Uint8Array([9, 9])); + assert.equal((await bucket.get('assets/a.png')).httpMetadata.contentType, 'image/png'); + assert.deepEqual(await storage.readBytes('assets/a.png'), new Uint8Array([9, 9])); + assert.equal(await storage.readBytes('assets/none.png'), null); +}); diff --git a/test/storage/MemoryBucket.test.js b/test/storage/MemoryBucket.test.js new file mode 100644 index 0000000..d50fe6b --- /dev/null +++ b/test/storage/MemoryBucket.test.js @@ -0,0 +1,35 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { MemoryBucket } from '../../src/common/storage/index.js'; + +// MemoryBucket stands in for the R2 binding in tests, so its list() must paginate like R2's. +test('list filters by prefix, sorts, and paginates with cursors', async () => { + const bucket = new MemoryBucket; + for (const key of ['assets/images/stage/b.png', 'assets/images/stage/a.png', 'assets/images/stage/deep/c.png', 'assets/images/weapon/w.png', 'data/x.json']) + await bucket.put(key, 'x'); + + const all = await bucket.list({ prefix: 'assets/images/stage/' }); + assert.deepEqual(all.objects.map(o => o.key), ['assets/images/stage/a.png', 'assets/images/stage/b.png', 'assets/images/stage/deep/c.png']); + assert.equal(all.truncated, false); + + const first = await bucket.list({ prefix: 'assets/', limit: 2 }); + assert.equal(first.truncated, true); + const second = await bucket.list({ prefix: 'assets/', limit: 2, cursor: first.cursor }); + assert.deepEqual([...first.objects, ...second.objects].map(o => o.key).length, 4); + assert.equal(second.truncated, false); +}); + +test('put and get round-trip strings, bytes, and streams with metadata', async () => { + const bucket = new MemoryBucket; + await bucket.put('a.json', '{"a":1}', { httpMetadata: { contentType: 'application/json' } }); + await bucket.put('b.png', new Uint8Array([1, 2])); + await bucket.put('c.png', new Response(new Uint8Array([3])).body); + + assert.deepEqual(await (await bucket.get('a.json')).json(), { a: 1 }); + assert.equal((await bucket.head('a.json')).httpMetadata.contentType, 'application/json'); + assert.deepEqual(new Uint8Array(await (await bucket.get('b.png')).arrayBuffer()), new Uint8Array([1, 2])); + assert.deepEqual(new Uint8Array(await (await bucket.get('c.png')).arrayBuffer()), new Uint8Array([3])); + assert.equal(await bucket.get('missing'), null); + await bucket.delete('a.json'); + assert.equal(await bucket.head('a.json'), null); +}); diff --git a/test/storage/storage-contract.test.js b/test/storage/storage-contract.test.js new file mode 100644 index 0000000..84ee734 --- /dev/null +++ b/test/storage/storage-contract.test.js @@ -0,0 +1,51 @@ +import { describe, test, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { MemoryBucket, BucketStorage, FilesystemStorage } from '../../src/common/storage/index.js'; + +// One contract for the storage interface the updaters use, run against every implementation. +const implementations = { + BucketStorage: { + create: async () => new BucketStorage(new MemoryBucket), + destroy: async () => {}, + }, + FilesystemStorage: { + create: async () => new FilesystemStorage(await mkdtemp(path.join(tmpdir(), 'storage-'))), + destroy: storage => rm(storage.root, { recursive: true, force: true }), + }, +}; + +for (const [name, { create, destroy }] of Object.entries(implementations)) { + describe(name, () => { + let storage; + beforeEach(async () => { storage = await create(); }); + afterEach(() => destroy(storage)); + + test('missing keys do not exist and read as null', async () => { + assert.equal(await storage.exists('data/missing.json'), false); + assert.equal(await storage.readJson('data/missing.json'), null); + assert.equal(await storage.readBytes('assets/missing.png'), null); + }); + + test('json round-trips and becomes visible to exists', async () => { + await storage.writeJson('data/nested/a.json', { hello: 'world' }); + assert.equal(await storage.exists('data/nested/a.json'), true); + assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'world' }); + + await storage.writeJson('data/nested/a.json', { hello: 'again' }); + assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'again' }); + }); + + test('bytes and text round-trip', async () => { + await storage.writeBytes('assets/x.png', new Uint8Array([1, 2, 3])); + assert.deepEqual(await storage.readBytes('assets/x.png'), new Uint8Array([1, 2, 3])); + assert.equal(await storage.exists('assets/x.png'), true); + + await storage.writeText('data/cal.ics', 'BEGIN:VCALENDAR'); + assert.equal(await storage.exists('data/cal.ics'), true); + assert.equal(new TextDecoder().decode(await storage.readBytes('data/cal.ics')), 'BEGIN:VCALENDAR'); + }); + }); +} diff --git a/test/updater/LocalizationProcessor.test.js b/test/updater/LocalizationProcessor.test.js new file mode 100644 index 0000000..8cc6d6f --- /dev/null +++ b/test/updater/LocalizationProcessor.test.js @@ -0,0 +1,54 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mock } from 'node:test'; +import { MemoryBucket, BucketStorage } from '../../src/common/storage/index.js'; +import LocalizationProcessor from '../../src/app/updater/LocalizationProcessor.js'; + +const en = { region: 'NA', language: 'en' }; +const ja = { region: 'JP', language: 'ja' }; +const stages = { name: 'stages', entities: ['$..stage_a', '$..stage_b'], id: 'id', values: 'name' }; +const rules = { name: 'rules', entities: '$..rule', id: 'key', values: ['name', 'multiline_name'] }; +const gear = { name: 'gear', entities: '$..gear', id: ['kind', 'id'], values: 'name' }; + +const data = { + regular: [{ stage_a: { id: '0', name: 'The Reef' }, stage_b: { id: '22', name: 'Skipper Pavilion' }, rule: { key: 'turf_war', name: 'Turf War', multiline_name: 'Turf\nWar' } }], +}; + +const json = async (bucket, key) => (await bucket.get(key)).json(); + +test('records localized values under name/ids/value and reports what is missing', async () => { + const bucket = new MemoryBucket; + await bucket.put('data/locale/en.json', JSON.stringify({ stages: { 0: { name: 'The Reef' } } })); + const storage = new BucketStorage(bucket); + + const stageProcessor = new LocalizationProcessor(stages, en, storage); + assert.equal(await stageProcessor.hasLocalizations(data), false); // stage 22 unknown + await stageProcessor.updateLocalizations(data); + assert.equal(await stageProcessor.hasLocalizations(data), true); + + const ruleProcessor = new LocalizationProcessor(rules, en, storage); + assert.equal(await ruleProcessor.hasLocalizations(data), false); + await ruleProcessor.updateLocalizations(data); + + assert.deepEqual(await json(bucket, 'data/locale/en.json'), { + stages: { 0: { name: 'The Reef' }, 22: { name: 'Skipper Pavilion' } }, + rules: { turf_war: { name: 'Turf War', multiline_name: 'Turf\nWar' } }, + }); + assert.equal((await bucket.head('data/locale/en.json')).httpMetadata.cacheControl, 'no-cache, stale-while-revalidate=5, stale-if-error=86400'); +}); + +test('supports multi-part ids and creates a missing document', async () => { + const bucket = new MemoryBucket; + const processor = new LocalizationProcessor(gear, en, new BucketStorage(bucket)); + await processor.updateLocalizations({ merchandises: [{ gear: { kind: 'head', id: '5', name: 'Fake Hat' } }] }); + assert.deepEqual(await json(bucket, 'data/locale/en.json'), { gear: { head: { 5: { name: 'Fake Hat' } } } }); +}); + +test('does not rewrite a document that already has every string', async () => { + const bucket = new MemoryBucket; + await bucket.put('data/locale/en.json', JSON.stringify({ stages: { 0: { name: 'The Reef' }, 22: { name: 'Skipper Pavilion' } } })); + const puts = mock.method(bucket, 'put'); + + await new LocalizationProcessor(stages, en, new BucketStorage(bucket)).updateLocalizations(data); + assert.equal(puts.mock.calls.length, 0); +}); diff --git a/test/updater/Updater.test.js b/test/updater/Updater.test.js new file mode 100644 index 0000000..2c18240 --- /dev/null +++ b/test/updater/Updater.test.js @@ -0,0 +1,109 @@ +import { test, beforeEach, afterEach, mock } from 'node:test'; +import assert from 'node:assert/strict'; +import Updater from '../../src/app/updater/updaters/Updater.js'; +import { fakeSplatNet, buckets, json, keys, setSessionEnvironment } from './support.js'; + +setSessionEnvironment(); + +const NAMES = { + en: { thing: 'Widget', other: 'Gadget' }, + ja: { thing: 'ウィジェット', other: 'ガジェット' }, +}; + +function thingsFor(language) { + const names = NAMES[language] ?? NAMES.en; + return { things: [{ id: '1', name: names.thing, image: '/images/things/one.png' }, { id: '2', name: names.other, image: '/images/things/two.png' }] }; +} + +class ThingsUpdater extends Updater { + constructor(buckets, options = {}) { + super({ + name: 'Things', + filename: 'things.json', + calendarFilename: 'things.ics', + request: splatnet => splatnet.getResponse('things'), + imagePaths: ['$..image'], + localization: [{ name: 'things', entities: '$.things[*]', id: 'id', values: 'name' }], + ...options, + }, buckets); + } + + getCalendarEntries(data) { + return data.things.map(thing => ({ id: `thing-${thing.id}`, title: thing.name, start_time: 3600, end_time: 7200 })); + } +} + +let b; +beforeEach(() => { b = buckets(); }); +afterEach(() => mock.restoreAll()); + +test('publishes data, locale document, and calendar with the right metadata', async () => { + const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) }); + await new ThingsUpdater(b).update(); + + assert.deepEqual(await json(b.publicBucket, 'data/things.json'), thingsFor('en')); + assert.deepEqual((await b.publicBucket.head('data/things.json')).httpMetadata, { + contentType: 'application/json', + cacheControl: 'no-cache, stale-while-revalidate=5, stale-if-error=86400', + }); + + assert.deepEqual(await json(b.publicBucket, 'data/locale/en.json'), { things: { 1: { name: 'Widget' }, 2: { name: 'Gadget' } } }); + assert.deepEqual(await json(b.publicBucket, 'data/locale/ja.json'), { things: { 1: { name: 'ウィジェット' }, 2: { name: 'ガジェット' } } }); + + const ics = await (await b.publicBucket.get('data/things.ics')).text(); + assert.match(ics, /SUMMARY:Widget/); + assert.match(ics, /X-WR-CALNAME:Things/); + assert.equal((await b.publicBucket.head('data/things.ics')).httpMetadata.contentType, 'text/calendar'); + + assert.equal(splatnet.api[0].language, 'en'); + assert.equal(splatnet.api[0].region, 'NA'); +}); + +test('fetches each language with missing strings using that region\'s session, then stops needing to', async () => { + const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) }); + await new ThingsUpdater(b).update(); + + assert.deepEqual(splatnet.api.map(r => `${r.region}/${r.language}`), [ + 'NA/en', 'EU/es', 'NA/es-MX', 'EU/fr', 'NA/fr-CA', 'EU/de', 'EU/nl', 'EU/it', 'EU/ru', 'JP/ja', + ]); + + const second = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) }); + const puts = mock.method(b.publicBucket, 'put'); + await new ThingsUpdater(b).update(); + assert.deepEqual(second.api.map(r => r.language), ['en']); + assert.deepEqual(puts.mock.calls.map(c => c.arguments[0]).filter(k => k.startsWith('data/locale/')), []); +}); + +test('mirrors only images that are missing, using one list per directory', async () => { + await b.publicBucket.put('assets/splatnet/images/things/one.png', 'existing'); + const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) }); + const list = mock.method(b.publicBucket, 'list'); + const head = mock.method(b.publicBucket, 'head'); + await new ThingsUpdater(b).update(); + + assert.deepEqual(splatnet.images.map(r => r.path), ['/images/things/two.png']); + assert.equal(splatnet.images[0].cookie, null); + assert.equal(await (await b.publicBucket.get('assets/splatnet/images/things/one.png')).text(), 'existing'); + assert.deepEqual(list.mock.calls.map(c => c.arguments[0].prefix), ['assets/splatnet/images/things/']); + assert.equal(head.mock.calls.length, 0); + assert.equal((await b.publicBucket.get('assets/splatnet/images/things/two.png')).httpMetadata.contentType, 'image/png'); +}); + +test('filters root keys, dropping hidden timeline items', async () => { + fakeSplatNet({ '/api/things': () => ({ keep: { importance: 1, x: 1 }, hidden: { importance: -1 }, extra: 'dropped' }) }); + await new ThingsUpdater(b, { rootKeys: ['keep', 'hidden', 'absent'], localization: undefined, imagePaths: undefined, calendarFilename: undefined }).update(); + assert.deepEqual(await json(b.publicBucket, 'data/things.json'), { keep: { importance: 1, x: 1 }, hidden: null, absent: null }); +}); + +test('fails the run and writes nothing when SplatNet rejects the session', async () => { + fakeSplatNet({ '/api/things': () => new Response('forbidden', { status: 403 }) }); + await assert.rejects(new ThingsUpdater(b).update(), /status 403/); + assert.deepEqual(keys(b.publicBucket), []); +}); + +test('leaves the existing locale document untouched when the update fails', async () => { + await b.publicBucket.put('data/locale/en.json', JSON.stringify({ things: { 9: { name: 'Old' } } })); + fakeSplatNet({ '/api/things': () => new Response('down', { status: 503 }) }); + await assert.rejects(new ThingsUpdater(b).update()); + assert.deepEqual(await json(b.publicBucket, 'data/locale/en.json'), { things: { 9: { name: 'Old' } } }); +}); diff --git a/test/updater/support.js b/test/updater/support.js new file mode 100644 index 0000000..415c847 --- /dev/null +++ b/test/updater/support.js @@ -0,0 +1,57 @@ +// Shared test helpers: a fake SplatNet behind global fetch, and buckets. +import { mock } from 'node:test'; +import { MemoryBucket, BucketStorage } from '../../src/common/storage/index.js'; + +export const SESSIONS = { NA: 'na-session', EU: 'eu-session', JP: 'jp-session' }; + +export function setSessionEnvironment() { + process.env.NINTENDO_SESSION_ID_NA = SESSIONS.NA; + process.env.NINTENDO_SESSION_ID_EU = SESSIONS.EU; + process.env.NINTENDO_SESSION_ID_JP = SESSIONS.JP; +} + +/** + * Install a fake SplatNet on global fetch. `routes` maps a path (e.g. "/api/schedules") to + * a handler receiving { language, region } and returning a JSON value; any "/images/" path + * returns a small PNG-ish payload. Every request is recorded. + */ +export function fakeSplatNet(routes = {}) { + const requests = []; + mock.method(globalThis, 'fetch', async (input, init = {}) => { + const url = new URL(input); + const headers = new Headers(init.headers); + const cookie = headers.get('Cookie'); + const region = Object.entries(SESSIONS).find(([, id]) => cookie === `iksm_session=${id}`)?.[0] ?? null; + const request = { path: url.pathname, language: headers.get('Accept-Language'), cookie, region }; + requests.push(request); + + if (url.pathname.startsWith('/images/')) + return new Response(new Uint8Array([0x89, 0x50, 0x4e, 0x47]), { headers: { 'content-type': 'image/png' } }); + const route = routes[url.pathname]; + if (!route) + return new Response('not found', { status: 404 }); + const result = await route(request); + return result instanceof Response ? result : Response.json(result); + }); + return { + requests, + get api() { return requests.filter(r => r.path.startsWith('/api/')); }, + get images() { return requests.filter(r => r.path.startsWith('/images/')); }, + }; +} + +/** + * Fresh in-memory buckets plus BucketStorage over each. The one object serves both as the + * updaters' storage argument and as a handle on the raw buckets for assertions. + */ +export function buckets() { + const publicBucket = new MemoryBucket, privateBucket = new MemoryBucket; + return { publicBucket, privateBucket, publicStorage: new BucketStorage(publicBucket), privateStorage: new BucketStorage(privateBucket) }; +} + +export async function json(bucket, key) { + const object = await bucket.get(key); + return object ? object.json() : null; +} + +export const keys = bucket => [...bucket.objects.keys()].sort(); diff --git a/test/updater/updaters.test.js b/test/updater/updaters.test.js new file mode 100644 index 0000000..8290691 --- /dev/null +++ b/test/updater/updaters.test.js @@ -0,0 +1,140 @@ +import { test, beforeEach, afterEach, mock } from 'node:test'; +import assert from 'node:assert/strict'; +import SchedulesUpdater from '../../src/app/updater/updaters/SchedulesUpdater.js'; +import CoopSchedulesUpdater from '../../src/app/updater/updaters/CoopSchedulesUpdater.js'; +import TimelineUpdater from '../../src/app/updater/updaters/TimelineUpdater.js'; +import MerchandisesUpdater from '../../src/app/updater/updaters/MerchandisesUpdater.js'; +import FestivalsUpdater from '../../src/app/updater/updaters/FestivalsUpdater.js'; +import OriginalGearImageUpdater from '../../src/app/updater/updaters/OriginalGearImageUpdater.js'; +import { createUpdaters } from '../../src/app/updater/updateAll.js'; +import { getTopOfCurrentHour, readJsonFile } from '../../src/common/utilities.js'; +import { fakeSplatNet, buckets, json, keys, setSessionEnvironment } from './support.js'; + +setSessionEnvironment(); + +let b; +beforeEach(() => { b = buckets(); }); +afterEach(() => mock.restoreAll()); + +const stage = (id, name) => ({ id, name, image: `/images/stage/${id}.png` }); +const rotation = (start, a, c, rule = 'turf_war') => ({ + id: start, start_time: start, end_time: start + 7200, + stage_a: a, stage_b: c, + game_mode: { key: 'regular', name: 'Regular Battle' }, + rule: { key: rule, name: 'Turf War', multiline_name: 'Turf\nWar' }, +}); + +test('createUpdaters builds every updater against the given buckets', () => { + const updaters = createUpdaters(b); + assert.deepEqual(updaters.map(u => u.options.name), [ + 'Original Gear', 'Schedules', 'Co-op Schedules', 'Timeline', 'Festivals NA', 'Festivals EU', 'Festivals JP', 'Merchandises', + ]); + assert.ok(updaters.every(u => u.publicStorage === b.publicStorage && u.privateStorage === b.privateStorage)); +}); + +test('schedules: seeds the known stages from SplatNet and records new stages once', async () => { + const reef = stage('0', 'The Reef'), fitness = stage('1', 'Musselforge Fitness'), skipper = stage('22', 'Skipper Pavilion'); + const schedules = { regular: [rotation(7200, skipper, reef), rotation(3600, reef, fitness)], gachi: [rotation(3600, reef, fitness)], league: [rotation(3600, fitness, reef)] }; + const splatnet = fakeSplatNet({ + '/api/schedules': () => schedules, + '/api/data/stages': () => ({ stages: [reef, fitness] }), + }); + await new SchedulesUpdater(b).update(); + + assert.deepEqual(await json(b.publicBucket, 'data/schedules.json'), schedules); + assert.deepEqual(await json(b.privateBucket, 'stages.json'), [ + { ...reef, first_seen: -1, first_available: -1 }, + { ...fitness, first_seen: -1, first_available: -1 }, + { ...skipper, first_seen: getTopOfCurrentHour(), first_available: 7200 }, + ]); + assert.ok(splatnet.api.some(r => r.path === '/api/data/stages')); + + const second = fakeSplatNet({ '/api/schedules': () => schedules }); + const puts = mock.method(b.privateBucket, 'put'); + await new SchedulesUpdater(b).update(); + assert.equal(second.api.filter(r => r.path === '/api/data/stages').length, 0); + assert.equal(puts.mock.calls.length, 0); +}); + +test('coop: publishes the calendar with stage and weapon details', async () => { + const coop = { + schedules: [{ start_time: 3600, end_time: 7200 }, { start_time: 90000, end_time: 93600 }], + details: [{ start_time: 3600, end_time: 7200, stage: { name: 'Spawning Grounds', image: '/images/coop_stage/a.png' }, weapons: [{ id: '0', weapon: { id: '0', name: 'Splattershot', image: '/images/weapon/0.png', sub: { id: '1', name: 'Bomb' }, special: { id: '2', name: 'Special' } } }, null] }], + }; + fakeSplatNet({ '/api/coop_schedules': () => coop }); + await new CoopSchedulesUpdater(b).update(); + + const ics = await (await b.publicBucket.get('data/coop-schedules.ics')).text(); + assert.match(ics, /SUMMARY:Salmon Run on Spawning Grounds/); + assert.match(ics, /Splattershot/); + assert.match(ics, /Random/); + assert.match(ics, /SUMMARY:Salmon Run\r?\n/); // the shift without details keeps the plain title + assert.deepEqual(await json(b.publicBucket, 'data/coop-schedules.json'), coop); +}); + +test('timeline: keeps only coop and weapon availability, dropping hidden items', async () => { + fakeSplatNet({ '/api/timeline': () => ({ coop: { importance: -1 }, weapon_availability: { importance: 1, availabilities: [] }, stats: { ignored: true } }) }); + await new TimelineUpdater(b).update(); + assert.deepEqual(await json(b.publicBucket, 'data/timeline.json'), { coop: null, weapon_availability: { importance: 1, availabilities: [] } }); +}); + +test('merchandises: attaches original gear from the bundled data without its brand', async () => { + const gearData = readJsonFile(new URL('../../src/common/data/gear.json', import.meta.url)); + const original = gearData.head[0]; + const brand = { id: '0', name: 'Brand', image: '/images/brand/0.png', frequent_skill: { id: '0', name: 'Skill', image: '/images/skill/0.png' } }; + const skill = { id: '0', name: 'Skill', image: '/images/skill/0.png' }; + fakeSplatNet({ '/api/onlineshop/merchandises': () => ({ + merchandises: [ + { end_time: 7200, gear: { kind: 'head', id: '1', name: original.name.toUpperCase(), image: '/images/gear/1.png', brand }, skill }, + { end_time: 7200, gear: { kind: 'head', id: '2', name: 'No Such Gear', image: '/images/gear/2.png', brand }, skill }, + ], + extra: 'dropped', + }) }); + await new MerchandisesUpdater(b).update(); + + const data = await json(b.publicBucket, 'data/merchandises.json'); + assert.deepEqual(Object.keys(data), ['merchandises']); + assert.equal(data.merchandises[0].original_gear.name, original.name); + assert.equal('brand' in data.merchandises[0].original_gear, false); + assert.ok(data.merchandises[0].original_gear.skill); + assert.equal(data.merchandises[1].original_gear, null); +}); + +test('festivals: merges one region into the shared file, fetches missing rankings, uses regional languages', async () => { + await b.publicBucket.put('data/festivals.json', JSON.stringify({ eu: { festivals: [], results: [] } })); + await b.publicBucket.put('data/festivals/na-1-rankings.json', JSON.stringify({ existing: true })); + const festival = id => ({ festival_id: id, names: { alpha_short: 'A', bravo_short: 'B' }, times: { start: 3600, end: 7200 }, images: { alpha: '/images/festival/a.png', bravo: '/images/festival/b.png', panel: '/images/festival/p.png' }, special_stage: { id: '100', name: 'Shifty Station', image: '/images/stage/100.png' } }); + const splatnet = fakeSplatNet({ + '/api/festivals/active': () => ({ festivals: [festival(2)] }), + '/api/festivals/pasts': () => ({ festivals: [festival(1)], results: [{ festival_id: 1 }, { festival_id: 2 }] }), + '/api/festivals/2/rankings': () => ({ rankings: 'two' }), + }); + await new FestivalsUpdater('NA', b).update(); + + const data = await json(b.publicBucket, 'data/festivals.json'); + assert.deepEqual(Object.keys(data).sort(), ['eu', 'na']); + assert.deepEqual(data.na.festivals.map(f => f.festival_id), [2, 1]); + assert.deepEqual(await json(b.publicBucket, 'data/festivals/na-1-rankings.json'), { existing: true }); + assert.deepEqual(await json(b.publicBucket, 'data/festivals/na-2-rankings.json'), { rankings: 'two' }); + assert.deepEqual(splatnet.api.filter(r => r.path.endsWith('/rankings')).map(r => r.path), ['/api/festivals/2/rankings']); + + // Only this region's languages are localized (NA: en, es-MX, fr-CA), all with the NA session + const localized = splatnet.api.filter(r => r.path === '/api/festivals/active'); + assert.deepEqual(localized.map(r => `${r.region}/${r.language}`), ['NA/en', 'NA/es-MX', 'NA/fr-CA']); + + const ics = await (await b.publicBucket.get('data/festivals-na.ics')).text(); + assert.match(ics, /X-WR-CALNAME:Splatfests \(NA\)/); + assert.match(ics, /SUMMARY:NA Splatfest: A vs. B/); +}); + +test('original gear: mirrors skill images from the bundled skills data', async () => { + const skills = readJsonFile(new URL('../../src/common/data/skills.json', import.meta.url)); + const images = Object.values(skills).map(s => s.image).filter(Boolean); + await b.publicBucket.put(`assets/splatnet${images[0]}`, 'existing'); + const splatnet = fakeSplatNet(); + await new OriginalGearImageUpdater(b).update(); + + assert.equal(splatnet.images.length, images.length - 1); + assert.equal(keys(b.publicBucket).length, images.length); + assert.ok(keys(b.publicBucket).every(k => k.startsWith('assets/splatnet/images/skill/'))); +});