Leahnaya/splatoon2.ink
1
0
Fork 0
mirror of https://github.com/misenhower/splatoon2.ink.git synced 2026-09-28 04:06:52 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move updaters onto public and private storage

Browse Source 
Updaters take a storage object instead of touching the filesystem: a
publicStorage (what the site serves: data/, assets/) and a privateStorage
(updater state). Two implementations share one small interface
(exists / readJson / writeJson / readBytes / writeText / writeBytes):

- FilesystemStorage over a directory, used by npm run splatnet, so dist/
  and storage/ end up exactly as they always have and the S3 sync is
  untouched;
- BucketStorage over an R2 bucket binding (MemoryBucket in tests), with a
  thin per-run cache: exists() lists a directory once and answers from
  memory, readJson() parses a document once, and writeJson() skips the put
  when the content is unchanged.

The updater base class and localization processor keep their original
structure with awaits added; the processor reads its document lazily and
writes once per update, as splatoon3.ink's does. Objects written to a
bucket carry the content type and cache-control the S3 sync used to apply.

Tests cover the storage contract (run against both implementations), the
BucketStorage cache behaviour, MemoryBucket's R2-style listing, the
localization processor, the updater base, and each concrete updater.
scripts/compare-data.mjs compares two data directories ignoring SplatNet's
random ordering and ICS timestamps.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Matt Isenhower
2026-09-04 20:12:56 -07:00
parent 2c636257f3
commit 19d1fb5d3d
26 changed files with 975 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
readme.md
View File

@@ -35,6 +35,11 @@ npm run serve # Start the webpack dev server
Data retrieved from Splatnet is stored in the `dist/data` directory.
One image, `src/common/cdn/images/skill/53c62995f9d2dc4a60f3850c5dbdd2323f1eef87.png`,
is no longer served by SplatNet's CDN. It is already in production storage; if you ever
start from an empty `dist/`, copy it to `dist/assets/splatnet/images/skill/` by hand
(or run `npm run sync:download`).
By default, the dev server will run on port 8080.
When running `npm run serve` you can access the site by going to http://localhost:8080.

77
scripts/compare-data.mjs Normal file
View File

@@ -0,0 +1,77 @@
// Compare two data directories (e.g. a saved baseline and dist/data) the way the site
// cares about: SplatNet returns object keys and some arrays in random order per request,
// and ICS files carry a per-run DTSTAMP, so byte diffs are noise. Usage:
//
// node scripts/compare-data.mjs <baseline-dir> <current-dir>
//
// Exits non-zero when a real difference is found.
import fs from 'node:fs';
import path from 'node:path';
const [base, current] = process.argv.slice(2);
if (!base || !current) {
console.error('Usage: node scripts/compare-data.mjs <baseline-dir> <current-dir>');
process.exit(2);
}
const canon = value => Array.isArray(value)
? value.map(canon)
: (value && typeof value === 'object')
? Object.fromEntries(Object.keys(value).sort().map(key => [key, canon(value[key])]))
: value;
function firstDifference(a, b, at = '$') {
if (Array.isArray(a) && Array.isArray(b)) {
if (a.length !== b.length)
return `${at}.length ${a.length} vs ${b.length}`;
// Festival result lists come back in random order
if (a.length && a[0] && typeof a[0] === 'object' && 'festival_id' in a[0]) {
let key = item => JSON.stringify(canon(item));
let sa = a.map(key).sort(), sb = b.map(key).sort();
return sa.every((v, i) => v === sb[i]) ? null : `${at}: set of items differs`;
}
for (let i = 0; i < a.length; i++) {
let d = firstDifference(a[i], b[i], `${at}[${i}]`);
if (d) return d;
}
return null;
}
if (a && b && typeof a === 'object' && typeof b === 'object') {
for (let key of new Set([...Object.keys(a), ...Object.keys(b)])) {
let d = firstDifference(a[key], b[key], `${at}.${key}`);
if (d) return d;
}
return null;
}
return a === b ? null : `${at}: ${JSON.stringify(a)?.slice(0, 60)} vs ${JSON.stringify(b)?.slice(0, 60)}`;
}
const walk = dir => fs.readdirSync(dir, { withFileTypes: true })
.flatMap(entry => entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]);
let problems = 0;
for (let file of walk(base)) {
let relative = path.relative(base, file);
let other = path.join(current, relative);
if (!fs.existsSync(other)) {
console.log(`${relative}: missing in ${current}`);
problems++;
continue;
}
let a = fs.readFileSync(file), b = fs.readFileSync(other);
if (a.equals(b))
continue;
if (relative.endsWith('.json')) {
let d = firstDifference(JSON.parse(a), JSON.parse(b));
if (d) { console.log(`${relative}: ${d}`); problems++; }
} else if (relative.endsWith('.ics')) {
let strip = s => s.toString().split(/\r?\n/).filter(line => !line.startsWith('DTSTAMP')).join('\n');
if (strip(a) !== strip(b)) { console.log(`${relative}: calendar content differs`); problems++; }
} else {
console.log(`${relative}: binary content differs`);
problems++;
}
}
console.log(problems ? `${problems} real difference(s)` : 'semantically identical');
process.exit(problems ? 1 : 0);

3
src/app/cron.js
View File

@@ -1,12 +1,13 @@
import '../common/bootstrap.js';
import { CronJob } from 'cron';
import { updateAll } from './updater/index.js';
import { filesystemStorage } from './storage.js';
import { maybePostTweets } from './twitter/index.js';
console.info('Starting periodic tasks...');
// Run every hour at 10sec after the hour
new CronJob('10 0 * * * *', async () => {
await updateAll();
await updateAll(filesystemStorage());
await maybePostTweets();
}, null, true);

3
src/app/index.js
View File

@@ -1,11 +1,12 @@
import '../common/bootstrap.js';
import { runAction } from '../common/cli.js';
import { updateAll } from './updater/index.js';
import { filesystemStorage } from './storage.js';
import { maybePostTweets, testScreenshots } from './twitter/index.js';
import { sync, syncUpload, syncDownload } from './sync/index.js';
await runAction({
splatnet: updateAll,
splatnet: () => updateAll(filesystemStorage()),
twitter: maybePostTweets,
twitterTest: testScreenshots,
sync,

12
src/app/storage.js Normal file
View File

@@ -0,0 +1,12 @@
import { FilesystemStorage } from '../common/storage/index.js';
/**
* The storage a local (Node) run uses: the same directories the site has always been
* built from, so `npm run splatnet` and the S3 sync keep working unchanged.
*/
export function filesystemStorage() {
return {
publicStorage: new FilesystemStorage('dist'),
privateStorage: new FilesystemStorage('storage'),
};
}

100
src/app/updater/LocalizationProcessor.js
View File

@@ -1,54 +1,41 @@
import path from 'node:path';
import fs from 'node:fs';
import { mkdirpSync as mkdirp } from 'mkdirp';
import jsonpath from '../../common/jsonpath.js';
import _ from 'lodash';
import { readJson, writeJson } from '../../common/utilities.js';
const localizationsPath = path.resolve('dist/data/locale');
import { DATA_CACHE_CONTROL } from '../../common/storage/index.js';
export default class LocalizationProcessor {
constructor(ruleset, languageInfo) {
/**
* @param {object} ruleset { name, entities, id, values }
* @param {{ region: string, language: string }} languageInfo
* @param {object} storage the public storage (BucketStorage or FilesystemStorage)
*/
constructor(ruleset, languageInfo, storage) {
this.ruleset = ruleset;
this.languageInfo = languageInfo;
this.storage = storage;
let entities = this.ruleset.entities;
this.entityExpressions = (Array.isArray(entities)) ? entities : [entities];
let values = this.ruleset.values;
this.valueExpressions = (Array.isArray(values)) ? values : [values];
this.readData();
}
getFilename() {
return `${localizationsPath}/${this.languageInfo.language}.json`;
getKey() {
return `data/locale/${this.languageInfo.language}.json`;
}
readData() {
if (fs.existsSync(this.getFilename()))
this.data = readJson(this.getFilename());
else
this.data = {};
async readData() {
return await this.storage.readJson(this.getKey()) ?? {};
}
writeData() {
mkdirp(path.dirname(this.getFilename()));
writeJson(this.getFilename(), this.data);
writeData(data) {
return this.storage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL });
}
getExpression(ids, valueKey) {
return [this.ruleset.name, ...ids, valueKey];
}
readValue(ids, valueKey) {
return _.get(this.data, this.getExpression(ids, valueKey));
}
writeValue(ids, valueKey, newValue) {
return _.setWith(this.data, this.getExpression(ids, valueKey), newValue, Object);
}
getIdValues(entity) {
let idParts = this.ruleset.id;
@@ -60,44 +47,31 @@ export default class LocalizationProcessor {
return idParts.map(id => _.get(entity, id));
}
eachEntity(data, callback) {
for (let expression of this.entityExpressions) {
let entities = jsonpath.query(data, expression);
*entities(data) {
for (let expression of this.entityExpressions)
yield* jsonpath.query(data, expression);
}
for (let entity of entities) {
let result = callback(entity);
if (result === false)
async updateLocalizations(data) {
let localizations = await this.readData();
for (let entity of this.entities(data)) {
let ids = this.getIdValues(entity);
for (let valueKey of this.valueExpressions)
_.setWith(localizations, this.getExpression(ids, valueKey), _.get(entity, valueKey), Object);
}
await this.writeData(localizations);
}
async hasLocalizations(data) {
let localizations = await this.readData();
for (let entity of this.entities(data)) {
let ids = this.getIdValues(entity);
for (let valueKey of this.valueExpressions)
if (_.get(localizations, this.getExpression(ids, valueKey)) === undefined)
return false;
}
}
}
updateLocalizations(data) {
this.eachEntity(data, entity => this.updateLocalizationsForEntity(entity));
}
updateLocalizationsForEntity(entity) {
for (let valueKey of this.valueExpressions) {
let ids = this.getIdValues(entity);
let value = _.get(entity, valueKey);
this.writeValue(ids, valueKey, value);
}
this.writeData();
}
hasLocalizations(data) {
let result = this.eachEntity(data, entity => this.hasLocalizationsForEntity(entity));
return (result !== false);
}
hasLocalizationsForEntity(entity) {
for (let valueKey of this.valueExpressions) {
let ids = this.getIdValues(entity);
if (this.readValue(ids, valueKey) === undefined)
return false;
}
return true;

31
src/app/updater/updateAll.js
View File

@@ -7,21 +7,28 @@ import MerchandisesUpdater from './updaters/MerchandisesUpdater.js';
import S3Syncer from '../sync/S3Syncer.js';
import { canSync } from '../sync/index.js';
const updaters = [
new OriginalGearImageUpdater,
new SchedulesUpdater,
new CoopSchedulesUpdater,
new TimelineUpdater,
new FestivalsUpdater('NA'),
new FestivalsUpdater('EU'),
new FestivalsUpdater('JP'),
new MerchandisesUpdater,
];
/** @param {{ publicStorage: object, privateStorage: object }} storage */
export function createUpdaters(storage) {
return [
new OriginalGearImageUpdater(storage),
new SchedulesUpdater(storage),
new CoopSchedulesUpdater(storage),
new TimelineUpdater(storage),
new FestivalsUpdater('NA', storage),
new FestivalsUpdater('EU', storage),
new FestivalsUpdater('JP', storage),
new MerchandisesUpdater(storage),
];
}
export default async function updateAll() {
/**
* Run every updater. `storage` is shared by all of them for the run (see createUpdaters);
* a Worker passes BucketStorage over its R2 bindings, a local run passes FilesystemStorage.
*/
export default async function updateAll(storage) {
const syncer = canSync() ? new S3Syncer() : null;
for (let updater of updaters) {
for (let updater of createUpdaters(storage)) {
try {
await updater.update();
} catch (e) {

4
src/app/updater/updaters/CoopSchedulesUpdater.js
View File

@@ -1,7 +1,7 @@
import Updater from './Updater.js';
export default class CoopSchedulesUpdater extends Updater {
constructor() {
constructor(storage) {
super({
name: 'Co-op Schedules',
filename: 'coop-schedules.json',
@@ -44,7 +44,7 @@ export default class CoopSchedulesUpdater extends Updater {
values: 'name',
},
],
});
}, storage);
}
getCalendarEntries({ details, schedules }) {

18
src/app/updater/updaters/FestivalsUpdater.js
View File

@@ -1,13 +1,12 @@
import Updater from './Updater.js';
import fs from 'node:fs';
import _ from 'lodash';
import jsonpath from '../../../common/jsonpath.js';
import { readJson, writeJson } from '../../../common/utilities.js';
import { DATA_CACHE_CONTROL } from '../../../common/storage/index.js';
import { languages } from '../../../common/regions.js';
import SplatNet from '../../../common/splatnet.js';
export default class FestivalsUpdater extends Updater {
constructor(region) {
constructor(region, storage) {
super({
name: `Festivals ${region}`,
filename: 'festivals.json',
@@ -33,7 +32,7 @@ export default class FestivalsUpdater extends Updater {
values: 'name',
},
],
});
}, storage);
this.region = region;
}
@@ -54,15 +53,15 @@ export default class FestivalsUpdater extends Updater {
// Download result ranking data
let festivalIds = jsonpath.query(regionData, '$.results..festival_id');
for (let id of festivalIds) {
let filename = `${this.getOutputPath()}/festivals/${this.region.toLowerCase()}-${id}-rankings.json`;
let key = `data/festivals/${this.region.toLowerCase()}-${id}-rankings.json`;
// Have we already downloaded these rankings?
if (!fs.existsSync(filename)) {
if (!await this.publicStorage.exists(key)) {
let splatnet = new SplatNet(this.region);
this.info(`Retrieving rankings for festival ID ${id}`);
try {
let rankings = await this.handleRequest(splatnet.getFestivalRankings(id));
writeJson(filename, rankings);
await this.publicStorage.writeJson(key, rankings, { cacheControl: DATA_CACHE_CONTROL });
}
catch (e) {
// Do nothing
@@ -71,10 +70,7 @@ export default class FestivalsUpdater extends Updater {
}
// Load existing data since we only need to modify this region's data
let data = {};
let filename = this.getFilename();
if (fs.existsSync(filename))
data = readJson(filename);
let data = await this.publicStorage.readJson(this.getKey()) ?? {};
let region = this.region.toLowerCase();
data[region] = regionData;

4
src/app/updater/updaters/MerchandisesUpdater.js
View File

@@ -2,7 +2,7 @@ import Updater from './Updater.js';
import { getOriginalGear } from '../../../common/splatoon.js';
export default class MerchandisesUpdater extends Updater {
constructor() {
constructor(storage) {
super({
name: 'Merchandises',
filename: 'merchandises.json',
@@ -40,7 +40,7 @@ export default class MerchandisesUpdater extends Updater {
values: 'name',
},
],
});
}, storage);
}
processData(data) {

10
src/app/updater/updaters/OriginalGearImageUpdater.js
View File

@@ -1,19 +1,17 @@
import Updater from './Updater.js';
import path from 'node:path';
import fs from 'node:fs';
const dataPath = path.resolve('src/common/data');
import { readJsonFile } from '../../../common/utilities.js';
export default class OriginalGearImageUpdater extends Updater {
constructor() {
constructor(storage) {
super({
name: 'Original Gear',
imagePaths: ['$..image'],
});
}, storage);
}
async update() {
// Get the list of skills
let data = JSON.parse(fs.readFileSync(`${dataPath}/skills.json`));
let data = readJsonFile(new URL('../../../common/data/skills.json', import.meta.url));
// Retrieve skill images
await this.downloadImages(data);

27
src/app/updater/updaters/SchedulesUpdater.js
View File

@@ -1,13 +1,11 @@
import Updater from './Updater.js';
import SplatNet from '../../../common/splatnet.js';
import path from 'node:path';
import fs from 'node:fs';
import { getTopOfCurrentHour, readJson } from '../../../common/utilities.js';
import { getTopOfCurrentHour } from '../../../common/utilities.js';
const stagesPath = path.resolve('storage/stages.json');
const STAGES_KEY = 'stages.json'; // private storage
export default class SchedulesUpdater extends Updater {
constructor() {
constructor(storage) {
super({
name: 'Schedules',
filename: 'schedules.json',
@@ -39,28 +37,25 @@ export default class SchedulesUpdater extends Updater {
values: ['name', 'multiline_name'],
},
],
});
}, storage);
}
async processData(data) {
// We need to track whether there are any new stages included with this schedule update
// If we don't have a "known stages" file, create it from the current list of stages
if (!fs.existsSync(stagesPath)) {
// Load known stages, or create the list from the current list of stages
let stages = await this.privateStorage.readJson(STAGES_KEY);
if (!stages) {
let splatnet = new SplatNet;
let stageData = await splatnet.getStages();
let stages = stageData.stages.map(s => Object.assign(s, { first_seen: -1, first_available: -1 }));
this.writeFile(stagesPath, JSON.stringify(stages));
stages = stageData.stages.map(s => Object.assign(s, { first_seen: -1, first_available: -1 }));
}
// Load known stages
let stages = readJson(stagesPath);
// Look for new stages (in Regular Battle)
let sortedSchedules = data.regular.sort((a, b) => a.start_time - b.start_time);
let sortedSchedules = [...data.regular].sort((a, b) => a.start_time - b.start_time);
for (let schedule of sortedSchedules) {
for (let stage of [schedule.stage_a, schedule.stage_b]) {
// Have wee seen this stage before?
// Have we seen this stage before?
let knownStage = stages.find(s => s.id == stage.id);
if (!knownStage) {
@@ -75,7 +70,7 @@ export default class SchedulesUpdater extends Updater {
}
// Update the stages file
this.writeFile(stagesPath, JSON.stringify(stages));
await this.privateStorage.writeJson(STAGES_KEY, stages);
return data;
}

4
src/app/updater/updaters/TimelineUpdater.js
View File

@@ -1,7 +1,7 @@
import Updater from './Updater.js';
export default class TimelineUpdater extends Updater {
constructor() {
constructor(storage) {
super({
name: 'Timeline',
filename: 'timeline.json',
@@ -46,6 +46,6 @@ export default class TimelineUpdater extends Updater {
values: 'name',
},
],
});
}, storage);
}
}

97
src/app/updater/updaters/Updater.js
View File

@@ -1,6 +1,3 @@
import path from 'node:path';
import fs from 'node:fs';
import { mkdirpSync as mkdirp } from 'mkdirp';
import _ from 'lodash';
import jsonpath from '../../../common/jsonpath.js';
import SplatNet from '../../../common/splatnet.js';
@@ -8,14 +5,18 @@ import { createEvents } from 'ics';
import * as Sentry from '@sentry/node';
import { languages } from '../../../common/regions.js';
import LocalizationProcessor from '../LocalizationProcessor.js';
const dataPath = path.resolve('dist/data');
const splatnetAssetPath = path.resolve('dist/assets/splatnet');
const cdnAltPath = path.resolve('src/common/cdn');
import { DATA_CACHE_CONTROL } from '../../../common/storage/index.js';
export default class Updater {
constructor(options = {}) {
/**
* @param {object} options
* @param {{ publicStorage: object, privateStorage: object }} storage
* publicStorage holds what the site serves (data/, assets/); privateStorage holds updater state.
*/
constructor(options = {}, storage = {}) {
this.options = options;
this.publicStorage = storage.publicStorage;
this.privateStorage = storage.privateStorage;
}
async update() {
@@ -36,14 +37,11 @@ export default class Updater {
// Apply any other processing
data = await this.processData(data);
// Convert the data to a JSON string
let dataString = JSON.stringify(data);
// Write the data to disk
this.writeFile(this.getFilename(), dataString);
// Write the data
await this.publicStorage.writeJson(this.getKey(), data, { cacheControl: DATA_CACHE_CONTROL });
// Update calendar events
this.updateCalendarEvents(data);
await this.updateCalendarEvents(data);
// Download images if necessary
await this.downloadImages(data);
@@ -51,17 +49,14 @@ export default class Updater {
this.info('Done.');
}
getOutputPath() {
return dataPath;
/** Key of this updater's output in the public storage */
getKey() {
return `data/${this.options.filename}`;
}
getFilename() {
return `${this.getOutputPath()}/${this.options.filename}`;
}
getCalendarFilename() {
getCalendarKey() {
if (this.options.calendarFilename)
return `${this.getOutputPath()}/${this.options.calendarFilename}`;
return `data/${this.options.calendarFilename}`;
}
getData({ region, language }) {
@@ -110,38 +105,34 @@ export default class Updater {
return _.uniqBy(languages, 'language');
}
forEachLanguage(callback) {
for (let languageInfo of this.getLanguages())
this.forEachRuleset(languageInfo, callback);
}
forEachRuleset(languageInfo, callback) {
for (let ruleset of (this.options.localization)) {
let processor = new LocalizationProcessor(ruleset, languageInfo);
callback(processor, languageInfo);
}
getProcessors(languageInfo) {
return this.options.localization.map(ruleset => new LocalizationProcessor(ruleset, languageInfo, this.publicStorage));
}
async updateLocalizations(data, initialLanguageInfo) {
if (this.options.localization) {
// Update localization data for the initial language
this.forEachRuleset(initialLanguageInfo, processor => processor.updateLocalizations(data));
for (let processor of this.getProcessors(initialLanguageInfo))
await processor.updateLocalizations(data);
// Do we need to retrieve data for any other languages?
let missingLanguages = [];
this.forEachLanguage((processor, languageInfo) => {
if (missingLanguages.indexOf(languageInfo) === -1) {
if (!processor.hasLocalizations(data))
for (let languageInfo of this.getLanguages()) {
for (let processor of this.getProcessors(languageInfo)) {
if (!await processor.hasLocalizations(data)) {
missingLanguages.push(languageInfo);
break;
}
}
});
}
// Retrieve data for missing languages
for (let missingLanguageInfo of missingLanguages) {
this.info(`Retrieving localized data for region: ${missingLanguageInfo.region}, language: ${missingLanguageInfo.language}`);
let localData = await this.handleRequest(this.getData(missingLanguageInfo));
localData = this.filterRootKeys(localData);
this.forEachRuleset(missingLanguageInfo, processor => processor.updateLocalizations(localData));
for (let processor of this.getProcessors(missingLanguageInfo))
await processor.updateLocalizations(localData);
}
}
@@ -159,11 +150,6 @@ export default class Updater {
return true;
}
writeFile(filename, data) {
mkdirp(path.dirname(filename));
fs.writeFileSync(filename, data);
}
async downloadImages(data) {
if (this.options.imagePaths) {
for (let expression of this.options.imagePaths) {
@@ -178,40 +164,31 @@ export default class Updater {
if (!imagePath)
return;
let localPath = splatnetAssetPath + imagePath;
let key = `assets/splatnet${imagePath}`;
// Check whether the image has already been downloaded
if (fs.existsSync(localPath))
if (await this.publicStorage.exists(key))
return;
// Certain images are not available on the CDN anymore
if (fs.existsSync(cdnAltPath + imagePath)) {
this.info(`Using CDN backup: ${imagePath}`);
let image = fs.readFileSync(cdnAltPath + imagePath);
this.writeFile(localPath, image);
return;
}
// Otherwise, download the image
// Download the image
this.info(`Downloading image: ${imagePath}`);
let splatnet = new SplatNet;
let image = await this.handleRequest(splatnet.getImage(imagePath));
this.writeFile(localPath, image);
await this.publicStorage.writeBytes(key, image);
}
/**
* Calendar output
*/
updateCalendarEvents(data) {
let filename = this.getCalendarFilename();
if (!filename)
async updateCalendarEvents(data) {
let key = this.getCalendarKey();
if (!key)
return;
let events = this.getCalendarEntries(data);
let ical = this.getiCalData(events);
this.writeFile(filename, ical);
await this.publicStorage.writeText(key, ical, { cacheControl: DATA_CACHE_CONTROL });
}
getCalendarTitle() {

2
src/common/splatnet.js
View File

@@ -1,5 +1,3 @@
import './bootstrap.js';
// SplatNet2 API
const userAgent = process.env.SPLATNET_USER_AGENT;
const splatnetBaseUrl = 'https://app.splatoon2.nintendo.net';

89
src/common/storage/BucketStorage.js Normal file
View File

@@ -0,0 +1,89 @@
// Storage over an R2 bucket binding (or MemoryBucket in tests), with a thin per-run cache.
//
// Callers keep the shape the filesystem code had (exists / readJson / writeJson) and this
// class keeps the round trips down because the bucket is remote:
// - exists(key) lists the key's directory once and answers from memory after that;
// - readJson(key) parses each document once and hands back the same object each time;
// - writeJson(key) skips the put when the content matches what is already stored.
// Writes go straight through. Create one instance per update run.
import { contentTypeFor } from './index.js';
function directoryOf(key) {
return key.slice(0, key.lastIndexOf('/') + 1);
}
export default class BucketStorage {
#bucket;
#listings = new Map; // directory prefix -> Set of keys under it
#documents = new Map; // key -> { data, serialized } for JSON documents (data may be null)
constructor(bucket) {
this.#bucket = bucket;
}
get bucket() {
return this.#bucket;
}
async #listing(directory) {
if (!this.#listings.has(directory)) {
let keys = new Set;
let cursor;
do {
let page = await this.#bucket.list({ prefix: directory, cursor, limit: 1000 });
for (let object of page.objects)
keys.add(object.key);
cursor = page.truncated ? page.cursor : undefined;
} while (cursor);
this.#listings.set(directory, keys);
}
return this.#listings.get(directory);
}
#noteWritten(key) {
this.#listings.get(directoryOf(key))?.add(key);
}
async exists(key) {
return (await this.#listing(directoryOf(key))).has(key);
}
/** The parsed document, or null if it does not exist. The same object is returned on every call. */
async readJson(key) {
if (!this.#documents.has(key)) {
let object = await this.#bucket.get(key);
let serialized = object ? await object.text() : null;
this.#documents.set(key, { data: serialized === null ? null : JSON.parse(serialized), serialized });
}
return this.#documents.get(key).data;
}
/** @returns {Promise<boolean>} whether anything was written */
async writeJson(key, data, { cacheControl } = {}) {
let serialized = JSON.stringify(data);
if (this.#documents.get(key)?.serialized === serialized)
return false;
await this.#bucket.put(key, serialized, { httpMetadata: { contentType: 'application/json', cacheControl } });
this.#documents.set(key, { data, serialized });
this.#noteWritten(key);
return true;
}
/** @returns {Promise<Uint8Array | null>} */
async readBytes(key) {
let object = await this.#bucket.get(key);
return object ? new Uint8Array(await object.arrayBuffer()) : null;
}
async writeText(key, text, { contentType = contentTypeFor(key), cacheControl } = {}) {
await this.#bucket.put(key, text, { httpMetadata: { contentType, cacheControl } });
this.#noteWritten(key);
}
async writeBytes(key, bytes, { contentType = contentTypeFor(key), cacheControl } = {}) {
await this.#bucket.put(key, bytes, { httpMetadata: { contentType, cacheControl } });
this.#noteWritten(key);
}
}

68
src/common/storage/FilesystemStorage.js Normal file
View File

@@ -0,0 +1,68 @@
// Storage over a local directory. This is what npm run splatnet uses: keys map to paths
// under the root, so dist/ and storage/ end up exactly as they always have. HTTP metadata
// is accepted and ignored.
import { promises as fs } from 'node:fs';
import path from 'node:path';
export default class FilesystemStorage {
constructor(root) {
this.root = path.resolve(root);
}
pathFor(key) {
return path.join(this.root, ...key.split('/'));
}
async exists(key) {
try {
return (await fs.stat(this.pathFor(key))).isFile();
} catch (error) {
if (error.code === 'ENOENT')
return false;
throw error;
}
}
async #read(key) {
try {
return await fs.readFile(this.pathFor(key));
} catch (error) {
if (error.code === 'ENOENT' || error.code === 'EISDIR')
return null;
throw error;
}
}
async #write(key, body) {
let filename = this.pathFor(key);
await fs.mkdir(path.dirname(filename), { recursive: true });
await fs.writeFile(filename, body);
}
/** The parsed document, or null if it does not exist. */
async readJson(key) {
let buffer = await this.#read(key);
return buffer === null ? null : JSON.parse(buffer.toString('utf8'));
}
/** @returns {Promise<boolean>} always true; the filesystem write is cheap enough not to dedupe */
async writeJson(key, data) {
await this.#write(key, JSON.stringify(data));
return true;
}
/** @returns {Promise<Uint8Array | null>} */
async readBytes(key) {
let buffer = await this.#read(key);
return buffer === null ? null : new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
}
async writeText(key, text) {
await this.#write(key, text);
}
async writeBytes(key, bytes) {
await this.#write(key, bytes);
}
}

61
src/common/storage/MemoryBucket.js Normal file
View File

@@ -0,0 +1,61 @@
// In-memory bucket with the subset of the R2 binding API the app uses. For tests.
function toBytes(value) {
if (typeof value === 'string')
return new TextEncoder().encode(value);
if (value instanceof ArrayBuffer)
return new Uint8Array(value);
if (ArrayBuffer.isView(value))
return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
throw new TypeError('MemoryBucket.put: unsupported value type');
}
export default class MemoryBucket {
objects = new Map;
async head(key) {
let object = this.objects.get(key);
return object ? { key, size: object.bytes.byteLength, uploaded: object.uploaded, httpMetadata: object.httpMetadata } : null;
}
async get(key) {
let object = this.objects.get(key);
if (!object)
return null;
let bytes = object.bytes;
return {
key,
size: bytes.byteLength,
uploaded: object.uploaded,
httpMetadata: object.httpMetadata,
json: async () => JSON.parse(new TextDecoder().decode(bytes)),
text: async () => new TextDecoder().decode(bytes),
arrayBuffer: async () => bytes.slice().buffer,
};
}
async put(key, value, options = {}) {
let bytes = value instanceof ReadableStream
? new Uint8Array(await new Response(value).arrayBuffer())
: toBytes(value);
this.objects.set(key, { bytes, uploaded: new Date, httpMetadata: options.httpMetadata ?? {} });
return { key };
}
async delete(keys) {
for (let key of [].concat(keys))
this.objects.delete(key);
}
async list({ prefix = '', cursor, limit = 1000 } = {}) {
let keys = [...this.objects.keys()].filter(key => key.startsWith(prefix)).sort();
let start = cursor ? Number(cursor) : 0;
let page = keys.slice(start, start + limit);
let truncated = start + limit < keys.length;
return {
objects: page.map(key => ({ key, size: this.objects.get(key).bytes.byteLength, uploaded: this.objects.get(key).uploaded })),
truncated,
cursor: truncated ? String(start + limit) : undefined,
};
}
}

22
src/common/storage/index.js Normal file
View File

@@ -0,0 +1,22 @@
export { default as MemoryBucket } from './MemoryBucket.js';
export { default as BucketStorage } from './BucketStorage.js';
export { default as FilesystemStorage } from './FilesystemStorage.js';
// What the S3 sync applied to data/ objects; the R2 objects carry it directly.
export const DATA_CACHE_CONTROL = 'no-cache, stale-while-revalidate=5, stale-if-error=86400';
const CONTENT_TYPES = {
json: 'application/json',
ics: 'text/calendar',
png: 'image/png',
jpg: 'image/jpeg',
jpeg: 'image/jpeg',
gif: 'image/gif',
svg: 'image/svg+xml',
webp: 'image/webp',
};
export function contentTypeFor(key) {
let extension = key.split('.').pop().toLowerCase();
return CONTENT_TYPES[extension];
}

70
test/storage/BucketStorage.test.js Normal file
View File

@@ -0,0 +1,70 @@
import { test, mock } from 'node:test';
import assert from 'node:assert/strict';
import { MemoryBucket, BucketStorage } from '../../src/common/storage/index.js';
test('exists lists a directory once and remembers what it wrote', async () => {
const bucket = new MemoryBucket;
await bucket.put('assets/splatnet/images/stage/a.png', 'a');
await bucket.put('assets/splatnet/images/weapon/w.png', 'w');
const list = mock.method(bucket, 'list');
const head = mock.method(bucket, 'head');
const storage = new BucketStorage(bucket);
assert.equal(await storage.exists('assets/splatnet/images/stage/a.png'), true);
assert.equal(await storage.exists('assets/splatnet/images/stage/b.png'), false);
assert.equal(await storage.exists('assets/splatnet/images/weapon/w.png'), true);
assert.deepEqual(list.mock.calls.map(c => c.arguments[0].prefix), ['assets/splatnet/images/stage/', 'assets/splatnet/images/weapon/']);
assert.equal(head.mock.calls.length, 0);
await storage.writeBytes('assets/splatnet/images/stage/b.png', new Uint8Array([1]));
assert.equal(await storage.exists('assets/splatnet/images/stage/b.png'), true);
assert.equal(list.mock.calls.length, 2);
});
test('readJson parses once and returns the same object, null for missing keys', async () => {
const bucket = new MemoryBucket;
await bucket.put('data/x.json', JSON.stringify({ a: 1 }));
const get = mock.method(bucket, 'get');
const storage = new BucketStorage(bucket);
const first = await storage.readJson('data/x.json');
const second = await storage.readJson('data/x.json');
assert.deepEqual(first, { a: 1 });
assert.equal(first, second);
assert.equal(await storage.readJson('data/missing.json'), null);
assert.equal(await storage.readJson('data/missing.json'), null);
assert.equal(get.mock.calls.length, 2);
});
test('writeJson skips unchanged content, writes changed content with metadata, and updates the cache', async () => {
const bucket = new MemoryBucket;
await bucket.put('data/x.json', JSON.stringify({ a: 1 }));
const put = mock.method(bucket, 'put');
const storage = new BucketStorage(bucket);
const doc = await storage.readJson('data/x.json');
assert.equal(await storage.writeJson('data/x.json', { a: 1 }), false);
assert.equal(put.mock.calls.length, 0);
doc.a = 2; // mutate the shared object, as the localization processor does
assert.equal(await storage.writeJson('data/x.json', doc, { cacheControl: 'no-cache' }), true);
assert.equal(put.mock.calls.length, 1);
assert.deepEqual((await bucket.get('data/x.json')).httpMetadata, { contentType: 'application/json', cacheControl: 'no-cache' });
assert.deepEqual(await (await bucket.get('data/x.json')).json(), { a: 2 });
assert.equal(await storage.writeJson('data/x.json', { a: 2 }), false);
assert.equal(await storage.writeJson('data/new.json', { fresh: true }), true);
assert.equal(await storage.exists('data/new.json'), true);
assert.deepEqual(await storage.readJson('data/new.json'), { fresh: true });
});
test('writeText and readBytes go straight through with content types by extension', async () => {
const bucket = new MemoryBucket;
const storage = new BucketStorage(bucket);
await storage.writeText('data/cal.ics', 'BEGIN:VCALENDAR', { cacheControl: 'no-cache' });
assert.deepEqual((await bucket.get('data/cal.ics')).httpMetadata, { contentType: 'text/calendar', cacheControl: 'no-cache' });
await storage.writeBytes('assets/a.png', new Uint8Array([9, 9]));
assert.equal((await bucket.get('assets/a.png')).httpMetadata.contentType, 'image/png');
assert.deepEqual(await storage.readBytes('assets/a.png'), new Uint8Array([9, 9]));
assert.equal(await storage.readBytes('assets/none.png'), null);
});

35
test/storage/MemoryBucket.test.js Normal file
View File

@@ -0,0 +1,35 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { MemoryBucket } from '../../src/common/storage/index.js';
// MemoryBucket stands in for the R2 binding in tests, so its list() must paginate like R2's.
test('list filters by prefix, sorts, and paginates with cursors', async () => {
const bucket = new MemoryBucket;
for (const key of ['assets/images/stage/b.png', 'assets/images/stage/a.png', 'assets/images/stage/deep/c.png', 'assets/images/weapon/w.png', 'data/x.json'])
await bucket.put(key, 'x');
const all = await bucket.list({ prefix: 'assets/images/stage/' });
assert.deepEqual(all.objects.map(o => o.key), ['assets/images/stage/a.png', 'assets/images/stage/b.png', 'assets/images/stage/deep/c.png']);
assert.equal(all.truncated, false);
const first = await bucket.list({ prefix: 'assets/', limit: 2 });
assert.equal(first.truncated, true);
const second = await bucket.list({ prefix: 'assets/', limit: 2, cursor: first.cursor });
assert.deepEqual([...first.objects, ...second.objects].map(o => o.key).length, 4);
assert.equal(second.truncated, false);
});
test('put and get round-trip strings, bytes, and streams with metadata', async () => {
const bucket = new MemoryBucket;
await bucket.put('a.json', '{"a":1}', { httpMetadata: { contentType: 'application/json' } });
await bucket.put('b.png', new Uint8Array([1, 2]));
await bucket.put('c.png', new Response(new Uint8Array([3])).body);
assert.deepEqual(await (await bucket.get('a.json')).json(), { a: 1 });
assert.equal((await bucket.head('a.json')).httpMetadata.contentType, 'application/json');
assert.deepEqual(new Uint8Array(await (await bucket.get('b.png')).arrayBuffer()), new Uint8Array([1, 2]));
assert.deepEqual(new Uint8Array(await (await bucket.get('c.png')).arrayBuffer()), new Uint8Array([3]));
assert.equal(await bucket.get('missing'), null);
await bucket.delete('a.json');
assert.equal(await bucket.head('a.json'), null);
});

51
test/storage/storage-contract.test.js Normal file
View File

@@ -0,0 +1,51 @@
import { describe, test, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { MemoryBucket, BucketStorage, FilesystemStorage } from '../../src/common/storage/index.js';
// One contract for the storage interface the updaters use, run against every implementation.
const implementations = {
BucketStorage: {
create: async () => new BucketStorage(new MemoryBucket),
destroy: async () => {},
},
FilesystemStorage: {
create: async () => new FilesystemStorage(await mkdtemp(path.join(tmpdir(), 'storage-'))),
destroy: storage => rm(storage.root, { recursive: true, force: true }),
},
};
for (const [name, { create, destroy }] of Object.entries(implementations)) {
describe(name, () => {
let storage;
beforeEach(async () => { storage = await create(); });
afterEach(() => destroy(storage));
test('missing keys do not exist and read as null', async () => {
assert.equal(await storage.exists('data/missing.json'), false);
assert.equal(await storage.readJson('data/missing.json'), null);
assert.equal(await storage.readBytes('assets/missing.png'), null);
});
test('json round-trips and becomes visible to exists', async () => {
await storage.writeJson('data/nested/a.json', { hello: 'world' });
assert.equal(await storage.exists('data/nested/a.json'), true);
assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'world' });
await storage.writeJson('data/nested/a.json', { hello: 'again' });
assert.deepEqual(await storage.readJson('data/nested/a.json'), { hello: 'again' });
});
test('bytes and text round-trip', async () => {
await storage.writeBytes('assets/x.png', new Uint8Array([1, 2, 3]));
assert.deepEqual(await storage.readBytes('assets/x.png'), new Uint8Array([1, 2, 3]));
assert.equal(await storage.exists('assets/x.png'), true);
await storage.writeText('data/cal.ics', 'BEGIN:VCALENDAR');
assert.equal(await storage.exists('data/cal.ics'), true);
assert.equal(new TextDecoder().decode(await storage.readBytes('data/cal.ics')), 'BEGIN:VCALENDAR');
});
});
}

54
test/updater/LocalizationProcessor.test.js Normal file
View File

@@ -0,0 +1,54 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mock } from 'node:test';
import { MemoryBucket, BucketStorage } from '../../src/common/storage/index.js';
import LocalizationProcessor from '../../src/app/updater/LocalizationProcessor.js';
const en = { region: 'NA', language: 'en' };
const ja = { region: 'JP', language: 'ja' };
const stages = { name: 'stages', entities: ['$..stage_a', '$..stage_b'], id: 'id', values: 'name' };
const rules = { name: 'rules', entities: '$..rule', id: 'key', values: ['name', 'multiline_name'] };
const gear = { name: 'gear', entities: '$..gear', id: ['kind', 'id'], values: 'name' };
const data = {
regular: [{ stage_a: { id: '0', name: 'The Reef' }, stage_b: { id: '22', name: 'Skipper Pavilion' }, rule: { key: 'turf_war', name: 'Turf War', multiline_name: 'Turf\nWar' } }],
};
const json = async (bucket, key) => (await bucket.get(key)).json();
test('records localized values under name/ids/value and reports what is missing', async () => {
const bucket = new MemoryBucket;
await bucket.put('data/locale/en.json', JSON.stringify({ stages: { 0: { name: 'The Reef' } } }));
const storage = new BucketStorage(bucket);
const stageProcessor = new LocalizationProcessor(stages, en, storage);
assert.equal(await stageProcessor.hasLocalizations(data), false); // stage 22 unknown
await stageProcessor.updateLocalizations(data);
assert.equal(await stageProcessor.hasLocalizations(data), true);
const ruleProcessor = new LocalizationProcessor(rules, en, storage);
assert.equal(await ruleProcessor.hasLocalizations(data), false);
await ruleProcessor.updateLocalizations(data);
assert.deepEqual(await json(bucket, 'data/locale/en.json'), {
stages: { 0: { name: 'The Reef' }, 22: { name: 'Skipper Pavilion' } },
rules: { turf_war: { name: 'Turf War', multiline_name: 'Turf\nWar' } },
});
assert.equal((await bucket.head('data/locale/en.json')).httpMetadata.cacheControl, 'no-cache, stale-while-revalidate=5, stale-if-error=86400');
});
test('supports multi-part ids and creates a missing document', async () => {
const bucket = new MemoryBucket;
const processor = new LocalizationProcessor(gear, en, new BucketStorage(bucket));
await processor.updateLocalizations({ merchandises: [{ gear: { kind: 'head', id: '5', name: 'Fake Hat' } }] });
assert.deepEqual(await json(bucket, 'data/locale/en.json'), { gear: { head: { 5: { name: 'Fake Hat' } } } });
});
test('does not rewrite a document that already has every string', async () => {
const bucket = new MemoryBucket;
await bucket.put('data/locale/en.json', JSON.stringify({ stages: { 0: { name: 'The Reef' }, 22: { name: 'Skipper Pavilion' } } }));
const puts = mock.method(bucket, 'put');
await new LocalizationProcessor(stages, en, new BucketStorage(bucket)).updateLocalizations(data);
assert.equal(puts.mock.calls.length, 0);
});

109
test/updater/Updater.test.js Normal file
View File

@@ -0,0 +1,109 @@
import { test, beforeEach, afterEach, mock } from 'node:test';
import assert from 'node:assert/strict';
import Updater from '../../src/app/updater/updaters/Updater.js';
import { fakeSplatNet, buckets, json, keys, setSessionEnvironment } from './support.js';
setSessionEnvironment();
const NAMES = {
en: { thing: 'Widget', other: 'Gadget' },
ja: { thing: 'ウィジェット', other: 'ガジェット' },
};
function thingsFor(language) {
const names = NAMES[language] ?? NAMES.en;
return { things: [{ id: '1', name: names.thing, image: '/images/things/one.png' }, { id: '2', name: names.other, image: '/images/things/two.png' }] };
}
class ThingsUpdater extends Updater {
constructor(buckets, options = {}) {
super({
name: 'Things',
filename: 'things.json',
calendarFilename: 'things.ics',
request: splatnet => splatnet.getResponse('things'),
imagePaths: ['$..image'],
localization: [{ name: 'things', entities: '$.things[*]', id: 'id', values: 'name' }],
...options,
}, buckets);
}
getCalendarEntries(data) {
return data.things.map(thing => ({ id: `thing-${thing.id}`, title: thing.name, start_time: 3600, end_time: 7200 }));
}
}
let b;
beforeEach(() => { b = buckets(); });
afterEach(() => mock.restoreAll());
test('publishes data, locale document, and calendar with the right metadata', async () => {
const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) });
await new ThingsUpdater(b).update();
assert.deepEqual(await json(b.publicBucket, 'data/things.json'), thingsFor('en'));
assert.deepEqual((await b.publicBucket.head('data/things.json')).httpMetadata, {
contentType: 'application/json',
cacheControl: 'no-cache, stale-while-revalidate=5, stale-if-error=86400',
});
assert.deepEqual(await json(b.publicBucket, 'data/locale/en.json'), { things: { 1: { name: 'Widget' }, 2: { name: 'Gadget' } } });
assert.deepEqual(await json(b.publicBucket, 'data/locale/ja.json'), { things: { 1: { name: 'ウィジェット' }, 2: { name: 'ガジェット' } } });
const ics = await (await b.publicBucket.get('data/things.ics')).text();
assert.match(ics, /SUMMARY:Widget/);
assert.match(ics, /X-WR-CALNAME:Things/);
assert.equal((await b.publicBucket.head('data/things.ics')).httpMetadata.contentType, 'text/calendar');
assert.equal(splatnet.api[0].language, 'en');
assert.equal(splatnet.api[0].region, 'NA');
});
test('fetches each language with missing strings using that region\'s session, then stops needing to', async () => {
const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) });
await new ThingsUpdater(b).update();
assert.deepEqual(splatnet.api.map(r => `${r.region}/${r.language}`), [
'NA/en', 'EU/es', 'NA/es-MX', 'EU/fr', 'NA/fr-CA', 'EU/de', 'EU/nl', 'EU/it', 'EU/ru', 'JP/ja',
]);
const second = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) });
const puts = mock.method(b.publicBucket, 'put');
await new ThingsUpdater(b).update();
assert.deepEqual(second.api.map(r => r.language), ['en']);
assert.deepEqual(puts.mock.calls.map(c => c.arguments[0]).filter(k => k.startsWith('data/locale/')), []);
});
test('mirrors only images that are missing, using one list per directory', async () => {
await b.publicBucket.put('assets/splatnet/images/things/one.png', 'existing');
const splatnet = fakeSplatNet({ '/api/things': ({ language }) => thingsFor(language) });
const list = mock.method(b.publicBucket, 'list');
const head = mock.method(b.publicBucket, 'head');
await new ThingsUpdater(b).update();
assert.deepEqual(splatnet.images.map(r => r.path), ['/images/things/two.png']);
assert.equal(splatnet.images[0].cookie, null);
assert.equal(await (await b.publicBucket.get('assets/splatnet/images/things/one.png')).text(), 'existing');
assert.deepEqual(list.mock.calls.map(c => c.arguments[0].prefix), ['assets/splatnet/images/things/']);
assert.equal(head.mock.calls.length, 0);
assert.equal((await b.publicBucket.get('assets/splatnet/images/things/two.png')).httpMetadata.contentType, 'image/png');
});
test('filters root keys, dropping hidden timeline items', async () => {
fakeSplatNet({ '/api/things': () => ({ keep: { importance: 1, x: 1 }, hidden: { importance: -1 }, extra: 'dropped' }) });
await new ThingsUpdater(b, { rootKeys: ['keep', 'hidden', 'absent'], localization: undefined, imagePaths: undefined, calendarFilename: undefined }).update();
assert.deepEqual(await json(b.publicBucket, 'data/things.json'), { keep: { importance: 1, x: 1 }, hidden: null, absent: null });
});
test('fails the run and writes nothing when SplatNet rejects the session', async () => {
fakeSplatNet({ '/api/things': () => new Response('forbidden', { status: 403 }) });
await assert.rejects(new ThingsUpdater(b).update(), /status 403/);
assert.deepEqual(keys(b.publicBucket), []);
});
test('leaves the existing locale document untouched when the update fails', async () => {
await b.publicBucket.put('data/locale/en.json', JSON.stringify({ things: { 9: { name: 'Old' } } }));
fakeSplatNet({ '/api/things': () => new Response('down', { status: 503 }) });
await assert.rejects(new ThingsUpdater(b).update());
assert.deepEqual(await json(b.publicBucket, 'data/locale/en.json'), { things: { 9: { name: 'Old' } } });
});

57
test/updater/support.js Normal file
View File

@@ -0,0 +1,57 @@
// Shared test helpers: a fake SplatNet behind global fetch, and buckets.
import { mock } from 'node:test';
import { MemoryBucket, BucketStorage } from '../../src/common/storage/index.js';
export const SESSIONS = { NA: 'na-session', EU: 'eu-session', JP: 'jp-session' };
export function setSessionEnvironment() {
process.env.NINTENDO_SESSION_ID_NA = SESSIONS.NA;
process.env.NINTENDO_SESSION_ID_EU = SESSIONS.EU;
process.env.NINTENDO_SESSION_ID_JP = SESSIONS.JP;
}
/**
* Install a fake SplatNet on global fetch. `routes` maps a path (e.g. "/api/schedules") to
* a handler receiving { language, region } and returning a JSON value; any "/images/" path
* returns a small PNG-ish payload. Every request is recorded.
*/
export function fakeSplatNet(routes = {}) {
const requests = [];
mock.method(globalThis, 'fetch', async (input, init = {}) => {
const url = new URL(input);
const headers = new Headers(init.headers);
const cookie = headers.get('Cookie');
const region = Object.entries(SESSIONS).find(([, id]) => cookie === `iksm_session=${id}`)?.[0] ?? null;
const request = { path: url.pathname, language: headers.get('Accept-Language'), cookie, region };
requests.push(request);
if (url.pathname.startsWith('/images/'))
return new Response(new Uint8Array([0x89, 0x50, 0x4e, 0x47]), { headers: { 'content-type': 'image/png' } });
const route = routes[url.pathname];
if (!route)
return new Response('not found', { status: 404 });
const result = await route(request);
return result instanceof Response ? result : Response.json(result);
});
return {
requests,
get api() { return requests.filter(r => r.path.startsWith('/api/')); },
get images() { return requests.filter(r => r.path.startsWith('/images/')); },
};
}
/**
* Fresh in-memory buckets plus BucketStorage over each. The one object serves both as the
* updaters' storage argument and as a handle on the raw buckets for assertions.
*/
export function buckets() {
const publicBucket = new MemoryBucket, privateBucket = new MemoryBucket;
return { publicBucket, privateBucket, publicStorage: new BucketStorage(publicBucket), privateStorage: new BucketStorage(privateBucket) };
}
export async function json(bucket, key) {
const object = await bucket.get(key);
return object ? object.json() : null;
}
export const keys = bucket => [...bucket.objects.keys()].sort();

140
test/updater/updaters.test.js Normal file
View File

@@ -0,0 +1,140 @@
import { test, beforeEach, afterEach, mock } from 'node:test';
import assert from 'node:assert/strict';
import SchedulesUpdater from '../../src/app/updater/updaters/SchedulesUpdater.js';
import CoopSchedulesUpdater from '../../src/app/updater/updaters/CoopSchedulesUpdater.js';
import TimelineUpdater from '../../src/app/updater/updaters/TimelineUpdater.js';
import MerchandisesUpdater from '../../src/app/updater/updaters/MerchandisesUpdater.js';
import FestivalsUpdater from '../../src/app/updater/updaters/FestivalsUpdater.js';
import OriginalGearImageUpdater from '../../src/app/updater/updaters/OriginalGearImageUpdater.js';
import { createUpdaters } from '../../src/app/updater/updateAll.js';
import { getTopOfCurrentHour, readJsonFile } from '../../src/common/utilities.js';
import { fakeSplatNet, buckets, json, keys, setSessionEnvironment } from './support.js';
setSessionEnvironment();
let b;
beforeEach(() => { b = buckets(); });
afterEach(() => mock.restoreAll());
const stage = (id, name) => ({ id, name, image: `/images/stage/${id}.png` });
const rotation = (start, a, c, rule = 'turf_war') => ({
id: start, start_time: start, end_time: start + 7200,
stage_a: a, stage_b: c,
game_mode: { key: 'regular', name: 'Regular Battle' },
rule: { key: rule, name: 'Turf War', multiline_name: 'Turf\nWar' },
});
test('createUpdaters builds every updater against the given buckets', () => {
const updaters = createUpdaters(b);
assert.deepEqual(updaters.map(u => u.options.name), [
'Original Gear', 'Schedules', 'Co-op Schedules', 'Timeline', 'Festivals NA', 'Festivals EU', 'Festivals JP', 'Merchandises',
]);
assert.ok(updaters.every(u => u.publicStorage === b.publicStorage && u.privateStorage === b.privateStorage));
});
test('schedules: seeds the known stages from SplatNet and records new stages once', async () => {
const reef = stage('0', 'The Reef'), fitness = stage('1', 'Musselforge Fitness'), skipper = stage('22', 'Skipper Pavilion');
const schedules = { regular: [rotation(7200, skipper, reef), rotation(3600, reef, fitness)], gachi: [rotation(3600, reef, fitness)], league: [rotation(3600, fitness, reef)] };
const splatnet = fakeSplatNet({
'/api/schedules': () => schedules,
'/api/data/stages': () => ({ stages: [reef, fitness] }),
});
await new SchedulesUpdater(b).update();
assert.deepEqual(await json(b.publicBucket, 'data/schedules.json'), schedules);
assert.deepEqual(await json(b.privateBucket, 'stages.json'), [
{ ...reef, first_seen: -1, first_available: -1 },
{ ...fitness, first_seen: -1, first_available: -1 },
{ ...skipper, first_seen: getTopOfCurrentHour(), first_available: 7200 },
]);
assert.ok(splatnet.api.some(r => r.path === '/api/data/stages'));
const second = fakeSplatNet({ '/api/schedules': () => schedules });
const puts = mock.method(b.privateBucket, 'put');
await new SchedulesUpdater(b).update();
assert.equal(second.api.filter(r => r.path === '/api/data/stages').length, 0);
assert.equal(puts.mock.calls.length, 0);
});
test('coop: publishes the calendar with stage and weapon details', async () => {
const coop = {
schedules: [{ start_time: 3600, end_time: 7200 }, { start_time: 90000, end_time: 93600 }],
details: [{ start_time: 3600, end_time: 7200, stage: { name: 'Spawning Grounds', image: '/images/coop_stage/a.png' }, weapons: [{ id: '0', weapon: { id: '0', name: 'Splattershot', image: '/images/weapon/0.png', sub: { id: '1', name: 'Bomb' }, special: { id: '2', name: 'Special' } } }, null] }],
};
fakeSplatNet({ '/api/coop_schedules': () => coop });
await new CoopSchedulesUpdater(b).update();
const ics = await (await b.publicBucket.get('data/coop-schedules.ics')).text();
assert.match(ics, /SUMMARY:Salmon Run on Spawning Grounds/);
assert.match(ics, /Splattershot/);
assert.match(ics, /Random/);
assert.match(ics, /SUMMARY:Salmon Run\r?\n/); // the shift without details keeps the plain title
assert.deepEqual(await json(b.publicBucket, 'data/coop-schedules.json'), coop);
});
test('timeline: keeps only coop and weapon availability, dropping hidden items', async () => {
fakeSplatNet({ '/api/timeline': () => ({ coop: { importance: -1 }, weapon_availability: { importance: 1, availabilities: [] }, stats: { ignored: true } }) });
await new TimelineUpdater(b).update();
assert.deepEqual(await json(b.publicBucket, 'data/timeline.json'), { coop: null, weapon_availability: { importance: 1, availabilities: [] } });
});
test('merchandises: attaches original gear from the bundled data without its brand', async () => {
const gearData = readJsonFile(new URL('../../src/common/data/gear.json', import.meta.url));
const original = gearData.head[0];
const brand = { id: '0', name: 'Brand', image: '/images/brand/0.png', frequent_skill: { id: '0', name: 'Skill', image: '/images/skill/0.png' } };
const skill = { id: '0', name: 'Skill', image: '/images/skill/0.png' };
fakeSplatNet({ '/api/onlineshop/merchandises': () => ({
merchandises: [
{ end_time: 7200, gear: { kind: 'head', id: '1', name: original.name.toUpperCase(), image: '/images/gear/1.png', brand }, skill },
{ end_time: 7200, gear: { kind: 'head', id: '2', name: 'No Such Gear', image: '/images/gear/2.png', brand }, skill },
],
extra: 'dropped',
}) });
await new MerchandisesUpdater(b).update();
const data = await json(b.publicBucket, 'data/merchandises.json');
assert.deepEqual(Object.keys(data), ['merchandises']);
assert.equal(data.merchandises[0].original_gear.name, original.name);
assert.equal('brand' in data.merchandises[0].original_gear, false);
assert.ok(data.merchandises[0].original_gear.skill);
assert.equal(data.merchandises[1].original_gear, null);
});
test('festivals: merges one region into the shared file, fetches missing rankings, uses regional languages', async () => {
await b.publicBucket.put('data/festivals.json', JSON.stringify({ eu: { festivals: [], results: [] } }));
await b.publicBucket.put('data/festivals/na-1-rankings.json', JSON.stringify({ existing: true }));
const festival = id => ({ festival_id: id, names: { alpha_short: 'A', bravo_short: 'B' }, times: { start: 3600, end: 7200 }, images: { alpha: '/images/festival/a.png', bravo: '/images/festival/b.png', panel: '/images/festival/p.png' }, special_stage: { id: '100', name: 'Shifty Station', image: '/images/stage/100.png' } });
const splatnet = fakeSplatNet({
'/api/festivals/active': () => ({ festivals: [festival(2)] }),
'/api/festivals/pasts': () => ({ festivals: [festival(1)], results: [{ festival_id: 1 }, { festival_id: 2 }] }),
'/api/festivals/2/rankings': () => ({ rankings: 'two' }),
});
await new FestivalsUpdater('NA', b).update();
const data = await json(b.publicBucket, 'data/festivals.json');
assert.deepEqual(Object.keys(data).sort(), ['eu', 'na']);
assert.deepEqual(data.na.festivals.map(f => f.festival_id), [2, 1]);
assert.deepEqual(await json(b.publicBucket, 'data/festivals/na-1-rankings.json'), { existing: true });
assert.deepEqual(await json(b.publicBucket, 'data/festivals/na-2-rankings.json'), { rankings: 'two' });
assert.deepEqual(splatnet.api.filter(r => r.path.endsWith('/rankings')).map(r => r.path), ['/api/festivals/2/rankings']);
// Only this region's languages are localized (NA: en, es-MX, fr-CA), all with the NA session
const localized = splatnet.api.filter(r => r.path === '/api/festivals/active');
assert.deepEqual(localized.map(r => `${r.region}/${r.language}`), ['NA/en', 'NA/es-MX', 'NA/fr-CA']);
const ics = await (await b.publicBucket.get('data/festivals-na.ics')).text();
assert.match(ics, /X-WR-CALNAME:Splatfests \(NA\)/);
assert.match(ics, /SUMMARY:NA Splatfest: A vs. B/);
});
test('original gear: mirrors skill images from the bundled skills data', async () => {
const skills = readJsonFile(new URL('../../src/common/data/skills.json', import.meta.url));
const images = Object.values(skills).map(s => s.image).filter(Boolean);
await b.publicBucket.put(`assets/splatnet${images[0]}`, 'existing');
const splatnet = fakeSplatNet();
await new OriginalGearImageUpdater(b).update();
assert.equal(splatnet.images.length, images.length - 1);
assert.equal(keys(b.publicBucket).length, images.length);
assert.ok(keys(b.publicBucket).every(k => k.startsWith('assets/splatnet/images/skill/')));
});