From 61736935201307352edca47bb34323e020625e6b Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sat, 2 Dec 2023 11:06:45 -0800 Subject: [PATCH] Make updaters more configurable This should reduce the number of queries needed during high-load times (like waiting for Splatfest results). --- app/cron.mjs | 36 +++++++++++++++----- app/data/index.mjs | 49 ++++++++++++--------------- app/data/updaters/DataUpdater.mjs | 6 ++++ app/data/updaters/FestivalUpdater.mjs | 4 ++- app/data/updaters/XRankUpdater.mjs | 10 ++++++ app/index.mjs | 8 ++--- package.json | 5 +-- 7 files changed, 76 insertions(+), 42 deletions(-) diff --git a/app/cron.mjs b/app/cron.mjs index feff36f..9f1b076 100644 --- a/app/cron.mjs +++ b/app/cron.mjs @@ -1,18 +1,38 @@ import { CronJob } from "cron"; -import { updatePrimary, updateLowPriority } from "./data/index.mjs"; +import { update } from "./data/index.mjs"; import { warmCaches } from "./splatnet/index.mjs"; import { sendStatuses } from "./social/index.mjs"; import { archiveData } from "./data/DataArchiver.mjs"; -export default function() { - new CronJob('5,20,35,50 * * * *', warmCaches, null, true); - new CronJob('15 0,2,5,10,15,30,45 * * * *', async () => { - await updatePrimary(); +let updating = false; + +async function updateIfNotUpdating(mode) { + if (updating) { + console.log('[Cron] Update already in progress'); + + return; + } + + updating = true; + + try { + await update(mode); await sendStatuses(); await archiveData(); + } finally { + updating = false; + } +} + +export default function() { + new CronJob('5,20,35,50 * * * *', warmCaches, null, true); + new CronJob('15 0,1,2,3,4 * * * *', () => { + return updateIfNotUpdating('quick'); }, null, true); - new CronJob('1 * * * *', async () => { - await updateLowPriority(); - await archiveData(); + new CronJob('15 5,10,15,30,45 * * * *', () => { + return updateIfNotUpdating('default'); + }, null, true); + new CronJob('20 * * * *', () => { + return updateIfNotUpdating('all'); }, null, true); } diff --git a/app/data/index.mjs b/app/data/index.mjs index 4c8e698..dd064e6 100644 --- a/app/data/index.mjs +++ b/app/data/index.mjs @@ -15,43 +15,38 @@ function updaters() { new FestivalUpdater('JP'), new FestivalUpdater('AP'), new StagesUpdater, - ]; -} - -function lowPriorityUpdaters() { - return [ new XRankUpdater('Tentatek', 'ATLANTIC'), new XRankUpdater('Takoroka', 'PACIFIC'), ]; } -async function run(updaters) { - for (let updater of updaters) { +const configs = { + quick: { + disableLocalizations: true, + disableXRank: true, + disableFestivalDetails: true, + }, + default: { + disableXRank: true, + }, + all: { + // Everything enabled + }, +}; + +export async function update(config = 'default') { + console.info(`Running ${config} updaters...`); + + let settings = configs[config]; + + for (let updater of updaters()) { + updater.settings = settings; try { await updater.updateIfNeeded(); } catch (e) { console.error(e); } } -} -export async function updateAll() { - console.info('Running all updaters...'); - await run([ - ...updaters(), - ...lowPriorityUpdaters(), - ]); - console.info('Done running all updaters'); -} - -export async function updatePrimary() { - console.info('Running primary updaters...'); - await run(updaters()); - console.info('Done running primary updaters'); -} - -export async function updateLowPriority() { - console.info('Running low-priority updaters...'); - await run(lowPriorityUpdaters()); - console.info('Done running low-priority updaters'); + console.info(`Done running ${config} updaters`); } diff --git a/app/data/updaters/DataUpdater.mjs b/app/data/updaters/DataUpdater.mjs index 8cd0e1c..9628975 100644 --- a/app/data/updaters/DataUpdater.mjs +++ b/app/data/updaters/DataUpdater.mjs @@ -26,6 +26,8 @@ export default class DataUpdater derivedIds = []; localizations = []; + settings = {}; + constructor(region = null) { this.selectedRegion = region; this.nsoClient = NsoClient.make(region); @@ -127,6 +129,10 @@ export default class DataUpdater let processor = new LocalizationProcessor(initialLocale, this.localizations); await processor.updateLocalizations(data); + if (this.settings.disableLocalizations) { + return; + } + // Retrieve data for missing languages for (let locale of this.locales.filter(l => l !== initialLocale)) { processor = new LocalizationProcessor(locale, this.localizations); diff --git a/app/data/updaters/FestivalUpdater.mjs b/app/data/updaters/FestivalUpdater.mjs index fbe1061..02a989b 100644 --- a/app/data/updaters/FestivalUpdater.mjs +++ b/app/data/updaters/FestivalUpdater.mjs @@ -65,7 +65,9 @@ export default class FestivalUpdater extends DataUpdater // Get the detailed data for each Splatfest // (unless we're getting localization-specific data) - if (locale === this.defaultLocale) { + let shouldGetDetails = locale === this.defaultLocale + && !this.settings.disableFestivalDetails; + if (shouldGetDetails) { for (let node of result.data.festRecords.nodes) { let detailResult = await this.getFestivalDetails(node); diff --git a/app/data/updaters/XRankUpdater.mjs b/app/data/updaters/XRankUpdater.mjs index 0cfa703..939c357 100644 --- a/app/data/updaters/XRankUpdater.mjs +++ b/app/data/updaters/XRankUpdater.mjs @@ -33,6 +33,16 @@ export default class XRankUpdater extends DataUpdater return this._console; } + shouldUpdate() { + if (this.settings.disableXRank) { + this.console.log('X-Rank updates disabled'); + + return false; + } + + return super.shouldUpdate(); + } + async getData(locale) { let result = await this.splatnet(locale).getXRankingData(this.divisionKey); let seasons = this.getSeasons(result.data); diff --git a/app/index.mjs b/app/index.mjs index ee68af8..20be096 100644 --- a/app/index.mjs +++ b/app/index.mjs @@ -2,7 +2,7 @@ import dotenv from 'dotenv'; import consoleStamp from 'console-stamp'; import cron from './cron.mjs'; import { sendStatuses, testStatuses } from './social/index.mjs'; -import { updatePrimary, updateAll } from './data/index.mjs'; +import { update } from './data/index.mjs'; import { warmCaches } from "./splatnet/index.mjs"; import MastodonClient from './social/clients/MastodonClient.mjs'; import ImageWriter from './social/clients/ImageWriter.mjs'; @@ -23,16 +23,16 @@ const actions = { socialTestBluesky: () => testStatuses([new BlueskyClient]), socialTestImage: () => testStatuses([new ImageWriter]), socialTestThreads: () => testStatuses([new ThreadsClient]), - splatnet: updatePrimary, - splatnetAll: updateAll, + splatnet: update, warmCaches, dataArchive: archiveData, } const command = process.argv[2]; +const params = process.argv.slice(3); const action = actions[command]; if (action) { - action(); + action(...params); } else { console.error(`Unrecognized command: ${command}`); } diff --git a/package.json b/package.json index a89584d..5830d4f 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,9 @@ "social:test:mastodon": "node app/index.mjs socialTestMastodon", "social:test:bluesky": "node app/index.mjs socialTestBluesky", "social:test:threads": "node app/index.mjs socialTestThreads", - "splatnet": "node app/index.mjs splatnet", - "splatnet:all": "node app/index.mjs splatnetAll", + "splatnet:quick": "node app/index.mjs splatnet quick", + "splatnet": "node app/index.mjs splatnet default", + "splatnet:all": "node app/index.mjs splatnet all", "warmCaches": "node app/index.mjs warmCaches", "data:archive": "node app/index.mjs dataArchive" },