Make updaters more configurable

This should reduce the number of queries needed during high-load times (like waiting for Splatfest results).

app/cron.mjs

@@ -1,18 +1,38 @@
 import { CronJob } from "cron";
-import { updatePrimary, updateLowPriority } from "./data/index.mjs";
+import { update } from "./data/index.mjs";
 import { warmCaches } from "./splatnet/index.mjs";
 import { sendStatuses } from "./social/index.mjs";
 import { archiveData } from "./data/DataArchiver.mjs";
 
-export default function() {
+let updating = false;
-  new CronJob('5,20,35,50 * * * *', warmCaches, null, true);
+
-  new CronJob('15 0,2,5,10,15,30,45 * * * *', async () => {
+async function updateIfNotUpdating(mode) {
-    await updatePrimary();
+  if (updating) {
+    console.log('[Cron] Update already in progress');
+
+    return;
+  }
+
+  updating = true;
+
+  try {
+    await update(mode);
     await sendStatuses();
     await archiveData();
+  } finally {
+    updating = false;
+  }
+}
+
+export default function() {
+  new CronJob('5,20,35,50 * * * *', warmCaches, null, true);
+  new CronJob('15 0,1,2,3,4 * * * *', () => {
+    return updateIfNotUpdating('quick');
   }, null, true);
-  new CronJob('1 * * * *', async () => {
+  new CronJob('15 5,10,15,30,45 * * * *', () => {
-    await updateLowPriority();
+    return updateIfNotUpdating('default');
-    await archiveData();
+  }, null, true);
+  new CronJob('20 * * * *', () => {
+    return updateIfNotUpdating('all');
   }, null, true);
 }

app/data/index.mjs

@@ -15,43 +15,38 @@ function updaters() {
     new FestivalUpdater('JP'),
     new FestivalUpdater('AP'),
     new StagesUpdater,
-  ];
-}
-
-function lowPriorityUpdaters() {
-  return [
     new XRankUpdater('Tentatek', 'ATLANTIC'),
     new XRankUpdater('Takoroka', 'PACIFIC'),
   ];
 }
 
-async function run(updaters) {
+const configs = {
-  for (let updater of updaters) {
+  quick: {
+    disableLocalizations: true,
+    disableXRank: true,
+    disableFestivalDetails: true,
+  },
+  default: {
+    disableXRank: true,
+  },
+  all: {
+    // Everything enabled
+  },
+};
+
+export async function update(config = 'default') {
+  console.info(`Running ${config} updaters...`);
+
+  let settings = configs[config];
+
+  for (let updater of updaters()) {
+    updater.settings = settings;
     try {
       await updater.updateIfNeeded();
     } catch (e) {
       console.error(e);
     }
   }
-}
 
-export async function updateAll() {
+  console.info(`Done running ${config} updaters`);
-  console.info('Running all updaters...');
-  await run([
-    ...updaters(),
-    ...lowPriorityUpdaters(),
-  ]);
-  console.info('Done running all updaters');
-}
-
-export async function updatePrimary() {
-  console.info('Running primary updaters...');
-  await run(updaters());
-  console.info('Done running primary updaters');
-}
-
-export async function updateLowPriority() {
-  console.info('Running low-priority updaters...');
-  await run(lowPriorityUpdaters());
-  console.info('Done running low-priority updaters');
 }

app/data/updaters/DataUpdater.mjs

@@ -26,6 +26,8 @@ export default class DataUpdater
   derivedIds = [];
   localizations = [];
 
+  settings = {};
+
   constructor(region = null) {
     this.selectedRegion = region;
     this.nsoClient = NsoClient.make(region);
@@ -127,6 +129,10 @@ export default class DataUpdater
     let processor = new LocalizationProcessor(initialLocale, this.localizations);
     await processor.updateLocalizations(data);
 
+    if (this.settings.disableLocalizations) {
+      return;
+    }
+
     // Retrieve data for missing languages
     for (let locale of this.locales.filter(l => l !== initialLocale)) {
       processor = new LocalizationProcessor(locale, this.localizations);

app/data/updaters/FestivalUpdater.mjs

@@ -65,7 +65,9 @@ export default class FestivalUpdater extends DataUpdater
 
     // Get the detailed data for each Splatfest
     // (unless we're getting localization-specific data)
-    if (locale === this.defaultLocale) {
+    let shouldGetDetails = locale === this.defaultLocale
+      && !this.settings.disableFestivalDetails;
+    if (shouldGetDetails) {
       for (let node of result.data.festRecords.nodes) {
         let detailResult = await this.getFestivalDetails(node);
 

app/data/updaters/XRankUpdater.mjs

@@ -33,6 +33,16 @@ export default class XRankUpdater extends DataUpdater
     return this._console;
   }
 
+  shouldUpdate() {
+    if (this.settings.disableXRank) {
+      this.console.log('X-Rank updates disabled');
+
+      return false;
+    }
+
+    return super.shouldUpdate();
+  }
+
   async getData(locale) {
     let result = await this.splatnet(locale).getXRankingData(this.divisionKey);
     let seasons = this.getSeasons(result.data);

app/index.mjs

@@ -2,7 +2,7 @@ import dotenv from 'dotenv';
 import consoleStamp from 'console-stamp';
 import cron from './cron.mjs';
 import { sendStatuses, testStatuses } from './social/index.mjs';
-import { updatePrimary, updateAll } from './data/index.mjs';
+import { update } from './data/index.mjs';
 import { warmCaches } from "./splatnet/index.mjs";
 import MastodonClient from './social/clients/MastodonClient.mjs';
 import ImageWriter from './social/clients/ImageWriter.mjs';
@@ -23,16 +23,16 @@ const actions = {
   socialTestBluesky: () => testStatuses([new BlueskyClient]),
   socialTestImage: () => testStatuses([new ImageWriter]),
   socialTestThreads: () => testStatuses([new ThreadsClient]),
-  splatnet: updatePrimary,
+  splatnet: update,
-  splatnetAll: updateAll,
   warmCaches,
   dataArchive: archiveData,
 }
 
 const command = process.argv[2];
+const params = process.argv.slice(3);
 const action = actions[command];
 if (action) {
-  action();
+  action(...params);
 } else {
   console.error(`Unrecognized command: ${command}`);
 }

package.json

@@ -14,8 +14,9 @@
     "social:test:mastodon": "node app/index.mjs socialTestMastodon",
     "social:test:bluesky": "node app/index.mjs socialTestBluesky",
     "social:test:threads": "node app/index.mjs socialTestThreads",
-    "splatnet": "node app/index.mjs splatnet",
+    "splatnet:quick": "node app/index.mjs splatnet quick",
-    "splatnet:all": "node app/index.mjs splatnetAll",
+    "splatnet": "node app/index.mjs splatnet default",
+    "splatnet:all": "node app/index.mjs splatnet all",
     "warmCaches": "node app/index.mjs warmCaches",
     "data:archive": "node app/index.mjs dataArchive"
   },