Retrieve localizations in parallel

app/data/updaters/DataUpdater.mjs

@@ -4,6 +4,7 @@ import { Console } from 'node:console';
 import mkdirp from 'mkdirp';
 import jsonpath from 'jsonpath';
 import ical from 'ical-generator';
+import pFilter from 'p-filter';
 import prefixedConsole from '../../common/prefixedConsole.mjs';
 import SplatNet3Client from '../../splatnet/SplatNet3Client.mjs';
 import ImageProcessor from '../ImageProcessor.mjs';
@@ -11,7 +12,6 @@ import NsoClient from '../../splatnet/NsoClient.mjs';
 import { locales, regionalLocales, defaultLocale } from '../../../src/common/i18n.mjs';
 import { LocalizationProcessor } from '../LocalizationProcessor.mjs';
 import { deriveId, getDateParts, getTopOfCurrentHour } from '../../common/util.mjs';
-
 export default class DataUpdater
 {
   name = null;
@@ -135,16 +135,18 @@ export default class DataUpdater
     }
 
     // Retrieve data for missing languages
-    for (let locale of this.locales.filter(l => l !== initialLocale)) {
+    let processors = this.locales.filter(l => l !== initialLocale)
-      processor = new LocalizationProcessor(locale, this.localizations);
+      .map(l => new LocalizationProcessor(l, this.localizations));
+    let missing = await pFilter(processors, p => p.hasMissingLocalizations(data));
 
-      if (await processor.hasMissingLocalizations(data)) {
+    if (missing.length > 0) {
-        this.console.info(`Retrieving localized data for ${locale.code}`);
+      await Promise.all(missing.map(async (processor) => {
-
+        let regionalData = await this.getData(processor.locale);
-        let regionalData = await this.getData(locale);
         this.deriveIds(regionalData);
         await processor.updateLocalizations(regionalData);
-      }
+
+        this.console.info(`Retrieved localized data for: ${processor.locale.code}`);
+      }));
     }
   }
 

app/splatnet/SplatNet3Client.mjs

@@ -1,9 +1,13 @@
 import fs from 'fs/promises';
+import pLimit from 'p-limit';
 import ValueCache from '../common/ValueCache.mjs';
 import prefixedConsole from '../common/prefixedConsole.mjs';
 
 export const SPLATNET3_WEB_SERVICE_ID = '4834290508791808';
 
+// Concurrent request limit
+const limit = pLimit(5);
+
 export default class SplatNet3Client
 {
   baseUrl = 'https://api.lp1.av5ja.srv.nintendo.net';
@@ -108,17 +112,18 @@ export default class SplatNet3Client
 
   async getGraphQL(body = {}) {
     await this._maybeStartSession();
+    let webViewVersion = await this._webViewVersion();
 
-    let response = await fetch(this.baseUrl + '/api/graphql', {
+    let response = await limit(() => fetch(this.baseUrl + '/api/graphql', {
       method: 'POST',
       headers: {
         'Authorization': `Bearer ${this.bulletToken.bulletToken}`,
-        'X-Web-View-Ver': await this._webViewVersion(),
+        'X-Web-View-Ver': webViewVersion,
         'Content-Type': 'application/json',
         'Accept-Language': this.acceptLanguage,
       },
       body: JSON.stringify(body),
-    });
+    }));
 
     if (!response.ok) {
       throw new Error(`Invalid GraphQL response code: ${response.status}`);

package-lock.json

@@ -24,6 +24,8 @@
         "masto": "^6.7.0",
         "mkdirp": "^1.0.4",
         "nxapi": "^1.4.0",
+        "p-filter": "^4.1.0",
+        "p-limit": "^6.1.0",
         "p-queue": "^8.0.1",
         "pinia": "^2.0.22",
         "puppeteer-core": "^23.8.0",
@@ -7924,16 +7926,31 @@
         "node": ">= 0.8.0"
       }
     },
-    "node_modules/p-limit": {
+    "node_modules/p-filter": {
-      "version": "3.1.0",
+      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/p-filter/-/p-filter-4.1.0.tgz",
-      "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+      "integrity": "sha512-37/tPdZ3oJwHaS3gNJdenCDB3Tz26i9sjhnguBtvN0vYlRIiDNnvTWkuh+0hETV9rLPdJ3rlL3yVOYPIAnM8rw==",
-      "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "yocto-queue": "^0.1.0"
+        "p-map": "^7.0.1"
       },
       "engines": {
-        "node": ">=10"
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-limit": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.1.0.tgz",
+      "integrity": "sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==",
+      "license": "MIT",
+      "dependencies": {
+        "yocto-queue": "^1.1.1"
+      },
+      "engines": {
+        "node": ">=18"
       },
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"
@@ -7954,6 +7971,47 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/p-locate/node_modules/p-limit": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+      "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "yocto-queue": "^0.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-locate/node_modules/yocto-queue": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
+      "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-map": {
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.2.tgz",
+      "integrity": "sha512-z4cYYMMdKHzw4O5UkWJImbZynVIo0lSGTXc7bzB1e/rrDqkgGUNysK/o4bTr+0+xKvvLoTyGqYC4Fgljy9qe1Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/p-queue": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz",
@@ -10504,12 +10562,12 @@
       }
     },
     "node_modules/yocto-queue": {
-      "version": "0.1.0",
+      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz",
-      "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+      "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==",
-      "dev": true,
+      "license": "MIT",
       "engines": {
-        "node": ">=10"
+        "node": ">=12.20"
       },
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"

package.json

@@ -41,6 +41,8 @@
     "masto": "^6.7.0",
     "mkdirp": "^1.0.4",
     "nxapi": "^1.4.0",
+    "p-filter": "^4.1.0",
+    "p-limit": "^6.1.0",
     "p-queue": "^8.0.1",
     "pinia": "^2.0.22",
     "puppeteer-core": "^23.8.0",