diff --git a/app/data/updaters/DataUpdater.mjs b/app/data/updaters/DataUpdater.mjs index 518e186..b41234c 100644 --- a/app/data/updaters/DataUpdater.mjs +++ b/app/data/updaters/DataUpdater.mjs @@ -4,6 +4,7 @@ import { Console } from 'node:console'; import mkdirp from 'mkdirp'; import jsonpath from 'jsonpath'; import ical from 'ical-generator'; +import pFilter from 'p-filter'; import prefixedConsole from '../../common/prefixedConsole.mjs'; import SplatNet3Client from '../../splatnet/SplatNet3Client.mjs'; import ImageProcessor from '../ImageProcessor.mjs'; @@ -11,7 +12,6 @@ import NsoClient from '../../splatnet/NsoClient.mjs'; import { locales, regionalLocales, defaultLocale } from '../../../src/common/i18n.mjs'; import { LocalizationProcessor } from '../LocalizationProcessor.mjs'; import { deriveId, getDateParts, getTopOfCurrentHour } from '../../common/util.mjs'; - export default class DataUpdater { name = null; @@ -135,16 +135,18 @@ export default class DataUpdater } // Retrieve data for missing languages - for (let locale of this.locales.filter(l => l !== initialLocale)) { - processor = new LocalizationProcessor(locale, this.localizations); + let processors = this.locales.filter(l => l !== initialLocale) + .map(l => new LocalizationProcessor(l, this.localizations)); + let missing = await pFilter(processors, p => p.hasMissingLocalizations(data)); - if (await processor.hasMissingLocalizations(data)) { - this.console.info(`Retrieving localized data for ${locale.code}`); - - let regionalData = await this.getData(locale); + if (missing.length > 0) { + await Promise.all(missing.map(async (processor) => { + let regionalData = await this.getData(processor.locale); this.deriveIds(regionalData); await processor.updateLocalizations(regionalData); - } + + this.console.info(`Retrieved localized data for: ${processor.locale.code}`); + })); } } diff --git a/app/splatnet/SplatNet3Client.mjs b/app/splatnet/SplatNet3Client.mjs index f27fcdf..0de9bb7 100644 --- a/app/splatnet/SplatNet3Client.mjs +++ b/app/splatnet/SplatNet3Client.mjs @@ -1,9 +1,13 @@ import fs from 'fs/promises'; +import pLimit from 'p-limit'; import ValueCache from '../common/ValueCache.mjs'; import prefixedConsole from '../common/prefixedConsole.mjs'; export const SPLATNET3_WEB_SERVICE_ID = '4834290508791808'; +// Concurrent request limit +const limit = pLimit(5); + export default class SplatNet3Client { baseUrl = 'https://api.lp1.av5ja.srv.nintendo.net'; @@ -108,17 +112,18 @@ export default class SplatNet3Client async getGraphQL(body = {}) { await this._maybeStartSession(); + let webViewVersion = await this._webViewVersion(); - let response = await fetch(this.baseUrl + '/api/graphql', { + let response = await limit(() => fetch(this.baseUrl + '/api/graphql', { method: 'POST', headers: { 'Authorization': `Bearer ${this.bulletToken.bulletToken}`, - 'X-Web-View-Ver': await this._webViewVersion(), + 'X-Web-View-Ver': webViewVersion, 'Content-Type': 'application/json', 'Accept-Language': this.acceptLanguage, }, body: JSON.stringify(body), - }); + })); if (!response.ok) { throw new Error(`Invalid GraphQL response code: ${response.status}`); diff --git a/package-lock.json b/package-lock.json index 155accb..2ba5e65 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,8 @@ "masto": "^6.7.0", "mkdirp": "^1.0.4", "nxapi": "^1.4.0", + "p-filter": "^4.1.0", + "p-limit": "^6.1.0", "p-queue": "^8.0.1", "pinia": "^2.0.22", "puppeteer-core": "^23.8.0", @@ -7924,16 +7926,31 @@ "node": ">= 0.8.0" } }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "node_modules/p-filter": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-filter/-/p-filter-4.1.0.tgz", + "integrity": "sha512-37/tPdZ3oJwHaS3gNJdenCDB3Tz26i9sjhnguBtvN0vYlRIiDNnvTWkuh+0hETV9rLPdJ3rlL3yVOYPIAnM8rw==", + "license": "MIT", "dependencies": { - "yocto-queue": "^0.1.0" + "p-map": "^7.0.1" }, "engines": { - "node": ">=10" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.1.0.tgz", + "integrity": "sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.1.1" + }, + "engines": { + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -7954,6 +7971,47 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-map": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.2.tgz", + "integrity": "sha512-z4cYYMMdKHzw4O5UkWJImbZynVIo0lSGTXc7bzB1e/rrDqkgGUNysK/o4bTr+0+xKvvLoTyGqYC4Fgljy9qe1Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-queue": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz", @@ -10504,12 +10562,12 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", + "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==", + "license": "MIT", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/package.json b/package.json index 1cc4672..ad529c7 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,8 @@ "masto": "^6.7.0", "mkdirp": "^1.0.4", "nxapi": "^1.4.0", + "p-filter": "^4.1.0", + "p-limit": "^6.1.0", "p-queue": "^8.0.1", "pinia": "^2.0.22", "puppeteer-core": "^23.8.0",