diff --git a/app/data/ImageProcessor.mjs b/app/data/ImageProcessor.mjs index dd84a9f..af195dc 100644 --- a/app/data/ImageProcessor.mjs +++ b/app/data/ImageProcessor.mjs @@ -1,10 +1,13 @@ import fs from 'fs/promises'; import path from 'path'; import mkdirp from 'mkdirp'; +import pQueue from 'p-queue'; import prefixedConsole from '../common/prefixedConsole.mjs'; import { normalizeSplatnetResourcePath } from '../common/util.mjs'; import { exists } from '../common/fs.mjs'; +const queue = new pQueue({ concurrency: 4 }); + export default class ImageProcessor { destinationDirectory = 'dist'; @@ -15,17 +18,27 @@ export default class ImageProcessor this.siteUrl = process.env.SITE_URL; } - async process(url) { + async process(url, defer = true) { // Normalize the path let destination = this.normalize(url); // Download the image if necessary - await this.maybeDownload(url, destination); + let job = () => this.maybeDownload(url, destination); + // defer ? queue.add(job) : await job(); + if (defer) { + queue.add(job); + } else { + await job(); + } // Return the new public URL return [destination, this.publicUrl(destination)]; } + static onIdle() { + return queue.onIdle(); + } + normalize(url) { return normalizeSplatnetResourcePath(url); } diff --git a/app/data/index.mjs b/app/data/index.mjs index b6adc91..20b4f17 100644 --- a/app/data/index.mjs +++ b/app/data/index.mjs @@ -7,6 +7,7 @@ import CoopUpdater from './updaters/CoopUpdater.mjs'; import FestivalUpdater from './updaters/FestivalUpdater.mjs'; import XRankUpdater from './updaters/XRankUpdater.mjs'; import StagesUpdater from './updaters/StagesUpdater.mjs'; +import ImageProcessor from './ImageProcessor.mjs'; function updaters() { return [ @@ -53,6 +54,7 @@ export async function update(config = 'default') { } if (canSync()) { + await ImageProcessor.onIdle(); await (new S3Syncer).upload(); } diff --git a/app/data/updaters/DataUpdater.mjs b/app/data/updaters/DataUpdater.mjs index 72a1cf8..518e186 100644 --- a/app/data/updaters/DataUpdater.mjs +++ b/app/data/updaters/DataUpdater.mjs @@ -166,6 +166,8 @@ export default class DataUpdater jsonpath.apply(data, expression, url => mapping[url]); } + await ImageProcessor.onIdle(); + return images; } diff --git a/package-lock.json b/package-lock.json index 06cf811..155accb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "masto": "^6.7.0", "mkdirp": "^1.0.4", "nxapi": "^1.4.0", + "p-queue": "^8.0.1", "pinia": "^2.0.22", "puppeteer-core": "^23.8.0", "s3-sync-client": "^4.3.1", @@ -5605,6 +5606,12 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "license": "MIT" + }, "node_modules/events-to-async": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/events-to-async/-/events-to-async-2.0.1.tgz", @@ -7947,6 +7954,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz", + "integrity": "sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.1", + "p-timeout": "^6.1.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.3.tgz", + "integrity": "sha512-UJUyfKbwvr/uZSV6btANfb+0t/mOhKV/KXcCUTp8FcQI+v/0d+wXqH4htrW0E4rR6WiEO/EPvUFiV9D5OI4vlw==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", diff --git a/package.json b/package.json index b2037e9..1cc4672 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "masto": "^6.7.0", "mkdirp": "^1.0.4", "nxapi": "^1.4.0", + "p-queue": "^8.0.1", "pinia": "^2.0.22", "puppeteer-core": "^23.8.0", "s3-sync-client": "^4.3.1",