Download images concurrently

app/data/ImageProcessor.mjs

@@ -1,10 +1,13 @@
 import fs from 'fs/promises';
 import path from 'path';
 import mkdirp from 'mkdirp';
+import pQueue from 'p-queue';
 import prefixedConsole from '../common/prefixedConsole.mjs';
 import { normalizeSplatnetResourcePath } from '../common/util.mjs';
 import { exists } from '../common/fs.mjs';
 
+const queue = new pQueue({ concurrency: 4 });
+
 export default class ImageProcessor
 {
   destinationDirectory = 'dist';
@@ -15,17 +18,27 @@ export default class ImageProcessor
     this.siteUrl = process.env.SITE_URL;
   }
 
-  async process(url) {
+  async process(url, defer = true) {
     // Normalize the path
     let destination = this.normalize(url);
 
     // Download the image if necessary
-    await this.maybeDownload(url, destination);
+    let job = () => this.maybeDownload(url, destination);
+    // defer ? queue.add(job) : await job();
+    if (defer) {
+      queue.add(job);
+    } else {
+      await job();
+    }
 
     // Return the new public URL
     return [destination, this.publicUrl(destination)];
   }
 
+  static onIdle() {
+    return queue.onIdle();
+  }
+
   normalize(url) {
     return normalizeSplatnetResourcePath(url);
   }

app/data/index.mjs

@@ -7,6 +7,7 @@ import CoopUpdater from './updaters/CoopUpdater.mjs';
 import FestivalUpdater from './updaters/FestivalUpdater.mjs';
 import XRankUpdater from './updaters/XRankUpdater.mjs';
 import StagesUpdater from './updaters/StagesUpdater.mjs';
+import ImageProcessor from './ImageProcessor.mjs';
 
 function updaters() {
   return [
@@ -53,6 +54,7 @@ export async function update(config = 'default') {
   }
 
   if (canSync()) {
+    await ImageProcessor.onIdle();
     await (new S3Syncer).upload();
   }
 

app/data/updaters/DataUpdater.mjs

@@ -166,6 +166,8 @@ export default class DataUpdater
       jsonpath.apply(data, expression, url => mapping[url]);
     }
 
+    await ImageProcessor.onIdle();
+
     return images;
   }
 

package-lock.json

@@ -24,6 +24,7 @@
         "masto": "^6.7.0",
         "mkdirp": "^1.0.4",
         "nxapi": "^1.4.0",
+        "p-queue": "^8.0.1",
         "pinia": "^2.0.22",
         "puppeteer-core": "^23.8.0",
         "s3-sync-client": "^4.3.1",
@@ -5605,6 +5606,12 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/eventemitter3": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
+      "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==",
+      "license": "MIT"
+    },
     "node_modules/events-to-async": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/events-to-async/-/events-to-async-2.0.1.tgz",
@@ -7947,6 +7954,34 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/p-queue": {
+      "version": "8.0.1",
+      "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz",
+      "integrity": "sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==",
+      "license": "MIT",
+      "dependencies": {
+        "eventemitter3": "^5.0.1",
+        "p-timeout": "^6.1.2"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-timeout": {
+      "version": "6.1.3",
+      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.3.tgz",
+      "integrity": "sha512-UJUyfKbwvr/uZSV6btANfb+0t/mOhKV/KXcCUTp8FcQI+v/0d+wXqH4htrW0E4rR6WiEO/EPvUFiV9D5OI4vlw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=14.16"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/p-try": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",

package.json

@@ -41,6 +41,7 @@
     "masto": "^6.7.0",
     "mkdirp": "^1.0.4",
     "nxapi": "^1.4.0",
+    "p-queue": "^8.0.1",
     "pinia": "^2.0.22",
     "puppeteer-core": "^23.8.0",
     "s3-sync-client": "^4.3.1",