Add S3 syncer

app/data/index.mjs

@@ -5,6 +5,8 @@ import CoopUpdater from './updaters/CoopUpdater.mjs';
 import FestivalUpdater from './updaters/FestivalUpdater.mjs';
 import XRankUpdater from './updaters/XRankUpdater.mjs';
 import StagesUpdater from './updaters/StagesUpdater.mjs';
+import S3Syncer from '../sync/S3Syncer.mjs';
+import { canSync } from '../sync/index.mjs';
 
 function updaters() {
   return [
@@ -50,5 +52,9 @@ export async function update(config = 'default') {
     }
   }
 
+  if (canSync()) {
+    await (new S3Syncer).upload();
+  }
+
   console.info(`Done running ${config} updaters`);
 }

app/index.mjs

@@ -10,6 +10,7 @@ import BlueskyClient from './social/clients/BlueskyClient.mjs';
 import ThreadsClient from './social/clients/ThreadsClient.mjs';
 import { archiveData } from './data/DataArchiver.mjs';
 import { sentryInit } from './common/sentry.mjs';
+import { sync, syncUpload, syncDownload } from './sync/index.mjs';
 
 consoleStamp(console);
 dotenv.config();
@@ -26,6 +27,9 @@ const actions = {
   splatnet: update,
   warmCaches,
   dataArchive: archiveData,
+  sync,
+  syncUpload,
+  syncDownload,
 };
 
 const command = process.argv[2];

app/social/index.mjs

@@ -16,6 +16,8 @@ import EggstraWorkUpcomingStatus from './generators/EggstraWorkUpcomingStatus.mj
 import BlueskyClient from './clients/BlueskyClient.mjs';
 import ChallengeStatus from './generators/ChallengeStatus.mjs';
 import ThreadsClient from './clients/ThreadsClient.mjs';
+import S3Syncer from '../sync/S3Syncer.mjs';
+import { canSync } from '../sync/index.mjs';
 
 function defaultStatusGenerators() {
   return [
@@ -63,8 +65,12 @@ export function testStatusGeneratorManager(additionalClients) {
   );
 }
 
-export function sendStatuses() {
-  return defaultStatusGeneratorManager().sendStatuses();
+export async function sendStatuses() {
+  await defaultStatusGeneratorManager().sendStatuses();
+
+  if (canSync()) {
+    await (new S3Syncer).upload();
+  }
 }
 
 export function testStatuses(additionalClients = []) {

app/sync/S3Syncer.mjs

@@ -0,0 +1,77 @@
+import path  from 'path';
+import { S3Client }  from '@aws-sdk/client-s3';
+import { S3SyncClient }  from 's3-sync-client';
+import mime  from 'mime-types';
+
+export default class S3Syncer
+{
+  download() {
+    this.log('Downloading files...');
+
+    return Promise.all([
+      this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, {
+        filters: this.filters,
+      }),
+      this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`),
+    ]);
+  }
+
+  upload() {
+    this.log('Uploading files...');
+
+    return Promise.all([
+      this.syncClient.sync(`${this.localPath}/dist`, this.publicBucket, {
+        filters: this.filters,
+        commandInput: input => ({
+          ACL: 'public-read',
+          ContentType: mime.lookup(input.Key),
+          CacheControl: input.Key.startsWith('data/')
+            ? 'no-cache, stale-while-revalidate=5, stale-if-error=86400'
+            : undefined,
+        }),
+      }),
+      this.syncClient.sync(`${this.localPath}/storage`, this.privateBucket),
+    ]);
+  }
+
+  get s3Client() {
+    return this._s3Client ??= new S3Client({
+      endpoint: process.env.AWS_S3_ENDPOINT,
+      region: process.env.AWS_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+      },
+    });
+  }
+
+  /** @returns {S3SyncClient} */
+  get syncClient() {
+    return this._syncClient ??= new S3SyncClient({ client: this.s3Client });
+  }
+
+  get publicBucket() {
+    return `s3://${process.env.AWS_S3_BUCKET}`;
+  }
+
+  get privateBucket() {
+    return `s3://${process.env.AWS_S3_PRIVATE_BUCKET}`;
+  }
+
+  get localPath() {
+    return path.resolve('.');
+  }
+
+  get filters() {
+    return [
+      { exclude: () => true }, // Exclude everything by default
+      { include: (key) => key.startsWith('assets/splatnet/') },
+      { include: (key) => key.startsWith('data/') },
+      { include: (key) => key.startsWith('status-screenshots/') },
+    ];
+  }
+
+  log(message) {
+    console.log(`[S3] ${message}`);
+  }
+}

app/sync/index.mjs

@@ -0,0 +1,41 @@
+import S3Syncer from './S3Syncer.mjs';
+
+export function canSync() {
+  return !!(
+    process.env.AWS_ACCESS_KEY_ID &&
+    process.env.AWS_SECRET_ACCESS_KEY &&
+    process.env.AWS_S3_BUCKET &&
+    process.env.AWS_S3_PRIVATE_BUCKET
+  );
+}
+
+async function doSync(download, upload) {
+  if (!canSync()) {
+    console.warn('Missing S3 connection parameters');
+    return;
+  }
+
+  const syncer = new S3Syncer();
+
+  if (download) {
+    console.info('Downloading files...');
+    await syncer.download();
+  }
+
+  if (upload) {
+    console.info('Uploading files...');
+    await syncer.upload();
+  }
+}
+
+export function sync() {
+  return doSync(true, true);
+}
+
+export function syncUpload() {
+  return doSync(false, true);
+}
+
+export function syncDownload() {
+  return doSync(true, false);
+}

package-lock.json

@@ -26,6 +26,7 @@
         "nxapi": "^1.4.0",
         "pinia": "^2.0.22",
         "puppeteer-core": "^23.8.0",
+        "s3-sync-client": "^4.3.1",
         "sharp": "^0.32.0",
         "threads-api": "^1.4.0",
         "twitter-api-v2": "^1.12.7",
@@ -261,6 +262,47 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
       "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
     },
+    "node_modules/@aws-sdk/abort-controller": {
+      "version": "3.370.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/abort-controller/-/abort-controller-3.370.0.tgz",
+      "integrity": "sha512-/W4arzC/+yVW/cvEXbuwvG0uly4yFSZnnIA+gkqgAm+0HVfacwcPpNf4BjyxjnvIdh03l7w2DriF6MlKUfiQ3A==",
+      "license": "Apache-2.0",
+      "peer": true,
+      "dependencies": {
+        "@aws-sdk/types": "3.370.0",
+        "tslib": "^2.5.0"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/@aws-sdk/abort-controller/node_modules/@aws-sdk/types": {
+      "version": "3.370.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.370.0.tgz",
+      "integrity": "sha512-8PGMKklSkRKjunFhzM2y5Jm0H2TBu7YRNISdYzXLUHKSP9zlMEYagseKVdmox0zKHf1LXVNuSlUV2b6SRrieCQ==",
+      "license": "Apache-2.0",
+      "peer": true,
+      "dependencies": {
+        "@smithy/types": "^1.1.0",
+        "tslib": "^2.5.0"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/@aws-sdk/abort-controller/node_modules/@smithy/types": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@smithy/types/-/types-1.2.0.tgz",
+      "integrity": "sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==",
+      "license": "Apache-2.0",
+      "peer": true,
+      "dependencies": {
+        "tslib": "^2.5.0"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
     "node_modules/@aws-sdk/client-s3": {
       "version": "3.535.0",
       "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.535.0.tgz",
@@ -8800,6 +8842,19 @@
         "tslib": "^2.1.0"
       }
     },
+    "node_modules/s3-sync-client": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/s3-sync-client/-/s3-sync-client-4.3.1.tgz",
+      "integrity": "sha512-nWbbKCNnXmWvD8XwdWhX25VNxIhgQEm6vXqSYjwyBNZI07OuMOr/LNOYmEPcLfqFFjy55ZNcFSBI18W29ybuUw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=16.0.0"
+      },
+      "peerDependencies": {
+        "@aws-sdk/abort-controller": "^3.x.x",
+        "@aws-sdk/client-s3": "^3.x.x"
+      }
+    },
     "node_modules/safe-array-concat": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz",

package.json

@@ -19,7 +19,10 @@
     "splatnet": "node app/index.mjs splatnet default",
     "splatnet:all": "node app/index.mjs splatnet all",
     "warmCaches": "node app/index.mjs warmCaches",
-    "data:archive": "node app/index.mjs dataArchive"
+    "data:archive": "node app/index.mjs dataArchive",
+    "sync": "node app/index.mjs sync",
+    "sync:upload": "node app/index.mjs syncUpload",
+    "sync:download": "node app/index.mjs syncDownload"
   },
   "dependencies": {
     "@atproto/api": "^0.11.2",
@@ -40,6 +43,7 @@
     "nxapi": "^1.4.0",
     "pinia": "^2.0.22",
     "puppeteer-core": "^23.8.0",
+    "s3-sync-client": "^4.3.1",
     "sharp": "^0.32.0",
     "threads-api": "^1.4.0",
     "twitter-api-v2": "^1.12.7",