Add S3 syncer

.env.example

@@ -12,6 +12,14 @@ VUE_APP_GOOGLE_ANALYTICS_ID=
 # (Optional) Sentry error reporting (https://sentry.io)
 SENTRY_DSN=
 
+# (Optional) S3 parameters
+AWS_S3_ENDPOINT=
+AWS_S3_REGION=
+AWS_S3_BUCKET=
+AWS_S3_PRIVATE_BUCKET=
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
 # (Optional) Twitter API parameters
 TWITTER_CONSUMER_KEY=
 TWITTER_CONSUMER_SECRET=

package.json

@@ -10,11 +10,13 @@
     "splatnet": "node src/app splatnet",
     "twitter": "node src/app twitter",
     "twitter:test": "node src/app twitterTest",
+    "sync": "node src/app sync",
     "utility:copyTranslation": "node src/utility copyTranslation",
     "utility:getSplatNetLanguageFiles": "node src/utility getSplatNetLanguageFiles",
     "utility:updateGear": "node src/utility updateGear"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.685.0",
     "axios": "^0.19.2",
     "babel-eslint": "^10.1.0",
     "bulma": "~0.6.1",
@@ -32,11 +34,13 @@
     "jsonpath": "^1.0.0",
     "lodash": "^4.17.4",
     "make-runnable": "^1.3.6",
+    "mime-types": "^2.1.35",
     "mkdirp": "^1.0.4",
     "module-alias": "^2.1.0",
     "moment-timezone": "^0.5.13",
     "puppeteer": "^13.6.0",
     "raven": "^2.1.1",
+    "s3-sync-client": "^4.3.1",
     "twitter-api-v2": "^1.15.0",
     "v-click-outside": "^3.0.1",
     "vue": "^2.6.11",

src/app/index.js

@@ -4,6 +4,7 @@ module.exports = {
     splatnet: require('./updater').updateAll,
     twitter: require('./twitter').maybePostTweets,
     twitterTest: require('./twitter').testScreenshots,
+    sync: require('./sync').sync,
 };
 
 require('make-runnable/custom')({ printOutputFrame: false });

src/app/sync/S3Syncer.js

@@ -0,0 +1,75 @@
+const path = require('path');
+const { S3Client } = require('@aws-sdk/client-s3');
+const { S3SyncClient } = require('s3-sync-client');
+const mime = require('mime-types');
+
+class S3Syncer
+{
+  async sync() {
+    await this.download();
+    await this.upload();
+  }
+
+  download() {
+    this.log('Downloading files...');
+
+    return Promise.all([
+      this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, {
+        filters: [
+          { exclude: () => true }, // Exclude everything by default
+          { include: (key) => key.startsWith('assets/splatnet/') },
+          { include: (key) => key.startsWith('data/') },
+        ],
+      }),
+      this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`),
+    ]);
+  }
+
+  upload() {
+    this.log('Uploading files...');
+
+    return Promise.all([
+      this.syncClient.sync(`${this.localPath}/dist`, this.publicBucket, {
+        commandInput: input => ({
+          ACL: 'public-read',
+          ContentType: mime.lookup(input.Key),
+        }),
+      }),
+      this.syncClient.sync(`${this.localPath}/storage`, this.privateBucket),
+    ]);
+  }
+
+  get s3Client() {
+    return this._s3Client ??= new S3Client({
+      endpoint: process.env.AWS_S3_ENDPOINT,
+      region: process.env.AWS_S3_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+      },
+    });
+  }
+
+  /** @returns {S3SyncClient} */
+  get syncClient() {
+    return this._syncClient ??= new S3SyncClient({ client: this.s3Client });
+  }
+
+  get publicBucket() {
+    return `s3://${process.env.AWS_S3_BUCKET}`;
+  }
+
+  get privateBucket() {
+    return `s3://${process.env.AWS_S3_PRIVATE_BUCKET}`;
+  }
+
+  get localPath() {
+    return path.resolve('.');
+  }
+
+  log(message) {
+    console.log(`[S3] ${message}`);
+  }
+}
+
+module.exports = S3Syncer;

src/app/sync/index.js

@@ -0,0 +1,22 @@
+const S3Syncer = require('./S3Syncer');
+
+function canSync() {
+  return !!(
+    process.env.AWS_ACCESS_KEY_ID &&
+    process.env.AWS_SECRET_ACCESS_KEY &&
+    process.env.AWS_S3_BUCKET &&
+    process.env.AWS_S3_PRIVATE_BUCKET
+  );
+}
+
+function sync() {
+  if (!canSync()) {
+    console.warn('Missing S3 connection parameters');
+    return;
+  }
+
+  const syncer = new S3Syncer();
+  return syncer.sync();
+}
+
+module.exports = { canSync, sync };

src/app/updater/updateAll.js

@@ -4,6 +4,8 @@ const TimelineUpdater = require('./updaters/TimelineUpdater');
 const OriginalGearImageUpdater = require('./updaters/OriginalGearImageUpdater');
 const FestivalsUpdater = require('./updaters/FestivalsUpdater');
 const MerchandisesUpdater = require('./updaters/MerchandisesUpdater');
+const S3Syncer = require('../sync/S3Syncer');
+const { canSync } = require('../sync');
 
 const updaters = [
     new OriginalGearImageUpdater,
@@ -17,6 +19,8 @@ const updaters = [
 ];
 
 async function updateAll() {
+    const syncer = canSync() ? new S3Syncer() : null;
+
     for (let updater of updaters) {
         try {
             await updater.update();
@@ -25,6 +29,10 @@ async function updateAll() {
         }
     }
 
+    if (syncer) {
+        await syncer.upload();
+    }
+
     return 'Done';
 }