Merge pull request #59 from misenhower/codex/split-public-private-storage

Split public and private storage credentials

.env.example

@@ -12,14 +12,20 @@ VUE_APP_GOOGLE_ANALYTICS_ID=
 # (Optional) Sentry error reporting (https://sentry.io)
 SENTRY_DSN=
 
-# (Optional) S3 parameters
+# (Optional) Public S3-compatible storage parameters
 AWS_S3_ENDPOINT=
 AWS_REGION=
 AWS_S3_BUCKET=
-AWS_S3_PRIVATE_BUCKET=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 
+# (Optional) Private S3-compatible storage parameters
+PRIVATE_AWS_S3_ENDPOINT=
+PRIVATE_AWS_REGION=
+PRIVATE_AWS_S3_BUCKET=
+PRIVATE_AWS_ACCESS_KEY_ID=
+PRIVATE_AWS_SECRET_ACCESS_KEY=
+
 # (Optional) Bluesky API parameters
 BLUESKY_SERVICE=https://bsky.social
 BLUESKY_IDENTIFIER=splatoon2.ink # Handle or email address

package.json

@@ -7,6 +7,7 @@
   "scripts": {
     "serve": "vue-cli-service serve",
     "build": "vue-cli-service build --modern --no-clean",
+    "test": "node --test",
     "lint": "vue-cli-service lint",
     "cron": "node src/app/cron",
     "start": "npm run sync:download && npm run splatnet && npm run twitter && npm run cron",

src/app/sync/S3Syncer.js

@@ -5,14 +5,48 @@ const mime = require('mime-types');
 
 class S3Syncer
 {
+  constructor({
+    publicConfig = S3Syncer.publicConfigFromEnvironment(),
+    privateConfig = S3Syncer.privateConfigFromEnvironment(),
+    localPath = path.resolve('.'),
+    publicSyncClient,
+    privateSyncClient,
+  } = {}) {
+    this.publicConfig = publicConfig;
+    this.privateConfig = privateConfig;
+    this._localPath = localPath;
+    this._publicSyncClient = publicSyncClient;
+    this._privateSyncClient = privateSyncClient;
+  }
+
+  static publicConfigFromEnvironment() {
+    return {
+      endpoint: process.env.AWS_S3_ENDPOINT,
+      region: process.env.AWS_REGION,
+      bucket: process.env.AWS_S3_BUCKET,
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+    };
+  }
+
+  static privateConfigFromEnvironment() {
+    return {
+      endpoint: process.env.PRIVATE_AWS_S3_ENDPOINT,
+      region: process.env.PRIVATE_AWS_REGION,
+      bucket: process.env.PRIVATE_AWS_S3_BUCKET,
+      accessKeyId: process.env.PRIVATE_AWS_ACCESS_KEY_ID,
+      secretAccessKey: process.env.PRIVATE_AWS_SECRET_ACCESS_KEY,
+    };
+  }
+
   download() {
     this.log('Downloading files...');
 
     return Promise.all([
-      this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, {
+      this.publicSyncClient.sync(this.publicBucket, `${this.localPath}/dist`, {
         filters: this.filters,
       }),
-      this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`),
+      this.privateSyncClient.sync(this.privateBucket, `${this.localPath}/storage`),
     ]);
   }
 
@@ -20,46 +54,60 @@ class S3Syncer
     this.log('Uploading files...');
 
     return Promise.all([
-      this.syncClient.sync(`${this.localPath}/dist`, this.publicBucket, {
+      this.publicSyncClient.sync(`${this.localPath}/dist`, this.publicBucket, {
         filters: this.filters,
         commandInput: input => ({
-          ACL: 'public-read',
-          ContentType: mime.lookup(input.Key),
+          ContentType: mime.lookup(input.Key) || undefined,
           CacheControl: input.Key.startsWith('data/')
             ? 'no-cache, stale-while-revalidate=5, stale-if-error=86400'
             : undefined,
         }),
       }),
-      this.syncClient.sync(`${this.localPath}/storage`, this.privateBucket),
+      this.privateSyncClient.sync(`${this.localPath}/storage`, this.privateBucket),
     ]);
   }
 
-  get s3Client() {
-    return this._s3Client ??= new S3Client({
-      endpoint: process.env.AWS_S3_ENDPOINT,
-      region: process.env.AWS_REGION,
+  createS3Client(config) {
+    return new S3Client({
+      endpoint: config.endpoint,
+      region: config.region,
+      requestChecksumCalculation: 'WHEN_REQUIRED',
+      responseChecksumValidation: 'WHEN_REQUIRED',
       credentials: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+        accessKeyId: config.accessKeyId,
+        secretAccessKey: config.secretAccessKey,
       },
     });
   }
 
+  get publicS3Client() {
+    return this._publicS3Client ??= this.createS3Client(this.publicConfig);
+  }
+
+  get privateS3Client() {
+    return this._privateS3Client ??= this.createS3Client(this.privateConfig);
+  }
+
   /** @returns {S3SyncClient} */
-  get syncClient() {
-    return this._syncClient ??= new S3SyncClient({ client: this.s3Client });
+  get publicSyncClient() {
+    return this._publicSyncClient ??= new S3SyncClient({ client: this.publicS3Client });
+  }
+
+  /** @returns {S3SyncClient} */
+  get privateSyncClient() {
+    return this._privateSyncClient ??= new S3SyncClient({ client: this.privateS3Client });
   }
 
   get publicBucket() {
-    return `s3://${process.env.AWS_S3_BUCKET}`;
+    return `s3://${this.publicConfig.bucket}`;
   }
 
   get privateBucket() {
-    return `s3://${process.env.AWS_S3_PRIVATE_BUCKET}`;
+    return `s3://${this.privateConfig.bucket}`;
   }
 
   get localPath() {
-    return path.resolve('.');
+    return this._localPath;
   }
 
   get filters() {

src/app/sync/index.js

@@ -1,12 +1,12 @@
 const S3Syncer = require('./S3Syncer');
 
 function canSync() {
-  return !!(
-    process.env.AWS_ACCESS_KEY_ID &&
-    process.env.AWS_SECRET_ACCESS_KEY &&
-    process.env.AWS_S3_BUCKET &&
-    process.env.AWS_S3_PRIVATE_BUCKET
-  );
+  const configurations = [
+    S3Syncer.publicConfigFromEnvironment(),
+    S3Syncer.privateConfigFromEnvironment(),
+  ];
+
+  return configurations.every(config => Object.values(config).every(Boolean));
 }
 
 async function doSync(download, upload) {

test/sync/S3Syncer.test.js

@@ -0,0 +1,87 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+
+const S3Syncer = require('../../src/app/sync/S3Syncer');
+
+const publicConfig = {
+  endpoint: 'https://public.example.com',
+  region: 'auto',
+  bucket: 'public-assets',
+  accessKeyId: 'public-key',
+  secretAccessKey: 'public-secret',
+};
+
+const privateConfig = {
+  endpoint: 'https://private.example.com',
+  region: 'us-west-1',
+  bucket: 'private-state',
+  accessKeyId: 'private-key',
+  secretAccessKey: 'private-secret',
+};
+
+test('uses independent public and private bucket configuration', () => {
+  const syncer = new S3Syncer({ publicConfig, privateConfig });
+
+  assert.equal(syncer.publicBucket, 's3://public-assets');
+  assert.equal(syncer.privateBucket, 's3://private-state');
+});
+
+test('downloads public and private files through their own clients', async () => {
+  const publicCalls = [];
+  const privateCalls = [];
+  const syncer = new S3Syncer({
+    publicConfig,
+    privateConfig,
+    localPath: '/app',
+    publicSyncClient: {
+      sync: (...args) => publicCalls.push(args),
+    },
+    privateSyncClient: {
+      sync: (...args) => privateCalls.push(args),
+    },
+  });
+
+  await syncer.download();
+
+  assert.equal(publicCalls.length, 1);
+  assert.equal(publicCalls[0][0], 's3://public-assets');
+  assert.equal(publicCalls[0][1], '/app/dist');
+  assert.equal(publicCalls[0][2].filters.length, 4);
+  assert.deepEqual(privateCalls, [[
+    's3://private-state',
+    '/app/storage',
+  ]]);
+});
+
+test('uploads public and private files through their own clients', async () => {
+  const publicCalls = [];
+  const privateCalls = [];
+  const syncer = new S3Syncer({
+    publicConfig,
+    privateConfig,
+    localPath: '/app',
+    publicSyncClient: {
+      sync: (...args) => publicCalls.push(args),
+    },
+    privateSyncClient: {
+      sync: (...args) => privateCalls.push(args),
+    },
+  });
+
+  await syncer.upload();
+
+  assert.equal(publicCalls.length, 1);
+  assert.equal(publicCalls[0][0], '/app/dist');
+  assert.equal(publicCalls[0][1], 's3://public-assets');
+
+  const publicDataInput = publicCalls[0][2].commandInput({ Key: 'data/schedules.json' });
+  assert.deepEqual(publicDataInput, {
+    ContentType: 'application/json',
+    CacheControl: 'no-cache, stale-while-revalidate=5, stale-if-error=86400',
+  });
+
+  assert.deepEqual(privateCalls, [[
+    '/app/storage',
+    's3://private-state',
+  ]]);
+});

test/sync/index.test.js

@@ -0,0 +1,58 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+
+const { canSync } = require('../../src/app/sync');
+
+const storageEnvironment = {
+  AWS_S3_ENDPOINT: 'https://public.example.com',
+  AWS_REGION: 'auto',
+  AWS_S3_BUCKET: 'public-assets',
+  AWS_ACCESS_KEY_ID: 'public-key',
+  AWS_SECRET_ACCESS_KEY: 'public-secret',
+  PRIVATE_AWS_S3_ENDPOINT: 'https://private.example.com',
+  PRIVATE_AWS_REGION: 'us-west-1',
+  PRIVATE_AWS_S3_BUCKET: 'private-state',
+  PRIVATE_AWS_ACCESS_KEY_ID: 'private-key',
+  PRIVATE_AWS_SECRET_ACCESS_KEY: 'private-secret',
+};
+
+function withStorageEnvironment(overrides, callback) {
+  const originalValues = {};
+
+  for (const name of Object.keys(storageEnvironment)) {
+    originalValues[name] = process.env[name];
+    const value = Object.hasOwn(overrides, name)
+      ? overrides[name]
+      : storageEnvironment[name];
+
+    if (value === undefined) {
+      delete process.env[name];
+    } else {
+      process.env[name] = value;
+    }
+  }
+
+  try {
+    callback();
+  } finally {
+    for (const [name, value] of Object.entries(originalValues)) {
+      if (value === undefined) {
+        delete process.env[name];
+      } else {
+        process.env[name] = value;
+      }
+    }
+  }
+}
+
+test('can sync when both public and private storage are configured', () => {
+  withStorageEnvironment({}, () => assert.equal(canSync(), true));
+});
+
+test('cannot sync when any public or private storage value is missing', () => {
+  for (const name of Object.keys(storageEnvironment)) {
+    withStorageEnvironment({ [name]: undefined }, () => {
+      assert.equal(canSync(), false, `${name} should be required`);
+    });
+  }
+});