diff --git a/.env.example b/.env.example index 69cb6f2..7db2db9 100644 --- a/.env.example +++ b/.env.example @@ -12,14 +12,20 @@ VUE_APP_GOOGLE_ANALYTICS_ID= # (Optional) Sentry error reporting (https://sentry.io) SENTRY_DSN= -# (Optional) S3 parameters +# (Optional) Public S3-compatible storage parameters AWS_S3_ENDPOINT= AWS_REGION= AWS_S3_BUCKET= -AWS_S3_PRIVATE_BUCKET= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= +# (Optional) Private S3-compatible storage parameters +PRIVATE_AWS_S3_ENDPOINT= +PRIVATE_AWS_REGION= +PRIVATE_AWS_S3_BUCKET= +PRIVATE_AWS_ACCESS_KEY_ID= +PRIVATE_AWS_SECRET_ACCESS_KEY= + # (Optional) Bluesky API parameters BLUESKY_SERVICE=https://bsky.social BLUESKY_IDENTIFIER=splatoon2.ink # Handle or email address diff --git a/package.json b/package.json index 5c1e4ab..11fa1b6 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "scripts": { "serve": "vue-cli-service serve", "build": "vue-cli-service build --modern --no-clean", + "test": "node --test", "lint": "vue-cli-service lint", "cron": "node src/app/cron", "start": "npm run sync:download && npm run splatnet && npm run twitter && npm run cron", diff --git a/src/app/sync/S3Syncer.js b/src/app/sync/S3Syncer.js index 5a939b0..f3c1dc1 100644 --- a/src/app/sync/S3Syncer.js +++ b/src/app/sync/S3Syncer.js @@ -5,14 +5,48 @@ const mime = require('mime-types'); class S3Syncer { + constructor({ + publicConfig = S3Syncer.publicConfigFromEnvironment(), + privateConfig = S3Syncer.privateConfigFromEnvironment(), + localPath = path.resolve('.'), + publicSyncClient, + privateSyncClient, + } = {}) { + this.publicConfig = publicConfig; + this.privateConfig = privateConfig; + this._localPath = localPath; + this._publicSyncClient = publicSyncClient; + this._privateSyncClient = privateSyncClient; + } + + static publicConfigFromEnvironment() { + return { + endpoint: process.env.AWS_S3_ENDPOINT, + region: process.env.AWS_REGION, + bucket: process.env.AWS_S3_BUCKET, + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }; + } + + static privateConfigFromEnvironment() { + return { + endpoint: process.env.PRIVATE_AWS_S3_ENDPOINT, + region: process.env.PRIVATE_AWS_REGION, + bucket: process.env.PRIVATE_AWS_S3_BUCKET, + accessKeyId: process.env.PRIVATE_AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.PRIVATE_AWS_SECRET_ACCESS_KEY, + }; + } + download() { this.log('Downloading files...'); return Promise.all([ - this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, { + this.publicSyncClient.sync(this.publicBucket, `${this.localPath}/dist`, { filters: this.filters, }), - this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`), + this.privateSyncClient.sync(this.privateBucket, `${this.localPath}/storage`), ]); } @@ -20,46 +54,60 @@ class S3Syncer this.log('Uploading files...'); return Promise.all([ - this.syncClient.sync(`${this.localPath}/dist`, this.publicBucket, { + this.publicSyncClient.sync(`${this.localPath}/dist`, this.publicBucket, { filters: this.filters, commandInput: input => ({ - ACL: 'public-read', - ContentType: mime.lookup(input.Key), + ContentType: mime.lookup(input.Key) || undefined, CacheControl: input.Key.startsWith('data/') ? 'no-cache, stale-while-revalidate=5, stale-if-error=86400' : undefined, }), }), - this.syncClient.sync(`${this.localPath}/storage`, this.privateBucket), + this.privateSyncClient.sync(`${this.localPath}/storage`, this.privateBucket), ]); } - get s3Client() { - return this._s3Client ??= new S3Client({ - endpoint: process.env.AWS_S3_ENDPOINT, - region: process.env.AWS_REGION, + createS3Client(config) { + return new S3Client({ + endpoint: config.endpoint, + region: config.region, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', credentials: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, }, }); } + get publicS3Client() { + return this._publicS3Client ??= this.createS3Client(this.publicConfig); + } + + get privateS3Client() { + return this._privateS3Client ??= this.createS3Client(this.privateConfig); + } + /** @returns {S3SyncClient} */ - get syncClient() { - return this._syncClient ??= new S3SyncClient({ client: this.s3Client }); + get publicSyncClient() { + return this._publicSyncClient ??= new S3SyncClient({ client: this.publicS3Client }); + } + + /** @returns {S3SyncClient} */ + get privateSyncClient() { + return this._privateSyncClient ??= new S3SyncClient({ client: this.privateS3Client }); } get publicBucket() { - return `s3://${process.env.AWS_S3_BUCKET}`; + return `s3://${this.publicConfig.bucket}`; } get privateBucket() { - return `s3://${process.env.AWS_S3_PRIVATE_BUCKET}`; + return `s3://${this.privateConfig.bucket}`; } get localPath() { - return path.resolve('.'); + return this._localPath; } get filters() { diff --git a/src/app/sync/index.js b/src/app/sync/index.js index 5bb78b3..d98b6a0 100644 --- a/src/app/sync/index.js +++ b/src/app/sync/index.js @@ -1,12 +1,12 @@ const S3Syncer = require('./S3Syncer'); function canSync() { - return !!( - process.env.AWS_ACCESS_KEY_ID && - process.env.AWS_SECRET_ACCESS_KEY && - process.env.AWS_S3_BUCKET && - process.env.AWS_S3_PRIVATE_BUCKET - ); + const configurations = [ + S3Syncer.publicConfigFromEnvironment(), + S3Syncer.privateConfigFromEnvironment(), + ]; + + return configurations.every(config => Object.values(config).every(Boolean)); } async function doSync(download, upload) { diff --git a/test/sync/S3Syncer.test.js b/test/sync/S3Syncer.test.js new file mode 100644 index 0000000..13c3ecc --- /dev/null +++ b/test/sync/S3Syncer.test.js @@ -0,0 +1,87 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const S3Syncer = require('../../src/app/sync/S3Syncer'); + +const publicConfig = { + endpoint: 'https://public.example.com', + region: 'auto', + bucket: 'public-assets', + accessKeyId: 'public-key', + secretAccessKey: 'public-secret', +}; + +const privateConfig = { + endpoint: 'https://private.example.com', + region: 'us-west-1', + bucket: 'private-state', + accessKeyId: 'private-key', + secretAccessKey: 'private-secret', +}; + +test('uses independent public and private bucket configuration', () => { + const syncer = new S3Syncer({ publicConfig, privateConfig }); + + assert.equal(syncer.publicBucket, 's3://public-assets'); + assert.equal(syncer.privateBucket, 's3://private-state'); +}); + +test('downloads public and private files through their own clients', async () => { + const publicCalls = []; + const privateCalls = []; + const syncer = new S3Syncer({ + publicConfig, + privateConfig, + localPath: '/app', + publicSyncClient: { + sync: (...args) => publicCalls.push(args), + }, + privateSyncClient: { + sync: (...args) => privateCalls.push(args), + }, + }); + + await syncer.download(); + + assert.equal(publicCalls.length, 1); + assert.equal(publicCalls[0][0], 's3://public-assets'); + assert.equal(publicCalls[0][1], '/app/dist'); + assert.equal(publicCalls[0][2].filters.length, 4); + assert.deepEqual(privateCalls, [[ + 's3://private-state', + '/app/storage', + ]]); +}); + +test('uploads public and private files through their own clients', async () => { + const publicCalls = []; + const privateCalls = []; + const syncer = new S3Syncer({ + publicConfig, + privateConfig, + localPath: '/app', + publicSyncClient: { + sync: (...args) => publicCalls.push(args), + }, + privateSyncClient: { + sync: (...args) => privateCalls.push(args), + }, + }); + + await syncer.upload(); + + assert.equal(publicCalls.length, 1); + assert.equal(publicCalls[0][0], '/app/dist'); + assert.equal(publicCalls[0][1], 's3://public-assets'); + + const publicDataInput = publicCalls[0][2].commandInput({ Key: 'data/schedules.json' }); + assert.deepEqual(publicDataInput, { + ContentType: 'application/json', + CacheControl: 'no-cache, stale-while-revalidate=5, stale-if-error=86400', + }); + + assert.deepEqual(privateCalls, [[ + '/app/storage', + 's3://private-state', + ]]); +}); diff --git a/test/sync/index.test.js b/test/sync/index.test.js new file mode 100644 index 0000000..8e4ce9b --- /dev/null +++ b/test/sync/index.test.js @@ -0,0 +1,58 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const { canSync } = require('../../src/app/sync'); + +const storageEnvironment = { + AWS_S3_ENDPOINT: 'https://public.example.com', + AWS_REGION: 'auto', + AWS_S3_BUCKET: 'public-assets', + AWS_ACCESS_KEY_ID: 'public-key', + AWS_SECRET_ACCESS_KEY: 'public-secret', + PRIVATE_AWS_S3_ENDPOINT: 'https://private.example.com', + PRIVATE_AWS_REGION: 'us-west-1', + PRIVATE_AWS_S3_BUCKET: 'private-state', + PRIVATE_AWS_ACCESS_KEY_ID: 'private-key', + PRIVATE_AWS_SECRET_ACCESS_KEY: 'private-secret', +}; + +function withStorageEnvironment(overrides, callback) { + const originalValues = {}; + + for (const name of Object.keys(storageEnvironment)) { + originalValues[name] = process.env[name]; + const value = Object.hasOwn(overrides, name) + ? overrides[name] + : storageEnvironment[name]; + + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + + try { + callback(); + } finally { + for (const [name, value] of Object.entries(originalValues)) { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + } +} + +test('can sync when both public and private storage are configured', () => { + withStorageEnvironment({}, () => assert.equal(canSync(), true)); +}); + +test('cannot sync when any public or private storage value is missing', () => { + for (const name of Object.keys(storageEnvironment)) { + withStorageEnvironment({ [name]: undefined }, () => { + assert.equal(canSync(), false, `${name} should be required`); + }); + } +});