diff --git a/app/data/index.mjs b/app/data/index.mjs index 8f7eae7..53781f1 100644 --- a/app/data/index.mjs +++ b/app/data/index.mjs @@ -5,6 +5,8 @@ import CoopUpdater from './updaters/CoopUpdater.mjs'; import FestivalUpdater from './updaters/FestivalUpdater.mjs'; import XRankUpdater from './updaters/XRankUpdater.mjs'; import StagesUpdater from './updaters/StagesUpdater.mjs'; +import S3Syncer from '../sync/S3Syncer.mjs'; +import { canSync } from '../sync/index.mjs'; function updaters() { return [ @@ -50,5 +52,9 @@ export async function update(config = 'default') { } } + if (canSync()) { + await (new S3Syncer).upload(); + } + console.info(`Done running ${config} updaters`); } diff --git a/app/index.mjs b/app/index.mjs index 2128c32..85c23fa 100644 --- a/app/index.mjs +++ b/app/index.mjs @@ -10,6 +10,7 @@ import BlueskyClient from './social/clients/BlueskyClient.mjs'; import ThreadsClient from './social/clients/ThreadsClient.mjs'; import { archiveData } from './data/DataArchiver.mjs'; import { sentryInit } from './common/sentry.mjs'; +import { sync, syncUpload, syncDownload } from './sync/index.mjs'; consoleStamp(console); dotenv.config(); @@ -26,6 +27,9 @@ const actions = { splatnet: update, warmCaches, dataArchive: archiveData, + sync, + syncUpload, + syncDownload, }; const command = process.argv[2]; diff --git a/app/social/index.mjs b/app/social/index.mjs index 9634eea..6518915 100644 --- a/app/social/index.mjs +++ b/app/social/index.mjs @@ -16,6 +16,8 @@ import EggstraWorkUpcomingStatus from './generators/EggstraWorkUpcomingStatus.mj import BlueskyClient from './clients/BlueskyClient.mjs'; import ChallengeStatus from './generators/ChallengeStatus.mjs'; import ThreadsClient from './clients/ThreadsClient.mjs'; +import S3Syncer from '../sync/S3Syncer.mjs'; +import { canSync } from '../sync/index.mjs'; function defaultStatusGenerators() { return [ @@ -63,8 +65,12 @@ export function testStatusGeneratorManager(additionalClients) { ); } -export function sendStatuses() { - return defaultStatusGeneratorManager().sendStatuses(); +export async function sendStatuses() { + await defaultStatusGeneratorManager().sendStatuses(); + + if (canSync()) { + await (new S3Syncer).upload(); + } } export function testStatuses(additionalClients = []) { diff --git a/app/sync/S3Syncer.mjs b/app/sync/S3Syncer.mjs new file mode 100644 index 0000000..11b29e3 --- /dev/null +++ b/app/sync/S3Syncer.mjs @@ -0,0 +1,77 @@ +import path from 'path'; +import { S3Client } from '@aws-sdk/client-s3'; +import { S3SyncClient } from 's3-sync-client'; +import mime from 'mime-types'; + +export default class S3Syncer +{ + download() { + this.log('Downloading files...'); + + return Promise.all([ + this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, { + filters: this.filters, + }), + this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`), + ]); + } + + upload() { + this.log('Uploading files...'); + + return Promise.all([ + this.syncClient.sync(`${this.localPath}/dist`, this.publicBucket, { + filters: this.filters, + commandInput: input => ({ + ACL: 'public-read', + ContentType: mime.lookup(input.Key), + CacheControl: input.Key.startsWith('data/') + ? 'no-cache, stale-while-revalidate=5, stale-if-error=86400' + : undefined, + }), + }), + this.syncClient.sync(`${this.localPath}/storage`, this.privateBucket), + ]); + } + + get s3Client() { + return this._s3Client ??= new S3Client({ + endpoint: process.env.AWS_S3_ENDPOINT, + region: process.env.AWS_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, + }); + } + + /** @returns {S3SyncClient} */ + get syncClient() { + return this._syncClient ??= new S3SyncClient({ client: this.s3Client }); + } + + get publicBucket() { + return `s3://${process.env.AWS_S3_BUCKET}`; + } + + get privateBucket() { + return `s3://${process.env.AWS_S3_PRIVATE_BUCKET}`; + } + + get localPath() { + return path.resolve('.'); + } + + get filters() { + return [ + { exclude: () => true }, // Exclude everything by default + { include: (key) => key.startsWith('assets/splatnet/') }, + { include: (key) => key.startsWith('data/') }, + { include: (key) => key.startsWith('status-screenshots/') }, + ]; + } + + log(message) { + console.log(`[S3] ${message}`); + } +} diff --git a/app/sync/index.mjs b/app/sync/index.mjs new file mode 100644 index 0000000..ddb7dea --- /dev/null +++ b/app/sync/index.mjs @@ -0,0 +1,41 @@ +import S3Syncer from './S3Syncer.mjs'; + +export function canSync() { + return !!( + process.env.AWS_ACCESS_KEY_ID && + process.env.AWS_SECRET_ACCESS_KEY && + process.env.AWS_S3_BUCKET && + process.env.AWS_S3_PRIVATE_BUCKET + ); +} + +async function doSync(download, upload) { + if (!canSync()) { + console.warn('Missing S3 connection parameters'); + return; + } + + const syncer = new S3Syncer(); + + if (download) { + console.info('Downloading files...'); + await syncer.download(); + } + + if (upload) { + console.info('Uploading files...'); + await syncer.upload(); + } +} + +export function sync() { + return doSync(true, true); +} + +export function syncUpload() { + return doSync(false, true); +} + +export function syncDownload() { + return doSync(true, false); +} diff --git a/package-lock.json b/package-lock.json index 407aaa9..06cf811 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "nxapi": "^1.4.0", "pinia": "^2.0.22", "puppeteer-core": "^23.8.0", + "s3-sync-client": "^4.3.1", "sharp": "^0.32.0", "threads-api": "^1.4.0", "twitter-api-v2": "^1.12.7", @@ -261,6 +262,47 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, + "node_modules/@aws-sdk/abort-controller": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/abort-controller/-/abort-controller-3.370.0.tgz", + "integrity": "sha512-/W4arzC/+yVW/cvEXbuwvG0uly4yFSZnnIA+gkqgAm+0HVfacwcPpNf4BjyxjnvIdh03l7w2DriF6MlKUfiQ3A==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.370.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/abort-controller/node_modules/@aws-sdk/types": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.370.0.tgz", + "integrity": "sha512-8PGMKklSkRKjunFhzM2y5Jm0H2TBu7YRNISdYzXLUHKSP9zlMEYagseKVdmox0zKHf1LXVNuSlUV2b6SRrieCQ==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/abort-controller/node_modules/@smithy/types": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-1.2.0.tgz", + "integrity": "sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@aws-sdk/client-s3": { "version": "3.535.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.535.0.tgz", @@ -8800,6 +8842,19 @@ "tslib": "^2.1.0" } }, + "node_modules/s3-sync-client": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/s3-sync-client/-/s3-sync-client-4.3.1.tgz", + "integrity": "sha512-nWbbKCNnXmWvD8XwdWhX25VNxIhgQEm6vXqSYjwyBNZI07OuMOr/LNOYmEPcLfqFFjy55ZNcFSBI18W29ybuUw==", + "license": "MIT", + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/abort-controller": "^3.x.x", + "@aws-sdk/client-s3": "^3.x.x" + } + }, "node_modules/safe-array-concat": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", diff --git a/package.json b/package.json index 48440e8..b2037e9 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,10 @@ "splatnet": "node app/index.mjs splatnet default", "splatnet:all": "node app/index.mjs splatnet all", "warmCaches": "node app/index.mjs warmCaches", - "data:archive": "node app/index.mjs dataArchive" + "data:archive": "node app/index.mjs dataArchive", + "sync": "node app/index.mjs sync", + "sync:upload": "node app/index.mjs syncUpload", + "sync:download": "node app/index.mjs syncDownload" }, "dependencies": { "@atproto/api": "^0.11.2", @@ -40,6 +43,7 @@ "nxapi": "^1.4.0", "pinia": "^2.0.22", "puppeteer-core": "^23.8.0", + "s3-sync-client": "^4.3.1", "sharp": "^0.32.0", "threads-api": "^1.4.0", "twitter-api-v2": "^1.12.7",