diff --git a/.env.example b/.env.example index 32cbda4..1fc6efa 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,13 @@ AWS_S3_PRIVATE_BUCKET= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= +# Cloudflare R2 public data and images +R2_ENDPOINT= +R2_BUCKET=splatoon3-ink-assets +R2_PUBLIC_URL=https://assets.splatoon3.ink +R2_ACCESS_KEY_ID= +R2_SECRET_ACCESS_KEY= + # Twitter API parameters TWITTER_CONSUMER_KEY= TWITTER_CONSUMER_SECRET= diff --git a/app/data/index.mjs b/app/data/index.mjs index 1352baa..85a5d62 100644 --- a/app/data/index.mjs +++ b/app/data/index.mjs @@ -1,6 +1,6 @@ import * as Sentry from '@sentry/node'; import S3Syncer from '../sync/S3Syncer.mjs'; -import { canSync } from '../sync/index.mjs'; +import { canSyncS3, canUpload, upload } from '../sync/index.mjs'; import GearUpdater from './updaters/GearUpdater.mjs'; import StageScheduleUpdater from './updaters/StageScheduleUpdater.mjs'; import CoopUpdater from './updaters/CoopUpdater.mjs'; @@ -44,7 +44,7 @@ export async function update(config = 'default') { let settings = configs[config]; // Download private files to get updated tokens if needed - if (canSync()) { + if (canSyncS3()) { await (new S3Syncer).download(false); } @@ -58,9 +58,9 @@ export async function update(config = 'default') { } })); - if (canSync()) { + if (canUpload()) { await ImageProcessor.onIdle(); - await (new S3Syncer).upload(); + await upload(); } console.info(`Done running ${config} updaters`); diff --git a/app/social/index.mjs b/app/social/index.mjs index 31a0785..e234c74 100644 --- a/app/social/index.mjs +++ b/app/social/index.mjs @@ -1,5 +1,4 @@ -import S3Syncer from '../sync/S3Syncer.mjs'; -import { canSync } from '../sync/index.mjs'; +import { canUpload, upload } from '../sync/index.mjs'; import FileWriter from './clients/FileWriter.mjs'; import ImageWriter from './clients/ImageWriter.mjs'; import MastodonClient from './clients/MastodonClient.mjs'; @@ -68,8 +67,8 @@ export function testStatusGeneratorManager(additionalClients) { export async function sendStatuses() { await defaultStatusGeneratorManager().sendStatuses(); - if (canSync()) { - await (new S3Syncer).upload(); + if (canUpload()) { + await upload(); } } diff --git a/app/sync/R2Syncer.mjs b/app/sync/R2Syncer.mjs new file mode 100644 index 0000000..e590f9c --- /dev/null +++ b/app/sync/R2Syncer.mjs @@ -0,0 +1,121 @@ +import path from 'node:path'; +import fs from 'node:fs'; +import { S3Client } from '@aws-sdk/client-s3'; +import { S3SyncClient } from 's3-sync-client'; +import mime from 'mime-types'; + +// s3-sync-client cannot replace multipart upload bodies. Force direct PUTs so +// rewritten data can never bypass the URL rewrite; R2 will reject an oversized object +// instead of silently publishing legacy URLs. +const forceSinglePartUploads = Number.MAX_SAFE_INTEGER; +const dataCacheControl = 'no-cache, stale-while-revalidate=5, stale-if-error=86400'; +const rewrittenDataExtensions = ['.ics', '.json']; + +function baseUrl(url) { + return url.replace(/\/+$/, ''); +} + +function isImage(key) { + let contentType = mime.lookup(key); + return typeof contentType === 'string' && contentType.startsWith('image/'); +} + +function isRewrittenData(key) { + return rewrittenDataExtensions.some(extension => key.endsWith(extension)); +} + +export default class R2Syncer +{ + constructor({ config = {}, localPath, s3Client, syncClient } = {}) { + this.config = config; + this._s3Client = s3Client; + this._syncClient = syncClient; + this._localPath = localPath; + } + + async upload() { + this.log('Uploading files...'); + + return this.syncClient.sync(this.localPath, this.publicBucket, { + filters: this.filters, + relocations: this.relocations, + partSize: forceSinglePartUploads, + commandInput: input => this.commandInput(input), + }); + } + + commandInput(input) { + let result = { + ContentType: mime.lookup(input.Key) || undefined, + CacheControl: input.Key.startsWith('data/') + ? dataCacheControl + : undefined, + }; + + if (isRewrittenData(input.Key)) { + let source = fs.readFileSync(input.Body.path, 'utf8'); + result.Body = Buffer.from(source.replaceAll( + this.legacyAssetUrl, + this.r2AssetUrl, + )); + result.ContentLength = result.Body.length; + input.Body.resume(); + } + + return result; + } + + get s3Client() { + return this._s3Client ??= new S3Client({ + endpoint: this.config.endpoint, + region: 'auto', + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + credentials: { + accessKeyId: this.config.accessKeyId, + secretAccessKey: this.config.secretAccessKey, + }, + }); + } + + /** @member {S3SyncClient} */ + get syncClient() { + return this._syncClient ??= new S3SyncClient({ client: this.s3Client }); + } + + get publicBucket() { + return `s3://${this.config.bucket}`; + } + + get localPath() { + return this._localPath ?? path.resolve('dist'); + } + + get legacyAssetUrl() { + return `${baseUrl(this.config.siteUrl)}/assets/splatnet/`; + } + + get r2AssetUrl() { + return `${baseUrl(this.config.publicUrl)}/splatnet/`; + } + + get filters() { + return [ + { exclude: () => true }, + { include: key => key.startsWith('assets/splatnet/') && isImage(key) }, + { include: key => key.startsWith('data/') && isRewrittenData(key) }, + { exclude: key => key.startsWith('data/archive/') }, + { include: key => key.startsWith('status-screenshots/') && isImage(key) }, + ]; + } + + get relocations() { + return [key => key.startsWith('assets/splatnet/') + ? key.slice('assets/'.length) + : key]; + } + + log(message) { + console.log(`[R2] ${message}`); + } +} diff --git a/app/sync/R2Syncer.test.mjs b/app/sync/R2Syncer.test.mjs new file mode 100644 index 0000000..b52272d --- /dev/null +++ b/app/sync/R2Syncer.test.mjs @@ -0,0 +1,205 @@ +import path from 'node:path'; +import os from 'node:os'; +import fs from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; +import { ListObjectsV2Command, PutObjectCommand } from '@aws-sdk/client-s3'; +import { S3SyncClient } from 's3-sync-client'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import R2Syncer from './R2Syncer.mjs'; + +class FakeSyncClient +{ + calls = []; + + async sync(source, target, options) { + this.calls.push({ options, source, target }); + return { created: [], deleted: [], updated: [] }; + } +} + +class FakeS3Client +{ + uploads = []; + + async send(command) { + if (command instanceof ListObjectsV2Command) { + return { Contents: [], IsTruncated: false }; + } + + if (command instanceof PutObjectCommand) { + this.uploads.push({ + ...command.input, + Body: await streamText(command.input.Body), + }); + return {}; + } + + throw new Error(`Unexpected command: ${command.constructor.name}`); + } +} + +function isIncluded(filters, key) { + let excluded = false; + + for (let filter of filters) { + if (!excluded && filter.exclude) { + excluded = filter.exclude(key); + } + if (excluded && filter.include) { + excluded = !filter.include(key); + } + } + + return !excluded; +} + +async function streamText(stream) { + if (Buffer.isBuffer(stream)) { + return stream.toString(); + } + + let chunks = []; + for await (let chunk of stream) { + chunks.push(Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString(); +} + +describe('R2Syncer', () => { + let temporaryDirectories = []; + let config; + + beforeEach(() => { + temporaryDirectories = []; + config = { + bucket: 'splatoon3-ink-assets', + publicUrl: 'https://assets.splatoon3.ink', + siteUrl: 'https://splatoon3.ink', + }; + }); + + afterEach(() => { + return Promise.all(temporaryDirectories.map(directory => fs.rm(directory, { + force: true, + recursive: true, + }))); + }); + + it('uploads only public data and generated images with canonical R2 keys', async () => { + let syncClient = new FakeSyncClient; + let syncer = new R2Syncer({ config, syncClient }); + + await syncer.upload(); + + expect(syncClient.calls).toHaveLength(1); + let call = syncClient.calls[0]; + expect(call.source).toBe(path.resolve('dist')); + expect(call.target).toBe('s3://splatoon3-ink-assets'); + expect(call.options.partSize).toBe(Number.MAX_SAFE_INTEGER); + expect(isIncluded(call.options.filters, 'data/schedules.json')).toBe(true); + expect(isIncluded(call.options.filters, 'data/festivals.US.ics')).toBe(true); + expect(isIncluded(call.options.filters, 'data/archive/old.json')).toBe(false); + expect(isIncluded(call.options.filters, 'data/index.html')).toBe(false); + expect(isIncluded(call.options.filters, 'assets/splatnet/v3/stage.png')).toBe(true); + expect(isIncluded(call.options.filters, 'assets/splatnet/.DS_Store')).toBe(false); + expect(isIncluded(call.options.filters, 'assets/main.js')).toBe(false); + expect(isIncluded(call.options.filters, 'status-screenshots/schedules.png')).toBe(true); + expect(isIncluded(call.options.filters, 'status-screenshots/index.html')).toBe(false); + expect(call.options.relocations[0]('assets/splatnet/v3/stage.png')) + .toBe('splatnet/v3/stage.png'); + expect(call.options.relocations[0]('data/schedules.json')) + .toBe('data/schedules.json'); + }); + + it('rewrites legacy asset URLs and adjusts the upload content length', async () => { + let localPath = await fs.mkdtemp(path.join(os.tmpdir(), 'r2-syncer-')); + temporaryDirectories.push(localPath); + let jsonPath = path.join(localPath, 'schedules.json'); + await fs.writeFile( + jsonPath, + '{"first":"https://splatoon3.ink/assets/splatnet/one.png",' + + '"second":"https://splatoon3.ink/assets/splatnet/two.png"}', + ); + config.publicUrl = 'https://cdn.example.com'; + let syncClient = new FakeSyncClient; + let syncer = new R2Syncer({ config, syncClient }); + await syncer.upload(); + let commandInput = syncClient.calls[0].options.commandInput; + let input = { + Body: createReadStream(jsonPath), + ContentLength: 137, + Key: 'data/schedules.json', + }; + + let result = commandInput(input); + + expect(result.Body.toString()).toBe( + '{"first":"https://cdn.example.com/splatnet/one.png",' + + '"second":"https://cdn.example.com/splatnet/two.png"}', + ); + expect(result.ContentLength).toBe(result.Body.length); + expect(result).toMatchObject({ + CacheControl: 'no-cache, stale-while-revalidate=5, stale-if-error=86400', + ContentType: 'application/json', + }); + expect(result).not.toHaveProperty('ACL'); + }); + + it('does not rewrite image upload bodies', async () => { + let syncClient = new FakeSyncClient; + let syncer = new R2Syncer({ config, syncClient }); + await syncer.upload(); + let commandInput = syncClient.calls[0].options.commandInput; + let body = { path: '/unused/image.png' }; + + let result = commandInput({ + Body: body, + ContentLength: 5, + Key: 'splatnet/stage.png', + }); + + expect(result.Body).toBeUndefined(); + expect(result.ContentType).toBe('image/png'); + }); + + it('uploads rewritten data and relocated images through the real sync client', async () => { + let localPath = await fs.mkdtemp(path.join(os.tmpdir(), 'r2-syncer-')); + temporaryDirectories.push(localPath); + await fs.mkdir(path.join(localPath, 'data'), { recursive: true }); + await fs.mkdir(path.join(localPath, 'assets/splatnet'), { recursive: true }); + await fs.writeFile( + path.join(localPath, 'data/schedules.json'), + '{"image":"https://splatoon3.ink/assets/splatnet/stage.png"}', + ); + await fs.writeFile( + path.join(localPath, 'data/festivals.US.ics'), + 'URL:https://splatoon3.ink\r\n' + + 'ATTACH:https://splatoon3.ink/assets/splatnet/fest.png\r\n', + ); + await fs.writeFile(path.join(localPath, 'assets/splatnet/stage.png'), 'image'); + await fs.writeFile(path.join(localPath, 'assets/main.js'), 'static site'); + let s3Client = new FakeS3Client; + let syncClient = new S3SyncClient({ client: s3Client }); + let syncer = new R2Syncer({ config, localPath, s3Client, syncClient }); + + await syncer.upload(); + + expect(s3Client.uploads.map(upload => upload.Key).sort()).toEqual([ + 'data/festivals.US.ics', + 'data/schedules.json', + 'splatnet/stage.png', + ]); + let json = s3Client.uploads.find(upload => upload.Key === 'data/schedules.json'); + expect(json.Body).toBe( + '{"image":"https://assets.splatoon3.ink/splatnet/stage.png"}', + ); + expect(json.ContentLength).toBe(Buffer.byteLength(json.Body)); + let calendar = s3Client.uploads.find(upload => upload.Key === 'data/festivals.US.ics'); + expect(calendar.Body).toBe( + 'URL:https://splatoon3.ink\r\n' + + 'ATTACH:https://assets.splatoon3.ink/splatnet/fest.png\r\n', + ); + expect(calendar.ContentLength).toBe(Buffer.byteLength(calendar.Body)); + expect(calendar.ContentType).toBe('text/calendar'); + }); +}); diff --git a/app/sync/index.mjs b/app/sync/index.mjs index ddb7dea..692d052 100644 --- a/app/sync/index.mjs +++ b/app/sync/index.mjs @@ -1,6 +1,7 @@ import S3Syncer from './S3Syncer.mjs'; +import R2Syncer from './R2Syncer.mjs'; -export function canSync() { +export function canSyncS3() { return !!( process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY && @@ -9,22 +10,55 @@ export function canSync() { ); } -async function doSync(download, upload) { - if (!canSync()) { - console.warn('Missing S3 connection parameters'); - return; +function r2Configuration() { + return { + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY, + bucket: process.env.R2_BUCKET, + endpoint: process.env.R2_ENDPOINT, + publicUrl: process.env.R2_PUBLIC_URL, + siteUrl: process.env.SITE_URL, + }; +} + +export function canUploadR2() { + return Object.values(r2Configuration()).every(Boolean); +} + +export function canUpload() { + return canSyncS3() || canUploadR2(); +} + +export async function upload() { + let uploads = []; + + if (canSyncS3()) { + uploads.push((new S3Syncer).upload()); + } + if (canUploadR2()) { + uploads.push((new R2Syncer({ config: r2Configuration() })).upload()); } - const syncer = new S3Syncer(); + if (uploads.length === 0) { + console.warn('Missing object storage connection parameters'); + } + await Promise.all(uploads); +} + +async function doSync(download, shouldUpload) { if (download) { - console.info('Downloading files...'); - await syncer.download(); + if (canSyncS3()) { + console.info('Downloading files...'); + await (new S3Syncer).download(); + } else { + console.warn('Missing S3 connection parameters for download'); + } } - if (upload) { + if (shouldUpload) { console.info('Uploading files...'); - await syncer.upload(); + await upload(); } } diff --git a/app/sync/index.test.mjs b/app/sync/index.test.mjs new file mode 100644 index 0000000..326943a --- /dev/null +++ b/app/sync/index.test.mjs @@ -0,0 +1,110 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + legacyDownload: vi.fn(), + legacyUpload: vi.fn(), + r2Upload: vi.fn(), +})); + +vi.mock('./S3Syncer.mjs', () => ({ + default: class { + download(...args) { + return mocks.legacyDownload(...args); + } + + upload(...args) { + return mocks.legacyUpload(...args); + } + }, +})); + +vi.mock('./R2Syncer.mjs', () => ({ + default: class { + upload(...args) { + return mocks.r2Upload(...args); + } + }, +})); + +const { + canSyncS3, + canUploadR2, + syncUpload, + upload, +} = await import('./index.mjs'); + +const environmentKeys = [ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_S3_BUCKET', + 'AWS_S3_PRIVATE_BUCKET', + 'R2_ACCESS_KEY_ID', + 'R2_SECRET_ACCESS_KEY', + 'R2_BUCKET', + 'R2_ENDPOINT', + 'R2_PUBLIC_URL', + 'SITE_URL', +]; + +function configureLegacy() { + process.env.AWS_ACCESS_KEY_ID = 'legacy-key'; + process.env.AWS_SECRET_ACCESS_KEY = 'legacy-secret'; + process.env.AWS_S3_BUCKET = 'legacy-public'; + process.env.AWS_S3_PRIVATE_BUCKET = 'legacy-private'; +} + +function configureR2() { + process.env.R2_ACCESS_KEY_ID = 'r2-key'; + process.env.R2_SECRET_ACCESS_KEY = 'r2-secret'; + process.env.R2_BUCKET = 'splatoon3-ink-assets'; + process.env.R2_ENDPOINT = 'https://account.r2.cloudflarestorage.com'; + process.env.R2_PUBLIC_URL = 'https://assets.splatoon3.ink'; + process.env.SITE_URL = 'https://splatoon3.ink'; +} + +describe('sync orchestration', () => { + beforeEach(() => { + for (let key of environmentKeys) { + delete process.env[key]; + } + vi.clearAllMocks(); + }); + + afterEach(() => { + for (let key of environmentKeys) { + delete process.env[key]; + } + }); + + it('detects legacy and R2 configuration independently', () => { + expect(canSyncS3()).toBe(false); + expect(canUploadR2()).toBe(false); + + configureR2(); + expect(canSyncS3()).toBe(false); + expect(canUploadR2()).toBe(true); + + configureLegacy(); + expect(canSyncS3()).toBe(true); + expect(canUploadR2()).toBe(true); + }); + + it('uploads to both configured public destinations', async () => { + configureLegacy(); + configureR2(); + + await upload(); + + expect(mocks.legacyUpload).toHaveBeenCalledOnce(); + expect(mocks.r2Upload).toHaveBeenCalledOnce(); + }); + + it('allows upload-only operation when only R2 is configured', async () => { + configureR2(); + + await syncUpload(); + + expect(mocks.legacyUpload).not.toHaveBeenCalled(); + expect(mocks.r2Upload).toHaveBeenCalledOnce(); + }); +});