Add dual Spaces and R2 uploader

.env.example

@@ -34,6 +34,13 @@ AWS_S3_PRIVATE_BUCKET=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 
+# Cloudflare R2 public data and images
+R2_ENDPOINT=
+R2_BUCKET=splatoon3-ink-assets
+R2_PUBLIC_URL=https://assets.splatoon3.ink
+R2_ACCESS_KEY_ID=
+R2_SECRET_ACCESS_KEY=
+
 # Twitter API parameters
 TWITTER_CONSUMER_KEY=
 TWITTER_CONSUMER_SECRET=

app/data/index.mjs

@@ -1,6 +1,6 @@
 import * as Sentry from '@sentry/node';
 import S3Syncer from '../sync/S3Syncer.mjs';
-import { canSync } from '../sync/index.mjs';
+import { canSyncS3, canUpload, upload } from '../sync/index.mjs';
 import GearUpdater from './updaters/GearUpdater.mjs';
 import StageScheduleUpdater from './updaters/StageScheduleUpdater.mjs';
 import CoopUpdater from './updaters/CoopUpdater.mjs';
@@ -44,7 +44,7 @@ export async function update(config = 'default') {
   let settings = configs[config];
 
   // Download private files to get updated tokens if needed
-  if (canSync()) {
+  if (canSyncS3()) {
     await (new S3Syncer).download(false);
   }
 
@@ -58,9 +58,9 @@ export async function update(config = 'default') {
     }
   }));
 
-  if (canSync()) {
+  if (canUpload()) {
     await ImageProcessor.onIdle();
-    await (new S3Syncer).upload();
+    await upload();
   }
 
   console.info(`Done running ${config} updaters`);

app/social/index.mjs

@@ -1,5 +1,4 @@
-import S3Syncer from '../sync/S3Syncer.mjs';
-import { canSync } from '../sync/index.mjs';
+import { canUpload, upload } from '../sync/index.mjs';
 import FileWriter from './clients/FileWriter.mjs';
 import ImageWriter from './clients/ImageWriter.mjs';
 import MastodonClient from './clients/MastodonClient.mjs';
@@ -68,8 +67,8 @@ export function testStatusGeneratorManager(additionalClients) {
 export async function sendStatuses() {
   await defaultStatusGeneratorManager().sendStatuses();
 
-  if (canSync()) {
-    await (new S3Syncer).upload();
+  if (canUpload()) {
+    await upload();
   }
 }
 

app/sync/R2Syncer.mjs

@@ -0,0 +1,121 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { S3Client } from '@aws-sdk/client-s3';
+import { S3SyncClient } from 's3-sync-client';
+import mime from 'mime-types';
+
+// s3-sync-client cannot replace multipart upload bodies. Force direct PUTs so
+// rewritten data can never bypass the URL rewrite; R2 will reject an oversized object
+// instead of silently publishing legacy URLs.
+const forceSinglePartUploads = Number.MAX_SAFE_INTEGER;
+const dataCacheControl = 'no-cache, stale-while-revalidate=5, stale-if-error=86400';
+const rewrittenDataExtensions = ['.ics', '.json'];
+
+function baseUrl(url) {
+  return url.replace(/\/+$/, '');
+}
+
+function isImage(key) {
+  let contentType = mime.lookup(key);
+  return typeof contentType === 'string' && contentType.startsWith('image/');
+}
+
+function isRewrittenData(key) {
+  return rewrittenDataExtensions.some(extension => key.endsWith(extension));
+}
+
+export default class R2Syncer
+{
+  constructor({ config = {}, localPath, s3Client, syncClient } = {}) {
+    this.config = config;
+    this._s3Client = s3Client;
+    this._syncClient = syncClient;
+    this._localPath = localPath;
+  }
+
+  async upload() {
+    this.log('Uploading files...');
+
+    return this.syncClient.sync(this.localPath, this.publicBucket, {
+      filters: this.filters,
+      relocations: this.relocations,
+      partSize: forceSinglePartUploads,
+      commandInput: input => this.commandInput(input),
+    });
+  }
+
+  commandInput(input) {
+    let result = {
+      ContentType: mime.lookup(input.Key) || undefined,
+      CacheControl: input.Key.startsWith('data/')
+        ? dataCacheControl
+        : undefined,
+    };
+
+    if (isRewrittenData(input.Key)) {
+      let source = fs.readFileSync(input.Body.path, 'utf8');
+      result.Body = Buffer.from(source.replaceAll(
+        this.legacyAssetUrl,
+        this.r2AssetUrl,
+      ));
+      result.ContentLength = result.Body.length;
+      input.Body.resume();
+    }
+
+    return result;
+  }
+
+  get s3Client() {
+    return this._s3Client ??= new S3Client({
+      endpoint: this.config.endpoint,
+      region: 'auto',
+      requestChecksumCalculation: 'WHEN_REQUIRED',
+      responseChecksumValidation: 'WHEN_REQUIRED',
+      credentials: {
+        accessKeyId: this.config.accessKeyId,
+        secretAccessKey: this.config.secretAccessKey,
+      },
+    });
+  }
+
+  /** @member {S3SyncClient} */
+  get syncClient() {
+    return this._syncClient ??= new S3SyncClient({ client: this.s3Client });
+  }
+
+  get publicBucket() {
+    return `s3://${this.config.bucket}`;
+  }
+
+  get localPath() {
+    return this._localPath ?? path.resolve('dist');
+  }
+
+  get legacyAssetUrl() {
+    return `${baseUrl(this.config.siteUrl)}/assets/splatnet/`;
+  }
+
+  get r2AssetUrl() {
+    return `${baseUrl(this.config.publicUrl)}/splatnet/`;
+  }
+
+  get filters() {
+    return [
+      { exclude: () => true },
+      { include: key => key.startsWith('assets/splatnet/') && isImage(key) },
+      { include: key => key.startsWith('data/') && isRewrittenData(key) },
+      { exclude: key => key.startsWith('data/archive/') },
+      { include: key => key.startsWith('status-screenshots/') && isImage(key) },
+    ];
+  }
+
+  get relocations() {
+    return [key => key.startsWith('assets/splatnet/')
+      ? key.slice('assets/'.length)
+      : key];
+  }
+
+  log(message) {
+    console.log(`[R2] ${message}`);
+  }
+}

app/sync/R2Syncer.test.mjs

@@ -0,0 +1,205 @@
+import path from 'node:path';
+import os from 'node:os';
+import fs from 'node:fs/promises';
+import { createReadStream } from 'node:fs';
+import { ListObjectsV2Command, PutObjectCommand } from '@aws-sdk/client-s3';
+import { S3SyncClient } from 's3-sync-client';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import R2Syncer from './R2Syncer.mjs';
+
+class FakeSyncClient
+{
+  calls = [];
+
+  async sync(source, target, options) {
+    this.calls.push({ options, source, target });
+    return { created: [], deleted: [], updated: [] };
+  }
+}
+
+class FakeS3Client
+{
+  uploads = [];
+
+  async send(command) {
+    if (command instanceof ListObjectsV2Command) {
+      return { Contents: [], IsTruncated: false };
+    }
+
+    if (command instanceof PutObjectCommand) {
+      this.uploads.push({
+        ...command.input,
+        Body: await streamText(command.input.Body),
+      });
+      return {};
+    }
+
+    throw new Error(`Unexpected command: ${command.constructor.name}`);
+  }
+}
+
+function isIncluded(filters, key) {
+  let excluded = false;
+
+  for (let filter of filters) {
+    if (!excluded && filter.exclude) {
+      excluded = filter.exclude(key);
+    }
+    if (excluded && filter.include) {
+      excluded = !filter.include(key);
+    }
+  }
+
+  return !excluded;
+}
+
+async function streamText(stream) {
+  if (Buffer.isBuffer(stream)) {
+    return stream.toString();
+  }
+
+  let chunks = [];
+  for await (let chunk of stream) {
+    chunks.push(Buffer.from(chunk));
+  }
+  return Buffer.concat(chunks).toString();
+}
+
+describe('R2Syncer', () => {
+  let temporaryDirectories = [];
+  let config;
+
+  beforeEach(() => {
+    temporaryDirectories = [];
+    config = {
+      bucket: 'splatoon3-ink-assets',
+      publicUrl: 'https://assets.splatoon3.ink',
+      siteUrl: 'https://splatoon3.ink',
+    };
+  });
+
+  afterEach(() => {
+    return Promise.all(temporaryDirectories.map(directory => fs.rm(directory, {
+      force: true,
+      recursive: true,
+    })));
+  });
+
+  it('uploads only public data and generated images with canonical R2 keys', async () => {
+    let syncClient = new FakeSyncClient;
+    let syncer = new R2Syncer({ config, syncClient });
+
+    await syncer.upload();
+
+    expect(syncClient.calls).toHaveLength(1);
+    let call = syncClient.calls[0];
+    expect(call.source).toBe(path.resolve('dist'));
+    expect(call.target).toBe('s3://splatoon3-ink-assets');
+    expect(call.options.partSize).toBe(Number.MAX_SAFE_INTEGER);
+    expect(isIncluded(call.options.filters, 'data/schedules.json')).toBe(true);
+    expect(isIncluded(call.options.filters, 'data/festivals.US.ics')).toBe(true);
+    expect(isIncluded(call.options.filters, 'data/archive/old.json')).toBe(false);
+    expect(isIncluded(call.options.filters, 'data/index.html')).toBe(false);
+    expect(isIncluded(call.options.filters, 'assets/splatnet/v3/stage.png')).toBe(true);
+    expect(isIncluded(call.options.filters, 'assets/splatnet/.DS_Store')).toBe(false);
+    expect(isIncluded(call.options.filters, 'assets/main.js')).toBe(false);
+    expect(isIncluded(call.options.filters, 'status-screenshots/schedules.png')).toBe(true);
+    expect(isIncluded(call.options.filters, 'status-screenshots/index.html')).toBe(false);
+    expect(call.options.relocations[0]('assets/splatnet/v3/stage.png'))
+      .toBe('splatnet/v3/stage.png');
+    expect(call.options.relocations[0]('data/schedules.json'))
+      .toBe('data/schedules.json');
+  });
+
+  it('rewrites legacy asset URLs and adjusts the upload content length', async () => {
+    let localPath = await fs.mkdtemp(path.join(os.tmpdir(), 'r2-syncer-'));
+    temporaryDirectories.push(localPath);
+    let jsonPath = path.join(localPath, 'schedules.json');
+    await fs.writeFile(
+      jsonPath,
+      '{"first":"https://splatoon3.ink/assets/splatnet/one.png",'
+      + '"second":"https://splatoon3.ink/assets/splatnet/two.png"}',
+    );
+    config.publicUrl = 'https://cdn.example.com';
+    let syncClient = new FakeSyncClient;
+    let syncer = new R2Syncer({ config, syncClient });
+    await syncer.upload();
+    let commandInput = syncClient.calls[0].options.commandInput;
+    let input = {
+      Body: createReadStream(jsonPath),
+      ContentLength: 137,
+      Key: 'data/schedules.json',
+    };
+
+    let result = commandInput(input);
+
+    expect(result.Body.toString()).toBe(
+      '{"first":"https://cdn.example.com/splatnet/one.png",'
+      + '"second":"https://cdn.example.com/splatnet/two.png"}',
+    );
+    expect(result.ContentLength).toBe(result.Body.length);
+    expect(result).toMatchObject({
+      CacheControl: 'no-cache, stale-while-revalidate=5, stale-if-error=86400',
+      ContentType: 'application/json',
+    });
+    expect(result).not.toHaveProperty('ACL');
+  });
+
+  it('does not rewrite image upload bodies', async () => {
+    let syncClient = new FakeSyncClient;
+    let syncer = new R2Syncer({ config, syncClient });
+    await syncer.upload();
+    let commandInput = syncClient.calls[0].options.commandInput;
+    let body = { path: '/unused/image.png' };
+
+    let result = commandInput({
+      Body: body,
+      ContentLength: 5,
+      Key: 'splatnet/stage.png',
+    });
+
+    expect(result.Body).toBeUndefined();
+    expect(result.ContentType).toBe('image/png');
+  });
+
+  it('uploads rewritten data and relocated images through the real sync client', async () => {
+    let localPath = await fs.mkdtemp(path.join(os.tmpdir(), 'r2-syncer-'));
+    temporaryDirectories.push(localPath);
+    await fs.mkdir(path.join(localPath, 'data'), { recursive: true });
+    await fs.mkdir(path.join(localPath, 'assets/splatnet'), { recursive: true });
+    await fs.writeFile(
+      path.join(localPath, 'data/schedules.json'),
+      '{"image":"https://splatoon3.ink/assets/splatnet/stage.png"}',
+    );
+    await fs.writeFile(
+      path.join(localPath, 'data/festivals.US.ics'),
+      'URL:https://splatoon3.ink\r\n'
+      + 'ATTACH:https://splatoon3.ink/assets/splatnet/fest.png\r\n',
+    );
+    await fs.writeFile(path.join(localPath, 'assets/splatnet/stage.png'), 'image');
+    await fs.writeFile(path.join(localPath, 'assets/main.js'), 'static site');
+    let s3Client = new FakeS3Client;
+    let syncClient = new S3SyncClient({ client: s3Client });
+    let syncer = new R2Syncer({ config, localPath, s3Client, syncClient });
+
+    await syncer.upload();
+
+    expect(s3Client.uploads.map(upload => upload.Key).sort()).toEqual([
+      'data/festivals.US.ics',
+      'data/schedules.json',
+      'splatnet/stage.png',
+    ]);
+    let json = s3Client.uploads.find(upload => upload.Key === 'data/schedules.json');
+    expect(json.Body).toBe(
+      '{"image":"https://assets.splatoon3.ink/splatnet/stage.png"}',
+    );
+    expect(json.ContentLength).toBe(Buffer.byteLength(json.Body));
+    let calendar = s3Client.uploads.find(upload => upload.Key === 'data/festivals.US.ics');
+    expect(calendar.Body).toBe(
+      'URL:https://splatoon3.ink\r\n'
+      + 'ATTACH:https://assets.splatoon3.ink/splatnet/fest.png\r\n',
+    );
+    expect(calendar.ContentLength).toBe(Buffer.byteLength(calendar.Body));
+    expect(calendar.ContentType).toBe('text/calendar');
+  });
+});

app/sync/index.mjs

@@ -1,6 +1,7 @@
 import S3Syncer from './S3Syncer.mjs';
+import R2Syncer from './R2Syncer.mjs';
 
-export function canSync() {
+export function canSyncS3() {
   return !!(
     process.env.AWS_ACCESS_KEY_ID &&
     process.env.AWS_SECRET_ACCESS_KEY &&
@@ -9,22 +10,55 @@ export function canSync() {
   );
 }
 
-async function doSync(download, upload) {
-  if (!canSync()) {
-    console.warn('Missing S3 connection parameters');
-    return;
+function r2Configuration() {
+  return {
+    accessKeyId: process.env.R2_ACCESS_KEY_ID,
+    secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
+    bucket: process.env.R2_BUCKET,
+    endpoint: process.env.R2_ENDPOINT,
+    publicUrl: process.env.R2_PUBLIC_URL,
+    siteUrl: process.env.SITE_URL,
+  };
+}
+
+export function canUploadR2() {
+  return Object.values(r2Configuration()).every(Boolean);
+}
+
+export function canUpload() {
+  return canSyncS3() || canUploadR2();
+}
+
+export async function upload() {
+  let uploads = [];
+
+  if (canSyncS3()) {
+    uploads.push((new S3Syncer).upload());
+  }
+  if (canUploadR2()) {
+    uploads.push((new R2Syncer({ config: r2Configuration() })).upload());
   }
 
-  const syncer = new S3Syncer();
+  if (uploads.length === 0) {
+    console.warn('Missing object storage connection parameters');
+  }
 
+  await Promise.all(uploads);
+}
+
+async function doSync(download, shouldUpload) {
   if (download) {
-    console.info('Downloading files...');
-    await syncer.download();
+    if (canSyncS3()) {
+      console.info('Downloading files...');
+      await (new S3Syncer).download();
+    } else {
+      console.warn('Missing S3 connection parameters for download');
+    }
   }
 
-  if (upload) {
+  if (shouldUpload) {
     console.info('Uploading files...');
-    await syncer.upload();
+    await upload();
   }
 }
 

app/sync/index.test.mjs

@@ -0,0 +1,110 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const mocks = vi.hoisted(() => ({
+  legacyDownload: vi.fn(),
+  legacyUpload: vi.fn(),
+  r2Upload: vi.fn(),
+}));
+
+vi.mock('./S3Syncer.mjs', () => ({
+  default: class {
+    download(...args) {
+      return mocks.legacyDownload(...args);
+    }
+
+    upload(...args) {
+      return mocks.legacyUpload(...args);
+    }
+  },
+}));
+
+vi.mock('./R2Syncer.mjs', () => ({
+  default: class {
+    upload(...args) {
+      return mocks.r2Upload(...args);
+    }
+  },
+}));
+
+const {
+  canSyncS3,
+  canUploadR2,
+  syncUpload,
+  upload,
+} = await import('./index.mjs');
+
+const environmentKeys = [
+  'AWS_ACCESS_KEY_ID',
+  'AWS_SECRET_ACCESS_KEY',
+  'AWS_S3_BUCKET',
+  'AWS_S3_PRIVATE_BUCKET',
+  'R2_ACCESS_KEY_ID',
+  'R2_SECRET_ACCESS_KEY',
+  'R2_BUCKET',
+  'R2_ENDPOINT',
+  'R2_PUBLIC_URL',
+  'SITE_URL',
+];
+
+function configureLegacy() {
+  process.env.AWS_ACCESS_KEY_ID = 'legacy-key';
+  process.env.AWS_SECRET_ACCESS_KEY = 'legacy-secret';
+  process.env.AWS_S3_BUCKET = 'legacy-public';
+  process.env.AWS_S3_PRIVATE_BUCKET = 'legacy-private';
+}
+
+function configureR2() {
+  process.env.R2_ACCESS_KEY_ID = 'r2-key';
+  process.env.R2_SECRET_ACCESS_KEY = 'r2-secret';
+  process.env.R2_BUCKET = 'splatoon3-ink-assets';
+  process.env.R2_ENDPOINT = 'https://account.r2.cloudflarestorage.com';
+  process.env.R2_PUBLIC_URL = 'https://assets.splatoon3.ink';
+  process.env.SITE_URL = 'https://splatoon3.ink';
+}
+
+describe('sync orchestration', () => {
+  beforeEach(() => {
+    for (let key of environmentKeys) {
+      delete process.env[key];
+    }
+    vi.clearAllMocks();
+  });
+
+  afterEach(() => {
+    for (let key of environmentKeys) {
+      delete process.env[key];
+    }
+  });
+
+  it('detects legacy and R2 configuration independently', () => {
+    expect(canSyncS3()).toBe(false);
+    expect(canUploadR2()).toBe(false);
+
+    configureR2();
+    expect(canSyncS3()).toBe(false);
+    expect(canUploadR2()).toBe(true);
+
+    configureLegacy();
+    expect(canSyncS3()).toBe(true);
+    expect(canUploadR2()).toBe(true);
+  });
+
+  it('uploads to both configured public destinations', async () => {
+    configureLegacy();
+    configureR2();
+
+    await upload();
+
+    expect(mocks.legacyUpload).toHaveBeenCalledOnce();
+    expect(mocks.r2Upload).toHaveBeenCalledOnce();
+  });
+
+  it('allows upload-only operation when only R2 is configured', async () => {
+    configureR2();
+
+    await syncUpload();
+
+    expect(mocks.legacyUpload).not.toHaveBeenCalled();
+    expect(mocks.r2Upload).toHaveBeenCalledOnce();
+  });
+});