From 70e81bbc38432ac2655c6db033b281f483fb6ccb Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sun, 30 Aug 2026 10:20:12 -0700 Subject: [PATCH] Generate canonical asset URLs upstream --- .env.example | 2 +- app/data/ImageProcessor.mjs | 5 +++++ app/data/ImageProcessor.test.mjs | 8 +++++++ app/sync/R2Syncer.mjs | 36 ++++---------------------------- app/sync/R2Syncer.test.mjs | 28 +++++++++---------------- app/sync/index.mjs | 2 -- app/sync/index.test.mjs | 2 -- 7 files changed, 28 insertions(+), 55 deletions(-) diff --git a/.env.example b/.env.example index 2b03b27..31124d6 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ # Site Config SITE_URL=https://splatoon3.ink +ASSET_URL=https://assets.splatoon3.ink # Load data from Splatoon3.ink during development VITE_DATA_FROM=https://splatoon3.ink @@ -44,7 +45,6 @@ AWS_SECRET_ACCESS_KEY= # Cloudflare R2 public data and images R2_ENDPOINT= R2_BUCKET=splatoon3-ink-assets -R2_PUBLIC_URL=https://assets.splatoon3.ink R2_ACCESS_KEY_ID= R2_SECRET_ACCESS_KEY= diff --git a/app/data/ImageProcessor.mjs b/app/data/ImageProcessor.mjs index 254b36f..8414a81 100644 --- a/app/data/ImageProcessor.mjs +++ b/app/data/ImageProcessor.mjs @@ -15,6 +15,7 @@ export default class ImageProcessor constructor() { this.console = prefixedConsole('Images'); this.siteUrl = process.env.SITE_URL; + this.assetUrl = process.env.ASSET_URL?.replace(/\/+$/, ''); } async process(url, defer = true) { @@ -47,6 +48,10 @@ export default class ImageProcessor } publicUrl(file) { + if (this.assetUrl) { + return `${this.assetUrl}/splatnet/${file}`; + } + return `${this.siteUrl ?? ''}/${this.outputDirectory}/${file}`; } diff --git a/app/data/ImageProcessor.test.mjs b/app/data/ImageProcessor.test.mjs index e08c4ad..e929c1e 100644 --- a/app/data/ImageProcessor.test.mjs +++ b/app/data/ImageProcessor.test.mjs @@ -13,6 +13,7 @@ describe('ImageProcessor', () => { beforeEach(() => { delete process.env.SITE_URL; + delete process.env.ASSET_URL; processor = new ImageProcessor(); }); @@ -39,5 +40,12 @@ describe('ImageProcessor', () => { const proc = new ImageProcessor(); expect(proc.publicUrl('v2/weapon/image.png')).toBe('https://splatoon3.ink/assets/splatnet/v2/weapon/image.png'); }); + + it('uses the canonical asset host when ASSET_URL is set', () => { + process.env.SITE_URL = 'https://splatoon3.ink'; + process.env.ASSET_URL = 'https://assets.splatoon3.ink/'; + const proc = new ImageProcessor(); + expect(proc.publicUrl('v2/weapon/image.png')).toBe('https://assets.splatoon3.ink/splatnet/v2/weapon/image.png'); + }); }); }); diff --git a/app/sync/R2Syncer.mjs b/app/sync/R2Syncer.mjs index e590f9c..4317e05 100644 --- a/app/sync/R2Syncer.mjs +++ b/app/sync/R2Syncer.mjs @@ -1,27 +1,18 @@ import path from 'node:path'; -import fs from 'node:fs'; import { S3Client } from '@aws-sdk/client-s3'; import { S3SyncClient } from 's3-sync-client'; import mime from 'mime-types'; -// s3-sync-client cannot replace multipart upload bodies. Force direct PUTs so -// rewritten data can never bypass the URL rewrite; R2 will reject an oversized object -// instead of silently publishing legacy URLs. -const forceSinglePartUploads = Number.MAX_SAFE_INTEGER; const dataCacheControl = 'no-cache, stale-while-revalidate=5, stale-if-error=86400'; -const rewrittenDataExtensions = ['.ics', '.json']; - -function baseUrl(url) { - return url.replace(/\/+$/, ''); -} +const publicDataExtensions = ['.ics', '.json']; function isImage(key) { let contentType = mime.lookup(key); return typeof contentType === 'string' && contentType.startsWith('image/'); } -function isRewrittenData(key) { - return rewrittenDataExtensions.some(extension => key.endsWith(extension)); +function isPublicData(key) { + return publicDataExtensions.some(extension => key.endsWith(extension)); } export default class R2Syncer @@ -39,7 +30,6 @@ export default class R2Syncer return this.syncClient.sync(this.localPath, this.publicBucket, { filters: this.filters, relocations: this.relocations, - partSize: forceSinglePartUploads, commandInput: input => this.commandInput(input), }); } @@ -52,16 +42,6 @@ export default class R2Syncer : undefined, }; - if (isRewrittenData(input.Key)) { - let source = fs.readFileSync(input.Body.path, 'utf8'); - result.Body = Buffer.from(source.replaceAll( - this.legacyAssetUrl, - this.r2AssetUrl, - )); - result.ContentLength = result.Body.length; - input.Body.resume(); - } - return result; } @@ -91,19 +71,11 @@ export default class R2Syncer return this._localPath ?? path.resolve('dist'); } - get legacyAssetUrl() { - return `${baseUrl(this.config.siteUrl)}/assets/splatnet/`; - } - - get r2AssetUrl() { - return `${baseUrl(this.config.publicUrl)}/splatnet/`; - } - get filters() { return [ { exclude: () => true }, { include: key => key.startsWith('assets/splatnet/') && isImage(key) }, - { include: key => key.startsWith('data/') && isRewrittenData(key) }, + { include: key => key.startsWith('data/') && isPublicData(key) }, { exclude: key => key.startsWith('data/archive/') }, { include: key => key.startsWith('status-screenshots/') && isImage(key) }, ]; diff --git a/app/sync/R2Syncer.test.mjs b/app/sync/R2Syncer.test.mjs index b52272d..b7b6635 100644 --- a/app/sync/R2Syncer.test.mjs +++ b/app/sync/R2Syncer.test.mjs @@ -1,7 +1,6 @@ import path from 'node:path'; import os from 'node:os'; import fs from 'node:fs/promises'; -import { createReadStream } from 'node:fs'; import { ListObjectsV2Command, PutObjectCommand } from '@aws-sdk/client-s3'; import { S3SyncClient } from 's3-sync-client'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -73,8 +72,6 @@ describe('R2Syncer', () => { temporaryDirectories = []; config = { bucket: 'splatoon3-ink-assets', - publicUrl: 'https://assets.splatoon3.ink', - siteUrl: 'https://splatoon3.ink', }; }); @@ -95,7 +92,7 @@ describe('R2Syncer', () => { let call = syncClient.calls[0]; expect(call.source).toBe(path.resolve('dist')); expect(call.target).toBe('s3://splatoon3-ink-assets'); - expect(call.options.partSize).toBe(Number.MAX_SAFE_INTEGER); + expect(call.options).not.toHaveProperty('partSize'); expect(isIncluded(call.options.filters, 'data/schedules.json')).toBe(true); expect(isIncluded(call.options.filters, 'data/festivals.US.ics')).toBe(true); expect(isIncluded(call.options.filters, 'data/archive/old.json')).toBe(false); @@ -111,33 +108,28 @@ describe('R2Syncer', () => { .toBe('data/schedules.json'); }); - it('rewrites legacy asset URLs and adjusts the upload content length', async () => { + it('does not transform public data upload bodies', async () => { let localPath = await fs.mkdtemp(path.join(os.tmpdir(), 'r2-syncer-')); temporaryDirectories.push(localPath); let jsonPath = path.join(localPath, 'schedules.json'); await fs.writeFile( jsonPath, - '{"first":"https://splatoon3.ink/assets/splatnet/one.png",' - + '"second":"https://splatoon3.ink/assets/splatnet/two.png"}', + '{"image":"https://assets.splatoon3.ink/splatnet/one.png"}', ); - config.publicUrl = 'https://cdn.example.com'; let syncClient = new FakeSyncClient; let syncer = new R2Syncer({ config, syncClient }); await syncer.upload(); let commandInput = syncClient.calls[0].options.commandInput; let input = { - Body: createReadStream(jsonPath), - ContentLength: 137, + Body: { path: jsonPath }, + ContentLength: 61, Key: 'data/schedules.json', }; let result = commandInput(input); - expect(result.Body.toString()).toBe( - '{"first":"https://cdn.example.com/splatnet/one.png",' - + '"second":"https://cdn.example.com/splatnet/two.png"}', - ); - expect(result.ContentLength).toBe(result.Body.length); + expect(result.Body).toBeUndefined(); + expect(result.ContentLength).toBeUndefined(); expect(result).toMatchObject({ CacheControl: 'no-cache, stale-while-revalidate=5, stale-if-error=86400', ContentType: 'application/json', @@ -162,19 +154,19 @@ describe('R2Syncer', () => { expect(result.ContentType).toBe('image/png'); }); - it('uploads rewritten data and relocated images through the real sync client', async () => { + it('uploads canonical data and relocated images through the real sync client', async () => { let localPath = await fs.mkdtemp(path.join(os.tmpdir(), 'r2-syncer-')); temporaryDirectories.push(localPath); await fs.mkdir(path.join(localPath, 'data'), { recursive: true }); await fs.mkdir(path.join(localPath, 'assets/splatnet'), { recursive: true }); await fs.writeFile( path.join(localPath, 'data/schedules.json'), - '{"image":"https://splatoon3.ink/assets/splatnet/stage.png"}', + '{"image":"https://assets.splatoon3.ink/splatnet/stage.png"}', ); await fs.writeFile( path.join(localPath, 'data/festivals.US.ics'), 'URL:https://splatoon3.ink\r\n' - + 'ATTACH:https://splatoon3.ink/assets/splatnet/fest.png\r\n', + + 'ATTACH:https://assets.splatoon3.ink/splatnet/fest.png\r\n', ); await fs.writeFile(path.join(localPath, 'assets/splatnet/stage.png'), 'image'); await fs.writeFile(path.join(localPath, 'assets/main.js'), 'static site'); diff --git a/app/sync/index.mjs b/app/sync/index.mjs index 692d052..f3c6b4c 100644 --- a/app/sync/index.mjs +++ b/app/sync/index.mjs @@ -16,8 +16,6 @@ function r2Configuration() { secretAccessKey: process.env.R2_SECRET_ACCESS_KEY, bucket: process.env.R2_BUCKET, endpoint: process.env.R2_ENDPOINT, - publicUrl: process.env.R2_PUBLIC_URL, - siteUrl: process.env.SITE_URL, }; } diff --git a/app/sync/index.test.mjs b/app/sync/index.test.mjs index 326943a..18ad37c 100644 --- a/app/sync/index.test.mjs +++ b/app/sync/index.test.mjs @@ -42,7 +42,6 @@ const environmentKeys = [ 'R2_SECRET_ACCESS_KEY', 'R2_BUCKET', 'R2_ENDPOINT', - 'R2_PUBLIC_URL', 'SITE_URL', ]; @@ -58,7 +57,6 @@ function configureR2() { process.env.R2_SECRET_ACCESS_KEY = 'r2-secret'; process.env.R2_BUCKET = 'splatoon3-ink-assets'; process.env.R2_ENDPOINT = 'https://account.r2.cloudflarestorage.com'; - process.env.R2_PUBLIC_URL = 'https://assets.splatoon3.ink'; process.env.SITE_URL = 'https://splatoon3.ink'; }