diff --git a/package-lock.json b/package-lock.json index 3a0a036..1ce7d3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "console-stamp": "^3.0.6", "cron": "^4.4.0", "dotenv": "^17.3.1", + "fast-xml-parser": "^5.11.1", "ical-generator": "^11.1.1", "jsonpath-plus": "^10.4.0", "lodash": "^4.17.21", @@ -3857,6 +3858,18 @@ "node": ">= 10" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -5692,6 +5705,18 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/are-docs-informative": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/are-docs-informative/-/are-docs-informative-0.0.2.tgz", @@ -7190,6 +7215,45 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.1.tgz", + "integrity": "sha512-TBw6K/fxoQGGjCmZDw9w/ZwP3uDcnTM4YH/g+PFRWr8sbe5idXtxNN6vITh4+1ruCZaho6uBFurElsA7F0zzgw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz", @@ -7823,6 +7887,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -9100,6 +9176,21 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -10098,6 +10189,21 @@ "node": ">=8" } }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -11889,6 +11995,21 @@ "node": ">=18" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 7b59f55..b67f5a8 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "console-stamp": "^3.0.6", "cron": "^4.4.0", "dotenv": "^17.3.1", + "fast-xml-parser": "^5.11.1", "ical-generator": "^11.1.1", "jsonpath-plus": "^10.4.0", "lodash": "^4.17.21", diff --git a/public/assets/index.html b/public/assets/index.html deleted file mode 100644 index 4eb4efc..0000000 --- a/public/assets/index.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - Splatoon3.ink Data - - - - -
- - - - - - diff --git a/public/data/archive/index.html b/public/data/archive/index.html deleted file mode 100644 index e2cc3a3..0000000 --- a/public/data/archive/index.html +++ /dev/null @@ -1,17 +0,0 @@ - - - - Splatoon3.ink Data Archives - - - -
- - - - - diff --git a/public/data/index.html b/public/data/index.html deleted file mode 100644 index ed4d15c..0000000 --- a/public/data/index.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - Splatoon3.ink Data - - - - -
- - - - - - - diff --git a/workers/assets-directory-browser/README.md b/workers/assets-directory-browser/README.md index 80f3fd0..7da7622 100644 --- a/workers/assets-directory-browser/README.md +++ b/workers/assets-directory-browser/README.md @@ -1,7 +1,20 @@ # Assets directory browser This Worker renders read-only directory listings for the public R2 bucket without -running on ordinary object requests. +running on ordinary object requests. It also mounts the public DigitalOcean +Spaces data archive at `/data/archive/`. + +## Virtual archive directory + +The `/data/` R2 listing includes a synthetic `archive/` directory. Listings +under `/data/archive/` use anonymous S3 `ListObjectsV2` requests against the +public archive origin instead of R2. The mount prefix is removed from S3 +requests and restored on browser-visible directory links. + +Archive file links point directly to `https://data-archive.splatoon3.ink`, so +downloads do not pass through the Worker. Directory and pagination links remain +under `https://assets.splatoon3.ink/data/archive/`. The public S3 listing API on +the archive domain is unchanged. ## Request routing @@ -57,8 +70,17 @@ npm run assets-browser:dev ``` Then open `http://localhost:8787/`. Local mode accepts the natural directory -paths and serves linked objects from Wrangler's simulated R2 bucket. It never -connects to the production bucket. +paths and serves linked R2 objects from Wrangler's simulated bucket. It never +connects to the production R2 bucket. + +The archive mount uses the anonymously readable production Spaces listing in +local mode, so it can be previewed at: + +```text +http://localhost:8787/data/archive/ +``` + +Archive file links in that preview still point to the public archive domain. ## Validate without deploying diff --git a/workers/assets-directory-browser/src/index.js b/workers/assets-directory-browser/src/index.js index a35f282..556f622 100644 --- a/workers/assets-directory-browser/src/index.js +++ b/workers/assets-directory-browser/src/index.js @@ -1,11 +1,95 @@ import Mustache from 'mustache'; +import { XMLParser } from 'fast-xml-parser'; import template from './template.html'; const internalPrefix = '/__directory'; const pageSize = 1000; +const archiveMountPrefix = 'data/archive/'; Mustache.parse(template); +const xmlParser = new XMLParser({ + parseTagValue: false, + trimValues: true, +}); + +/** + * @param {unknown} value + * @returns {unknown[]} The value as an array. + */ +function asArray(value) { + if (value === undefined) { + return []; + } + + return Array.isArray(value) ? value : [value]; +} + +/** + * @param {Env} env + * @param {string} prefix + * @param {string | undefined} cursor + * @returns {Promise} The normalized archive directory listing. + */ +async function listArchive(env, prefix, cursor) { + let archivePrefix = prefix.slice(archiveMountPrefix.length); + let archiveOrigin = new URL(env.ARCHIVE_ORIGIN); + let listUrl = new URL('/', archiveOrigin); + listUrl.searchParams.set('list-type', '2'); + listUrl.searchParams.set('delimiter', '/'); + listUrl.searchParams.set('max-keys', String(pageSize)); + listUrl.searchParams.set('prefix', archivePrefix); + if (cursor) { + listUrl.searchParams.set('continuation-token', cursor); + } + + let response = await fetch(new Request(listUrl, { + headers: { Accept: 'application/xml' }, + })); + if (!response.ok) { + throw new Error(`Archive listing returned ${response.status}`); + } + + let document = xmlParser.parse(await response.text()); + let result = document?.ListBucketResult; + if (!result || typeof result !== 'object') { + throw new Error('Archive listing returned invalid XML'); + } + + return { + cursor: result.NextContinuationToken || undefined, + delimitedPrefixes: asArray(result.CommonPrefixes).map(item => ( + `${archiveMountPrefix}${item.Prefix}` + )), + objects: asArray(result.Contents).map(item => ({ + browserKey: `${archiveMountPrefix}${item.Key}`, + etag: item.ETag, + key: item.Key, + size: Number(item.Size), + uploaded: new Date(item.LastModified), + url: new URL(publicPath(item.Key), archiveOrigin).href, + })), + truncated: result.IsTruncated === 'true', + }; +} + +/** + * @param {string} prefix + * @param {string | undefined} cursor + * @param {R2Objects} listing + * @returns {R2Objects} The listing with any virtual directories added. + */ +function withVirtualDirectories(prefix, cursor, listing) { + if (prefix !== 'data/' || cursor || listing.delimitedPrefixes.includes(archiveMountPrefix)) { + return listing; + } + + return { + ...listing, + delimitedPrefixes: [...listing.delimitedPrefixes, archiveMountPrefix].sort(), + }; +} + /** @param {string} key */ function publicPath(key) { return `/${key.split('/').map(encodeURIComponent).join('/')}`; @@ -100,15 +184,19 @@ function directoryPage(origin, prefix, listing, cursor) { prefix: directoryPrefix, url: new URL(publicPath(directoryPrefix), origin).href, })), - files: listing.objects.map(object => ({ - etag: object.etag, - key: object.key, - name: displayName(object.key, prefix), - pathname: publicPath(object.key), - size: object.size, - uploaded: object.uploaded, - url: new URL(publicPath(object.key), origin).href, - })), + files: listing.objects.map(object => { + let pathname = publicPath(object.browserKey ?? object.key); + return { + etag: object.etag, + href: object.url ?? pathname, + key: object.key, + name: displayName(object.browserKey ?? object.key, prefix), + pathname, + size: object.size, + uploaded: object.uploaded, + url: object.url ?? new URL(publicPath(object.key), origin).href, + }; + }), nextCursor: listing.truncated ? listing.cursor : undefined, pathname, prefix, @@ -157,7 +245,7 @@ function renderListing(page, styleNonce) { ...page.files.map(file => ({ date: formatDate(file.uploaded), datetime: file.uploaded.toISOString(), - href: file.pathname, + href: file.href, isFile: true, name: file.name, size: formatSize(file.size), @@ -319,19 +407,24 @@ export default { } let cursor = url.searchParams.get('cursor') || undefined; + let archiveListing = prefix.startsWith(archiveMountPrefix); /** @type {R2Objects} */ let listing; try { - listing = await env.ASSETS.list({ - cursor, - delimiter: '/', - limit: pageSize, - prefix, - }); + listing = archiveListing + ? await listArchive(env, prefix, cursor) + : await env.ASSETS.list({ + cursor, + delimiter: '/', + limit: pageSize, + prefix, + }); } catch (error) { console.error(JSON.stringify({ error: error instanceof Error ? error.message : String(error), - message: 'R2 directory listing failed', + message: archiveListing + ? 'Archive directory listing failed' + : 'R2 directory listing failed', prefix, })); return listingErrorResponse(request, 500, 'listing_unavailable', 'Directory listing unavailable'); @@ -340,6 +433,7 @@ export default { return listingErrorResponse(request, 404, 'directory_not_found', 'Directory not found'); } + listing = withVirtualDirectories(prefix, cursor, listing); let page = directoryPage(url.origin, prefix, listing, cursor); if (url.searchParams.get('format') === 'json') { return jsonListingResponse(request, page); diff --git a/workers/assets-directory-browser/src/index.test.mjs b/workers/assets-directory-browser/src/index.test.mjs index 5e373a4..d448da4 100644 --- a/workers/assets-directory-browser/src/index.test.mjs +++ b/workers/assets-directory-browser/src/index.test.mjs @@ -90,6 +90,64 @@ describe('assets directory browser', () => { expect(html).not.toContain('__directory'); }); + it('shows the data archive as a directory in the R2 data listing', async () => { + let bucket = new FakeBucket({ + cursor: undefined, + delimitedPrefixes: ['data/locale/'], + objects: [], + truncated: false, + }); + + let response = await worker.fetch(new Request( + 'https://assets.splatoon3.ink/__directory/data/', + ), { ASSETS: bucket }); + let html = await response.text(); + + expect(response.status).toBe(200); + expect(html).toContain('href="/data/archive/"'); + expect(html).toContain('archive/'); + }); + + it('browses archive directories while linking files to the public archive origin', async () => { + let fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(` + + + 2026/08/ + true + + 2026/08/2026-08-01.tar.zst + 2026-08-31T01:02:03.000Z + "archive-etag" + 1536 + + 2026/08/31/ + next archive page + + `, { headers: { 'Content-Type': 'application/xml' } })); + + let response = await worker.fetch(new Request( + 'http://localhost:8787/data/archive/2026/08/?cursor=current%20archive%20page', + ), { + ARCHIVE_ORIGIN: 'https://data-archive.splatoon3.ink', + ASSETS: new FakeBucket, + LOCAL_DEVELOPMENT: 'true', + }); + let html = await response.text(); + + expect(response.status).toBe(200); + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(fetchSpy.mock.calls[0][0].url).toBe( + 'https://data-archive.splatoon3.ink/?list-type=2&delimiter=%2F&max-keys=1000&prefix=2026%2F08%2F&continuation-token=current+archive+page', + ); + expect(html).toContain('href="/data/archive/2026/08/31/"'); + expect(html).toContain('href="https://data-archive.splatoon3.ink/2026/08/2026-08-01.tar.zst"'); + expect(html).toContain('href="/data/archive/2026/08/?cursor=next%20archive%20page"'); + expect(html).toContain('1.5 KB'); + expect(html).toContain('2026-08-31 01:02:03 UTC'); + + fetchSpy.mockRestore(); + }); + it('returns a public JSON representation of a directory page', async () => { let bucket = new FakeBucket({ cursor: 'next page', @@ -283,4 +341,30 @@ describe('assets directory browser', () => { })); errorLog.mockRestore(); }); + + it('returns a controlled error when the archive listing fails', async () => { + let errorLog = vi.spyOn(console, 'error').mockImplementation(() => {}); + let fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response('Unavailable', { status: 503 }), + ); + + let response = await worker.fetch(new Request( + 'https://assets.splatoon3.ink/__directory/data/archive/2026/', + ), { + ARCHIVE_ORIGIN: 'https://data-archive.splatoon3.ink', + ASSETS: new FakeBucket, + }); + + expect(response.status).toBe(500); + expect(response.headers.get('cache-control')).toBe('no-store'); + expect(await response.text()).toBe('Directory listing unavailable'); + expect(errorLog).toHaveBeenCalledWith(JSON.stringify({ + error: 'Archive listing returned 503', + message: 'Archive directory listing failed', + prefix: 'data/archive/2026/', + })); + + fetchSpy.mockRestore(); + errorLog.mockRestore(); + }); }); diff --git a/workers/assets-directory-browser/wrangler.jsonc b/workers/assets-directory-browser/wrangler.jsonc index 484d4b1..58a4bb0 100644 --- a/workers/assets-directory-browser/wrangler.jsonc +++ b/workers/assets-directory-browser/wrangler.jsonc @@ -6,6 +6,9 @@ "compatibility_flags": ["nodejs_compat"], "workers_dev": false, "preview_urls": false, + "vars": { + "ARCHIVE_ORIGIN": "https://data-archive.splatoon3.ink" + }, "routes": [ { "pattern": "assets.splatoon3.ink/__directory/*",