Merge pull request #121 from misenhower/codex/archive-virtual-directory

Add virtual data archive browser
2026-09-13 11:05:28 -05:00 · 2026-08-31 21:19:05 -07:00
parent 93ce803fc0 f72e7cd7c9
commit 6ae419a0be
9 changed files with 345 additions and 101 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -16,6 +16,7 @@
        "console-stamp": "^3.0.6",
        "cron": "^4.4.0",
        "dotenv": "^17.3.1",
+        "fast-xml-parser": "^5.11.1",
        "ical-generator": "^11.1.1",
        "jsonpath-plus": "^10.4.0",
        "lodash": "^4.17.21",
@@ -3857,6 +3858,18 @@
        "node": ">= 10"
      }
    },
+    "node_modules/@nodable/entities": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz",
+      "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/nodable"
+        }
+      ],
+      "license": "MIT"
+    },
    "node_modules/@nodelib/fs.scandir": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -5692,6 +5705,18 @@
        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
      }
    },
+    "node_modules/anynum": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz",
+      "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/NaturalIntelligence"
+        }
+      ],
+      "license": "MIT"
+    },
    "node_modules/are-docs-informative": {
      "version": "0.0.2",
      "resolved": "https://registry.npmjs.org/are-docs-informative/-/are-docs-informative-0.0.2.tgz",
@@ -7190,6 +7215,45 @@
      "dev": true,
      "license": "MIT"
    },
+    "node_modules/fast-xml-builder": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz",
+      "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/NaturalIntelligence"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "path-expression-matcher": "^1.6.2",
+        "xml-naming": "^0.3.0"
+      }
+    },
+    "node_modules/fast-xml-parser": {
+      "version": "5.11.1",
+      "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.1.tgz",
+      "integrity": "sha512-TBw6K/fxoQGGjCmZDw9w/ZwP3uDcnTM4YH/g+PFRWr8sbe5idXtxNN6vITh4+1ruCZaho6uBFurElsA7F0zzgw==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/NaturalIntelligence"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "@nodable/entities": "^3.0.0",
+        "fast-xml-builder": "^1.2.0",
+        "is-unsafe": "^2.0.0",
+        "path-expression-matcher": "^1.6.2",
+        "strnum": "^2.4.2",
+        "xml-naming": "^0.3.0"
+      },
+      "bin": {
+        "fxparser": "src/cli/cli.js"
+      }
+    },
    "node_modules/fastq": {
      "version": "1.20.3",
      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
@@ -7823,6 +7887,18 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/is-unsafe": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz",
+      "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/NaturalIntelligence"
+        }
+      ],
+      "license": "MIT"
+    },
    "node_modules/is-wsl": {
      "version": "2.2.0",
      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
@@ -9100,6 +9176,21 @@
        "node": ">=8"
      }
    },
+    "node_modules/path-expression-matcher": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz",
+      "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/NaturalIntelligence"
+        }
+      ],
+      "license": "MIT",
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
    "node_modules/path-key": {
      "version": "3.1.1",
      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
@@ -10098,6 +10189,21 @@
        "node": ">=8"
      }
    },
+    "node_modules/strnum": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz",
+      "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/NaturalIntelligence"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "anynum": "^1.0.1"
+      }
+    },
    "node_modules/supports-color": {
      "version": "7.2.0",
      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -11889,6 +11995,21 @@
        "node": ">=18"
      }
    },
+    "node_modules/xml-naming": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz",
+      "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/NaturalIntelligence"
+        }
+      ],
+      "license": "MIT",
+      "engines": {
+        "node": ">=16.0.0"
+      }
+    },
    "node_modules/y18n": {
      "version": "5.0.8",
      "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
--- a/package.json
+++ b/package.json
@@ -47,6 +47,7 @@
    "console-stamp": "^3.0.6",
    "cron": "^4.4.0",
    "dotenv": "^17.3.1",
+    "fast-xml-parser": "^5.11.1",
    "ical-generator": "^11.1.1",
    "jsonpath-plus": "^10.4.0",
    "lodash": "^4.17.21",
--- a/public/assets/index.html
+++ b/public/assets/index.html
@@ -1,22 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <title>Splatoon3.ink Data</title>
-</head>
-
-<body>
-  <div id="navigation"></div>
-  <div id="listing"></div>
-  <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
-  <script type="text/javascript">
-    var S3BL_IGNORE_PATH = true;
-    var S3B_SORT = 'A2Z';
-    var BUCKET_URL = 'https://s3-data.splatoon3.ink';
-    var S3B_ROOT_DIR = 'assets';
-    var BUCKET_WEBSITE_URL = 'https://splatoon3.ink';
-  </script>
-  <script type="text/javascript" src="https://rufuspollock.github.io/s3-bucket-listing/list.js"></script>
-</body>
-
-</html>
--- a/public/data/archive/index.html
+++ b/public/data/archive/index.html
@@ -1,17 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-  <title>Splatoon3.ink Data Archives</title>
-</head>
-<body>
-<div id="navigation"></div>
-<div id="listing"></div>
-<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
-<script type="text/javascript">
-  var S3BL_IGNORE_PATH = true;
-  var S3B_SORT = 'A2Z';
-  var BUCKET_URL = 'https://data-archive.splatoon3.ink';
-</script>
-<script type="text/javascript" src="https://rufuspollock.github.io/s3-bucket-listing/list.js"></script>
-</body>
-</html>
--- a/public/data/index.html
+++ b/public/data/index.html
@@ -1,42 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <title>Splatoon3.ink Data</title>
-</head>
-
-<body>
-  <div id="navigation"></div>
-  <div id="listing"></div>
-  <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
-  <script type="text/javascript">
-    var S3BL_IGNORE_PATH = true;
-    var S3B_SORT = 'A2Z';
-    var BUCKET_URL = 'https://s3-data.splatoon3.ink';
-    var S3B_ROOT_DIR = 'data';
-    var BUCKET_WEBSITE_URL = 'https://splatoon3.ink';
-  </script>
-  <script type="text/javascript" src="https://rufuspollock.github.io/s3-bucket-listing/list.js"></script>
-  <script type="text/javascript">
-    var listing = document.querySelector('#listing');
-
-    function rewriteArchiveLink() {
-      var archiveLink = Array.from(listing.querySelectorAll('a'))
-        .find(link => link.textContent.trim() === 'archive/');
-
-      if (archiveLink) {
-        archiveLink.href = '/data/archive/index.html';
-        return true;
-      }
-
-      return false;
-    }
-
-    rewriteArchiveLink();
-
-    var archiveLinkObserver = new MutationObserver(rewriteArchiveLink);
-    archiveLinkObserver.observe(listing, { childList: true, subtree: true });
-  </script>
-</body>
-
-</html>
--- a/workers/assets-directory-browser/README.md
+++ b/workers/assets-directory-browser/README.md
@@ -1,7 +1,20 @@
 # Assets directory browser

 This Worker renders read-only directory listings for the public R2 bucket without
-running on ordinary object requests.
+running on ordinary object requests. It also mounts the public DigitalOcean
+Spaces data archive at `/data/archive/`.
+
+## Virtual archive directory
+
+The `/data/` R2 listing includes a synthetic `archive/` directory. Listings
+under `/data/archive/` use anonymous S3 `ListObjectsV2` requests against the
+public archive origin instead of R2. The mount prefix is removed from S3
+requests and restored on browser-visible directory links.
+
+Archive file links point directly to `https://data-archive.splatoon3.ink`, so
+downloads do not pass through the Worker. Directory and pagination links remain
+under `https://assets.splatoon3.ink/data/archive/`. The public S3 listing API on
+the archive domain is unchanged.

 ## Request routing

@@ -57,8 +70,17 @@ npm run assets-browser:dev
 ```

 Then open `http://localhost:8787/`. Local mode accepts the natural directory
-paths and serves linked objects from Wrangler's simulated R2 bucket. It never
-connects to the production bucket.
+paths and serves linked R2 objects from Wrangler's simulated bucket. It never
+connects to the production R2 bucket.
+
+The archive mount uses the anonymously readable production Spaces listing in
+local mode, so it can be previewed at:
+
+```text
+http://localhost:8787/data/archive/
+```
+
+Archive file links in that preview still point to the public archive domain.

 ## Validate without deploying

--- a/workers/assets-directory-browser/src/index.js
+++ b/workers/assets-directory-browser/src/index.js
@@ -1,11 +1,95 @@
 import Mustache from 'mustache';
+import { XMLParser } from 'fast-xml-parser';
 import template from './template.html';

 const internalPrefix = '/__directory';
 const pageSize = 1000;
+const archiveMountPrefix = 'data/archive/';

 Mustache.parse(template);

+const xmlParser = new XMLParser({
+  parseTagValue: false,
+  trimValues: true,
+});
+
+/**
+ * @param {unknown} value
+ * @returns {unknown[]} The value as an array.
+ */
+function asArray(value) {
+  if (value === undefined) {
+    return [];
+  }
+
+  return Array.isArray(value) ? value : [value];
+}
+
+/**
+ * @param {Env} env
+ * @param {string} prefix
+ * @param {string | undefined} cursor
+ * @returns {Promise<R2Objects>} The normalized archive directory listing.
+ */
+async function listArchive(env, prefix, cursor) {
+  let archivePrefix = prefix.slice(archiveMountPrefix.length);
+  let archiveOrigin = new URL(env.ARCHIVE_ORIGIN);
+  let listUrl = new URL('/', archiveOrigin);
+  listUrl.searchParams.set('list-type', '2');
+  listUrl.searchParams.set('delimiter', '/');
+  listUrl.searchParams.set('max-keys', String(pageSize));
+  listUrl.searchParams.set('prefix', archivePrefix);
+  if (cursor) {
+    listUrl.searchParams.set('continuation-token', cursor);
+  }
+
+  let response = await fetch(new Request(listUrl, {
+    headers: { Accept: 'application/xml' },
+  }));
+  if (!response.ok) {
+    throw new Error(`Archive listing returned ${response.status}`);
+  }
+
+  let document = xmlParser.parse(await response.text());
+  let result = document?.ListBucketResult;
+  if (!result || typeof result !== 'object') {
+    throw new Error('Archive listing returned invalid XML');
+  }
+
+  return {
+    cursor: result.NextContinuationToken || undefined,
+    delimitedPrefixes: asArray(result.CommonPrefixes).map(item => (
+      `${archiveMountPrefix}${item.Prefix}`
+    )),
+    objects: asArray(result.Contents).map(item => ({
+      browserKey: `${archiveMountPrefix}${item.Key}`,
+      etag: item.ETag,
+      key: item.Key,
+      size: Number(item.Size),
+      uploaded: new Date(item.LastModified),
+      url: new URL(publicPath(item.Key), archiveOrigin).href,
+    })),
+    truncated: result.IsTruncated === 'true',
+  };
+}
+
+/**
+ * @param {string} prefix
+ * @param {string | undefined} cursor
+ * @param {R2Objects} listing
+ * @returns {R2Objects} The listing with any virtual directories added.
+ */
+function withVirtualDirectories(prefix, cursor, listing) {
+  if (prefix !== 'data/' || cursor || listing.delimitedPrefixes.includes(archiveMountPrefix)) {
+    return listing;
+  }
+
+  return {
+    ...listing,
+    delimitedPrefixes: [...listing.delimitedPrefixes, archiveMountPrefix].sort(),
+  };
+}
+
 /** @param {string} key */
 function publicPath(key) {
  return `/${key.split('/').map(encodeURIComponent).join('/')}`;
@@ -100,15 +184,19 @@ function directoryPage(origin, prefix, listing, cursor) {
      prefix: directoryPrefix,
      url: new URL(publicPath(directoryPrefix), origin).href,
    })),
-    files: listing.objects.map(object => ({
-      etag: object.etag,
-      key: object.key,
-      name: displayName(object.key, prefix),
-      pathname: publicPath(object.key),
-      size: object.size,
-      uploaded: object.uploaded,
-      url: new URL(publicPath(object.key), origin).href,
-    })),
+    files: listing.objects.map(object => {
+      let pathname = publicPath(object.browserKey ?? object.key);
+      return {
+        etag: object.etag,
+        href: object.url ?? pathname,
+        key: object.key,
+        name: displayName(object.browserKey ?? object.key, prefix),
+        pathname,
+        size: object.size,
+        uploaded: object.uploaded,
+        url: object.url ?? new URL(publicPath(object.key), origin).href,
+      };
+    }),
    nextCursor: listing.truncated ? listing.cursor : undefined,
    pathname,
    prefix,
@@ -157,7 +245,7 @@ function renderListing(page, styleNonce) {
    ...page.files.map(file => ({
      date: formatDate(file.uploaded),
      datetime: file.uploaded.toISOString(),
-      href: file.pathname,
+      href: file.href,
      isFile: true,
      name: file.name,
      size: formatSize(file.size),
@@ -319,19 +407,24 @@ export default {
    }

    let cursor = url.searchParams.get('cursor') || undefined;
+    let archiveListing = prefix.startsWith(archiveMountPrefix);
    /** @type {R2Objects} */
    let listing;
    try {
-      listing = await env.ASSETS.list({
-        cursor,
-        delimiter: '/',
-        limit: pageSize,
-        prefix,
-      });
+      listing = archiveListing
+        ? await listArchive(env, prefix, cursor)
+        : await env.ASSETS.list({
+          cursor,
+          delimiter: '/',
+          limit: pageSize,
+          prefix,
+        });
    } catch (error) {
      console.error(JSON.stringify({
        error: error instanceof Error ? error.message : String(error),
-        message: 'R2 directory listing failed',
+        message: archiveListing
+          ? 'Archive directory listing failed'
+          : 'R2 directory listing failed',
        prefix,
      }));
      return listingErrorResponse(request, 500, 'listing_unavailable', 'Directory listing unavailable');
@@ -340,6 +433,7 @@ export default {
      return listingErrorResponse(request, 404, 'directory_not_found', 'Directory not found');
    }

+    listing = withVirtualDirectories(prefix, cursor, listing);
    let page = directoryPage(url.origin, prefix, listing, cursor);
    if (url.searchParams.get('format') === 'json') {
      return jsonListingResponse(request, page);
--- a/workers/assets-directory-browser/src/index.test.mjs
+++ b/workers/assets-directory-browser/src/index.test.mjs
@@ -90,6 +90,64 @@ describe('assets directory browser', () => {
    expect(html).not.toContain('__directory');
  });

+  it('shows the data archive as a directory in the R2 data listing', async () => {
+    let bucket = new FakeBucket({
+      cursor: undefined,
+      delimitedPrefixes: ['data/locale/'],
+      objects: [],
+      truncated: false,
+    });
+
+    let response = await worker.fetch(new Request(
+      'https://assets.splatoon3.ink/__directory/data/',
+    ), { ASSETS: bucket });
+    let html = await response.text();
+
+    expect(response.status).toBe(200);
+    expect(html).toContain('href="/data/archive/"');
+    expect(html).toContain('<span class="entry-name">archive&#x2F;</span>');
+  });
+
+  it('browses archive directories while linking files to the public archive origin', async () => {
+    let fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(`
+      <?xml version="1.0" encoding="UTF-8"?>
+      <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+        <Prefix>2026/08/</Prefix>
+        <IsTruncated>true</IsTruncated>
+        <Contents>
+          <Key>2026/08/2026-08-01.tar.zst</Key>
+          <LastModified>2026-08-31T01:02:03.000Z</LastModified>
+          <ETag>&quot;archive-etag&quot;</ETag>
+          <Size>1536</Size>
+        </Contents>
+        <CommonPrefixes><Prefix>2026/08/31/</Prefix></CommonPrefixes>
+        <NextContinuationToken>next archive page</NextContinuationToken>
+      </ListBucketResult>
+    `, { headers: { 'Content-Type': 'application/xml' } }));
+
+    let response = await worker.fetch(new Request(
+      'http://localhost:8787/data/archive/2026/08/?cursor=current%20archive%20page',
+    ), {
+      ARCHIVE_ORIGIN: 'https://data-archive.splatoon3.ink',
+      ASSETS: new FakeBucket,
+      LOCAL_DEVELOPMENT: 'true',
+    });
+    let html = await response.text();
+
+    expect(response.status).toBe(200);
+    expect(fetchSpy).toHaveBeenCalledOnce();
+    expect(fetchSpy.mock.calls[0][0].url).toBe(
+      'https://data-archive.splatoon3.ink/?list-type=2&delimiter=%2F&max-keys=1000&prefix=2026%2F08%2F&continuation-token=current+archive+page',
+    );
+    expect(html).toContain('href="/data/archive/2026/08/31/"');
+    expect(html).toContain('href="https://data-archive.splatoon3.ink/2026/08/2026-08-01.tar.zst"');
+    expect(html).toContain('href="/data/archive/2026/08/?cursor=next%20archive%20page"');
+    expect(html).toContain('1.5 KB');
+    expect(html).toContain('2026-08-31 01:02:03 UTC');
+
+    fetchSpy.mockRestore();
+  });
+
  it('returns a public JSON representation of a directory page', async () => {
    let bucket = new FakeBucket({
      cursor: 'next page',
@@ -283,4 +341,30 @@ describe('assets directory browser', () => {
    }));
    errorLog.mockRestore();
  });
+
+  it('returns a controlled error when the archive listing fails', async () => {
+    let errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
+    let fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
+      new Response('Unavailable', { status: 503 }),
+    );
+
+    let response = await worker.fetch(new Request(
+      'https://assets.splatoon3.ink/__directory/data/archive/2026/',
+    ), {
+      ARCHIVE_ORIGIN: 'https://data-archive.splatoon3.ink',
+      ASSETS: new FakeBucket,
+    });
+
+    expect(response.status).toBe(500);
+    expect(response.headers.get('cache-control')).toBe('no-store');
+    expect(await response.text()).toBe('Directory listing unavailable');
+    expect(errorLog).toHaveBeenCalledWith(JSON.stringify({
+      error: 'Archive listing returned 503',
+      message: 'Archive directory listing failed',
+      prefix: 'data/archive/2026/',
+    }));
+
+    fetchSpy.mockRestore();
+    errorLog.mockRestore();
+  });
 });
--- a/workers/assets-directory-browser/wrangler.jsonc
+++ b/workers/assets-directory-browser/wrangler.jsonc
@@ -6,6 +6,9 @@
  "compatibility_flags": ["nodejs_compat"],
  "workers_dev": false,
  "preview_urls": false,
+  "vars": {
+    "ARCHIVE_ORIGIN": "https://data-archive.splatoon3.ink"
+  },
  "routes": [
    {
      "pattern": "assets.splatoon3.ink/__directory/*",