From 41c9f9a3153ff565abcfe6205d3d433214bcaa96 Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sun, 22 Feb 2026 09:26:05 -0800 Subject: [PATCH 1/6] Add VFS layer to avoid downloading files only needed for existence checks Uses S3 ListObjectsV2 to build an in-memory file listing at startup, allowing exists() and olderThan() to resolve from the listing instead of requiring files on disk. sync:download now skips assets/splatnet/, data/xrank/, data/festivals.ranking.*, and status-screenshots/. Co-Authored-By: Claude Opus 4.6 --- app/common/fs.mjs | 11 +++ app/common/vfs.mjs | 142 ++++++++++++++++++++++++++++++ app/data/ImageProcessor.mjs | 2 + app/data/updaters/DataUpdater.mjs | 8 +- app/sync/S3Syncer.mjs | 16 +++- app/sync/index.mjs | 4 + 6 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 app/common/vfs.mjs diff --git a/app/common/fs.mjs b/app/common/fs.mjs index c673623..8f93c15 100644 --- a/app/common/fs.mjs +++ b/app/common/fs.mjs @@ -1,10 +1,16 @@ import fs from 'fs/promises'; +import vfs from './vfs.mjs'; export function mkdirp(dir) { return fs.mkdir(dir, { recursive: true }); } export async function exists(file) { + const vfsResult = vfs.has(file); + if (vfsResult !== null) { + return vfsResult; + } + try { await fs.access(file); @@ -16,6 +22,11 @@ export async function exists(file) { // Determine whether a file is older than a given cutoff date (or doesn't exist) export async function olderThan(file, cutoff) { + const mtime = vfs.getMtime(file); + if (mtime !== null) { + return mtime < cutoff; + } + try { let stat = await fs.stat(file); diff --git a/app/common/vfs.mjs b/app/common/vfs.mjs new file mode 100644 index 0000000..1b64f59 --- /dev/null +++ b/app/common/vfs.mjs @@ -0,0 +1,142 @@ +import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3'; + +class VirtualFileSystem { + // Map of S3 key to { lastModified: Date, size: number } + _listing = new Map(); + _loaded = false; + _trackedPrefixes = []; + _localPrefix = 'dist'; + + get _canUseS3() { + return !!( + process.env.AWS_ACCESS_KEY_ID && + process.env.AWS_SECRET_ACCESS_KEY && + process.env.AWS_S3_BUCKET + ); + } + + get _s3Client() { + return this.__s3Client ??= new S3Client({ + endpoint: process.env.AWS_S3_ENDPOINT, + region: process.env.AWS_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, + }); + } + + /** + * Load S3 listing for the given prefixes. + * Call once at startup before any exists/olderThan checks. + * @param {string[]} prefixes + */ + async loadFromS3(prefixes) { + if (!this._canUseS3) { + return; + } + + this._trackedPrefixes = prefixes; + const bucket = process.env.AWS_S3_BUCKET; + + console.log('[VFS] Loading S3 listing...'); + + for (const prefix of prefixes) { + let continuationToken; + let count = 0; + + do { + const response = await this._s3Client.send(new ListObjectsV2Command({ + Bucket: bucket, + Prefix: prefix, + ContinuationToken: continuationToken, + })); + + for (const obj of response.Contents ?? []) { + this._listing.set(obj.Key, { + lastModified: obj.LastModified, + size: obj.Size, + }); + count++; + } + + continuationToken = response.IsTruncated + ? response.NextContinuationToken + : undefined; + } while (continuationToken); + + console.log(`[VFS] Loaded ${count} entries for prefix "${prefix}"`); + } + + this._loaded = true; + } + + /** + * Check if a local file path is known to exist in the VFS listing. + * Returns true/false if the path is within a tracked prefix, + * or null if VFS has no opinion (not loaded, or path outside tracked prefixes). + * @param {string} localPath + */ + has(localPath) { + if (!this._loaded) return null; + + const key = this._localPathToKey(localPath); + if (key === null) return null; + if (!this._isTrackedKey(key)) return null; + + return this._listing.has(key); + } + + /** + * Get the last modified time for a file from the S3 listing. + * Returns Date if found, null if not tracked or VFS not loaded. + * @param {string} localPath + */ + getMtime(localPath) { + if (!this._loaded) return null; + + const key = this._localPathToKey(localPath); + if (key === null) return null; + if (!this._isTrackedKey(key)) return null; + + const entry = this._listing.get(key); + return entry ? entry.lastModified : null; + } + + /** + * Track a file that was just written locally. + * Ensures subsequent has() calls return true without hitting disk. + * @param {string} localPath + */ + track(localPath) { + if (!this._loaded) return; + + const key = this._localPathToKey(localPath); + if (key === null) return; + + this._listing.set(key, { + lastModified: new Date(), + size: 0, + }); + } + + /** + * Convert a local path (e.g. 'dist/assets/splatnet/foo.png') + * to an S3 key (e.g. 'assets/splatnet/foo.png'). + * @param {string} localPath + */ + _localPathToKey(localPath) { + const prefix = this._localPrefix + '/'; + if (localPath.startsWith(prefix)) { + return localPath.slice(prefix.length); + } + return null; + } + + _isTrackedKey(key) { + return this._trackedPrefixes.some(prefix => key.startsWith(prefix)); + } +} + +const vfs = new VirtualFileSystem(); +export default vfs; diff --git a/app/data/ImageProcessor.mjs b/app/data/ImageProcessor.mjs index 254b36f..343949c 100644 --- a/app/data/ImageProcessor.mjs +++ b/app/data/ImageProcessor.mjs @@ -4,6 +4,7 @@ import PQueue from 'p-queue'; import prefixedConsole from '../common/prefixedConsole.mjs'; import { normalizeSplatnetResourcePath } from '../common/util.mjs'; import { exists, mkdirp } from '../common/fs.mjs'; +import vfs from '../common/vfs.mjs'; const queue = new PQueue({ concurrency: 4 }); @@ -71,6 +72,7 @@ export default class ImageProcessor await mkdirp(path.dirname(this.localPath(destination))); await fs.writeFile(this.localPath(destination), result.body); + vfs.track(this.localPath(destination)); } catch (e) { this.console.error(`Image download failed for ${destination}`, e); } diff --git a/app/data/updaters/DataUpdater.mjs b/app/data/updaters/DataUpdater.mjs index 46c546c..6573778 100644 --- a/app/data/updaters/DataUpdater.mjs +++ b/app/data/updaters/DataUpdater.mjs @@ -268,8 +268,12 @@ export default class DataUpdater const filename = images[event.imageUrl]; if (filename) { - const data = await fs.readFile(this.imageProcessor.localPath(filename)); - imageData[event.imageUrl] = data; + try { + const data = await fs.readFile(this.imageProcessor.localPath(filename)); + imageData[event.imageUrl] = data; + } catch { + // Image not available locally (may only exist in S3); skip inline embed + } } } diff --git a/app/sync/S3Syncer.mjs b/app/sync/S3Syncer.mjs index 911eac6..1e0c070 100644 --- a/app/sync/S3Syncer.mjs +++ b/app/sync/S3Syncer.mjs @@ -10,7 +10,7 @@ export default class S3Syncer return Promise.all([ dist && this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, { - filters: this.filters, + filters: this.downloadFilters, }), storage && this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`, { filters: this.privateFilters, @@ -76,6 +76,20 @@ export default class S3Syncer ]; } + // Download filters skip files that are handled by the VFS layer + // (existence/mtime checks only, no content reads needed) + get downloadFilters() { + return [ + { exclude: () => true }, // Exclude everything by default + // assets/splatnet/ - handled by VFS (existence checks only) + // status-screenshots/ - regenerated by social posting + { include: (key) => key.startsWith('data/') }, + { exclude: (key) => key.startsWith('data/archive/') }, + { exclude: (key) => key.startsWith('data/xrank/') }, + { exclude: (key) => key.startsWith('data/festivals.ranking.') }, + ]; + } + get privateFilters() { return [ { exclude: (key) => key.startsWith('archive/') }, diff --git a/app/sync/index.mjs b/app/sync/index.mjs index ddb7dea..5e4b3da 100644 --- a/app/sync/index.mjs +++ b/app/sync/index.mjs @@ -1,4 +1,5 @@ import S3Syncer from './S3Syncer.mjs'; +import vfs from '../common/vfs.mjs'; export function canSync() { return !!( @@ -18,6 +19,9 @@ async function doSync(download, upload) { const syncer = new S3Syncer(); if (download) { + // Load VFS listing for prefixes that won't be downloaded + await vfs.loadFromS3(['assets/splatnet/', 'data/']); + console.info('Downloading files...'); await syncer.download(); } From e6e96ac8f9ac181501a722d1360be8b9f9ada18e Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sun, 22 Feb 2026 09:27:00 -0800 Subject: [PATCH 2/6] Check local disk before VFS, removing need for manual vfs.track() exists() and olderThan() now check the local filesystem first and only consult the VFS as a fallback. This means freshly written files are found naturally without callers needing to register them. Co-Authored-By: Claude Opus 4.6 --- app/common/fs.mjs | 19 ++++++++----------- app/common/vfs.mjs | 17 ----------------- app/data/ImageProcessor.mjs | 2 -- 3 files changed, 8 insertions(+), 30 deletions(-) diff --git a/app/common/fs.mjs b/app/common/fs.mjs index 8f93c15..5bed895 100644 --- a/app/common/fs.mjs +++ b/app/common/fs.mjs @@ -6,32 +6,29 @@ export function mkdirp(dir) { } export async function exists(file) { - const vfsResult = vfs.has(file); - if (vfsResult !== null) { - return vfsResult; - } - try { await fs.access(file); return true; } catch (e) { - return false; + // Not on local disk; check VFS (S3 listing) as fallback + return vfs.has(file) ?? false; } } // Determine whether a file is older than a given cutoff date (or doesn't exist) export async function olderThan(file, cutoff) { - const mtime = vfs.getMtime(file); - if (mtime !== null) { - return mtime < cutoff; - } - try { let stat = await fs.stat(file); return stat.mtime < cutoff; } catch (e) { + // Not on local disk; check VFS (S3 listing) as fallback + const mtime = vfs.getMtime(file); + if (mtime !== null) { + return mtime < cutoff; + } + return true; } } diff --git a/app/common/vfs.mjs b/app/common/vfs.mjs index 1b64f59..22ae017 100644 --- a/app/common/vfs.mjs +++ b/app/common/vfs.mjs @@ -103,23 +103,6 @@ class VirtualFileSystem { return entry ? entry.lastModified : null; } - /** - * Track a file that was just written locally. - * Ensures subsequent has() calls return true without hitting disk. - * @param {string} localPath - */ - track(localPath) { - if (!this._loaded) return; - - const key = this._localPathToKey(localPath); - if (key === null) return; - - this._listing.set(key, { - lastModified: new Date(), - size: 0, - }); - } - /** * Convert a local path (e.g. 'dist/assets/splatnet/foo.png') * to an S3 key (e.g. 'assets/splatnet/foo.png'). diff --git a/app/data/ImageProcessor.mjs b/app/data/ImageProcessor.mjs index 343949c..254b36f 100644 --- a/app/data/ImageProcessor.mjs +++ b/app/data/ImageProcessor.mjs @@ -4,7 +4,6 @@ import PQueue from 'p-queue'; import prefixedConsole from '../common/prefixedConsole.mjs'; import { normalizeSplatnetResourcePath } from '../common/util.mjs'; import { exists, mkdirp } from '../common/fs.mjs'; -import vfs from '../common/vfs.mjs'; const queue = new PQueue({ concurrency: 4 }); @@ -72,7 +71,6 @@ export default class ImageProcessor await mkdirp(path.dirname(this.localPath(destination))); await fs.writeFile(this.localPath(destination), result.body); - vfs.track(this.localPath(destination)); } catch (e) { this.console.error(`Image download failed for ${destination}`, e); } From 7125f30bfc01b85cc1cc117728a1b61046ce1664 Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sun, 22 Feb 2026 09:44:04 -0800 Subject: [PATCH 3/6] Remove inline base64 image embedding from iCal output No major calendar client renders these inline binary attachments. The ATTACH URL is kept, which links to the publicly hosted image. Co-Authored-By: Claude Opus 4.6 --- app/data/updaters/DataUpdater.mjs | 36 +++++-------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/app/data/updaters/DataUpdater.mjs b/app/data/updaters/DataUpdater.mjs index 6573778..9afb683 100644 --- a/app/data/updaters/DataUpdater.mjs +++ b/app/data/updaters/DataUpdater.mjs @@ -89,13 +89,13 @@ export default class DataUpdater await this.updateLocalizations(this.defaultLocale, data); // Download any new images - const images = await this.downloadImages(data); + await this.downloadImages(data); // Write the data to disk await this.saveData(data); // Update iCal data - await this.updateCalendarEvents(data, images); + await this.updateCalendarEvents(data); this.console.info('Done'); } @@ -223,11 +223,11 @@ export default class DataUpdater // Calendar output - async updateCalendarEvents(data, images) { + async updateCalendarEvents(data) { const events = this.getCalendarEntries(data); if (!events) return; - const ical = await this.getiCalData(events, images); + const ical = await this.getiCalData(events); await this.writeFile(this.getCalendarPath(this.calendarFilename ?? this.filename), ical); } @@ -239,7 +239,7 @@ export default class DataUpdater // } - async getiCalData(events, images) { + async getiCalData(events) { // Create a calendar object const calendar = ical({ name: this.calendarName ?? this.name, @@ -252,9 +252,6 @@ export default class DataUpdater timezone: 'UTC', }); - // Create a map of image URLs to image data - const imageData = {}; - // Add event entries for (let event of events) { let calEvent = calendar.createEvent({ @@ -265,34 +262,11 @@ export default class DataUpdater url: event.url, }); calEvent.createAttachment(event.imageUrl); - - const filename = images[event.imageUrl]; - if (filename) { - try { - const data = await fs.readFile(this.imageProcessor.localPath(filename)); - imageData[event.imageUrl] = data; - } catch { - // Image not available locally (may only exist in S3); skip inline embed - } - } } // Convert the calendar to an ICS string let ics = calendar.toString(); - // Embed image attachments - ics = ics.replaceAll(/^ATTACH:((.|\r\n )*)$/gm, (match, url) => { - url = url.replaceAll('\r\n ', ''); - - const filename = images[url]; - const data = imageData[url]; - if (!filename || !data) return match; - - const ical = `ATTACH;ENCODING=BASE64;VALUE=BINARY;X-APPLE-FILENAME=${path.basename(filename)}:${data.toString('base64')}`; - - return ical.replace(/(.{72})/g, '$1\r\n ').trim(); - }); - return ics; } } From 6dac044a69cb82a99dc2d6ae794017b499438f59 Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sun, 22 Feb 2026 09:54:44 -0800 Subject: [PATCH 4/6] Make VFS load lazily VFS now loads on first has()/getMtime() call rather than requiring explicit initialization in sync/index.mjs. This ensures it works regardless of entry point (sync:download, splatnet:quick, etc.). Co-Authored-By: Claude Opus 4.6 --- app/common/fs.mjs | 4 ++-- app/common/vfs.mjs | 39 ++++++++++++++++++++------------------- app/sync/index.mjs | 4 ---- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/app/common/fs.mjs b/app/common/fs.mjs index 5bed895..25422d9 100644 --- a/app/common/fs.mjs +++ b/app/common/fs.mjs @@ -12,7 +12,7 @@ export async function exists(file) { return true; } catch (e) { // Not on local disk; check VFS (S3 listing) as fallback - return vfs.has(file) ?? false; + return (await vfs.has(file)) ?? false; } } @@ -24,7 +24,7 @@ export async function olderThan(file, cutoff) { return stat.mtime < cutoff; } catch (e) { // Not on local disk; check VFS (S3 listing) as fallback - const mtime = vfs.getMtime(file); + const mtime = await vfs.getMtime(file); if (mtime !== null) { return mtime < cutoff; } diff --git a/app/common/vfs.mjs b/app/common/vfs.mjs index 22ae017..8a2bef0 100644 --- a/app/common/vfs.mjs +++ b/app/common/vfs.mjs @@ -1,10 +1,11 @@ import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3'; +const prefixes = ['assets/splatnet/', 'data/']; + class VirtualFileSystem { // Map of S3 key to { lastModified: Date, size: number } _listing = new Map(); - _loaded = false; - _trackedPrefixes = []; + _loadPromise = null; _localPrefix = 'dist'; get _canUseS3() { @@ -27,16 +28,18 @@ class VirtualFileSystem { } /** - * Load S3 listing for the given prefixes. - * Call once at startup before any exists/olderThan checks. - * @param {string[]} prefixes + * Ensure the S3 listing is loaded. Loads once on first call, + * subsequent calls return the same promise. */ - async loadFromS3(prefixes) { - if (!this._canUseS3) { - return; - } + async _ensureLoaded() { + if (!this._canUseS3) return false; - this._trackedPrefixes = prefixes; + this._loadPromise ??= this._loadFromS3(); + await this._loadPromise; + return true; + } + + async _loadFromS3() { const bucket = process.env.AWS_S3_BUCKET; console.log('[VFS] Loading S3 listing...'); @@ -67,18 +70,16 @@ class VirtualFileSystem { console.log(`[VFS] Loaded ${count} entries for prefix "${prefix}"`); } - - this._loaded = true; } /** * Check if a local file path is known to exist in the VFS listing. * Returns true/false if the path is within a tracked prefix, - * or null if VFS has no opinion (not loaded, or path outside tracked prefixes). + * or null if VFS is not available. * @param {string} localPath */ - has(localPath) { - if (!this._loaded) return null; + async has(localPath) { + if (!(await this._ensureLoaded())) return null; const key = this._localPathToKey(localPath); if (key === null) return null; @@ -89,11 +90,11 @@ class VirtualFileSystem { /** * Get the last modified time for a file from the S3 listing. - * Returns Date if found, null if not tracked or VFS not loaded. + * Returns Date if found, null if not tracked or VFS not available. * @param {string} localPath */ - getMtime(localPath) { - if (!this._loaded) return null; + async getMtime(localPath) { + if (!(await this._ensureLoaded())) return null; const key = this._localPathToKey(localPath); if (key === null) return null; @@ -117,7 +118,7 @@ class VirtualFileSystem { } _isTrackedKey(key) { - return this._trackedPrefixes.some(prefix => key.startsWith(prefix)); + return prefixes.some(prefix => key.startsWith(prefix)); } } diff --git a/app/sync/index.mjs b/app/sync/index.mjs index 5e4b3da..ddb7dea 100644 --- a/app/sync/index.mjs +++ b/app/sync/index.mjs @@ -1,5 +1,4 @@ import S3Syncer from './S3Syncer.mjs'; -import vfs from '../common/vfs.mjs'; export function canSync() { return !!( @@ -19,9 +18,6 @@ async function doSync(download, upload) { const syncer = new S3Syncer(); if (download) { - // Load VFS listing for prefixes that won't be downloaded - await vfs.loadFromS3(['assets/splatnet/', 'data/']); - console.info('Downloading files...'); await syncer.download(); } From 941a6eb2584224581cbd6f1ebb2485d6399b92e0 Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sun, 22 Feb 2026 10:09:17 -0800 Subject: [PATCH 5/6] Remove unused catch binding variables in fs.mjs Co-Authored-By: Claude Opus 4.6 --- app/common/fs.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/common/fs.mjs b/app/common/fs.mjs index 25422d9..9d8fab8 100644 --- a/app/common/fs.mjs +++ b/app/common/fs.mjs @@ -10,7 +10,7 @@ export async function exists(file) { await fs.access(file); return true; - } catch (e) { + } catch { // Not on local disk; check VFS (S3 listing) as fallback return (await vfs.has(file)) ?? false; } @@ -22,7 +22,7 @@ export async function olderThan(file, cutoff) { let stat = await fs.stat(file); return stat.mtime < cutoff; - } catch (e) { + } catch { // Not on local disk; check VFS (S3 listing) as fallback const mtime = await vfs.getMtime(file); if (mtime !== null) { From 96725eecbbf5b03ad448c647c49fc8a1f4efa565 Mon Sep 17 00:00:00 2001 From: Matt Isenhower Date: Sun, 22 Feb 2026 10:11:40 -0800 Subject: [PATCH 6/6] Use prefixedConsole for VFS logging Lazily initialized to avoid console-stamp issues in test environment. Co-Authored-By: Claude Opus 4.6 --- app/common/vfs.mjs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/common/vfs.mjs b/app/common/vfs.mjs index 8a2bef0..c6c5dfa 100644 --- a/app/common/vfs.mjs +++ b/app/common/vfs.mjs @@ -1,4 +1,5 @@ import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3'; +import prefixedConsole from './prefixedConsole.mjs'; const prefixes = ['assets/splatnet/', 'data/']; @@ -8,6 +9,10 @@ class VirtualFileSystem { _loadPromise = null; _localPrefix = 'dist'; + get _console() { + return this.__console ??= prefixedConsole('VFS'); + } + get _canUseS3() { return !!( process.env.AWS_ACCESS_KEY_ID && @@ -42,7 +47,7 @@ class VirtualFileSystem { async _loadFromS3() { const bucket = process.env.AWS_S3_BUCKET; - console.log('[VFS] Loading S3 listing...'); + this._console.info('Loading S3 listing...'); for (const prefix of prefixes) { let continuationToken; @@ -68,7 +73,7 @@ class VirtualFileSystem { : undefined; } while (continuationToken); - console.log(`[VFS] Loaded ${count} entries for prefix "${prefix}"`); + this._console.info(`Loaded ${count} entries for prefix "${prefix}"`); } }