diff --git a/app/common/fs.mjs b/app/common/fs.mjs index c673623..9d8fab8 100644 --- a/app/common/fs.mjs +++ b/app/common/fs.mjs @@ -1,4 +1,5 @@ import fs from 'fs/promises'; +import vfs from './vfs.mjs'; export function mkdirp(dir) { return fs.mkdir(dir, { recursive: true }); @@ -9,8 +10,9 @@ export async function exists(file) { await fs.access(file); return true; - } catch (e) { - return false; + } catch { + // Not on local disk; check VFS (S3 listing) as fallback + return (await vfs.has(file)) ?? false; } } @@ -20,7 +22,13 @@ export async function olderThan(file, cutoff) { let stat = await fs.stat(file); return stat.mtime < cutoff; - } catch (e) { + } catch { + // Not on local disk; check VFS (S3 listing) as fallback + const mtime = await vfs.getMtime(file); + if (mtime !== null) { + return mtime < cutoff; + } + return true; } } diff --git a/app/common/vfs.mjs b/app/common/vfs.mjs new file mode 100644 index 0000000..c6c5dfa --- /dev/null +++ b/app/common/vfs.mjs @@ -0,0 +1,131 @@ +import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3'; +import prefixedConsole from './prefixedConsole.mjs'; + +const prefixes = ['assets/splatnet/', 'data/']; + +class VirtualFileSystem { + // Map of S3 key to { lastModified: Date, size: number } + _listing = new Map(); + _loadPromise = null; + _localPrefix = 'dist'; + + get _console() { + return this.__console ??= prefixedConsole('VFS'); + } + + get _canUseS3() { + return !!( + process.env.AWS_ACCESS_KEY_ID && + process.env.AWS_SECRET_ACCESS_KEY && + process.env.AWS_S3_BUCKET + ); + } + + get _s3Client() { + return this.__s3Client ??= new S3Client({ + endpoint: process.env.AWS_S3_ENDPOINT, + region: process.env.AWS_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, + }); + } + + /** + * Ensure the S3 listing is loaded. Loads once on first call, + * subsequent calls return the same promise. + */ + async _ensureLoaded() { + if (!this._canUseS3) return false; + + this._loadPromise ??= this._loadFromS3(); + await this._loadPromise; + return true; + } + + async _loadFromS3() { + const bucket = process.env.AWS_S3_BUCKET; + + this._console.info('Loading S3 listing...'); + + for (const prefix of prefixes) { + let continuationToken; + let count = 0; + + do { + const response = await this._s3Client.send(new ListObjectsV2Command({ + Bucket: bucket, + Prefix: prefix, + ContinuationToken: continuationToken, + })); + + for (const obj of response.Contents ?? []) { + this._listing.set(obj.Key, { + lastModified: obj.LastModified, + size: obj.Size, + }); + count++; + } + + continuationToken = response.IsTruncated + ? response.NextContinuationToken + : undefined; + } while (continuationToken); + + this._console.info(`Loaded ${count} entries for prefix "${prefix}"`); + } + } + + /** + * Check if a local file path is known to exist in the VFS listing. + * Returns true/false if the path is within a tracked prefix, + * or null if VFS is not available. + * @param {string} localPath + */ + async has(localPath) { + if (!(await this._ensureLoaded())) return null; + + const key = this._localPathToKey(localPath); + if (key === null) return null; + if (!this._isTrackedKey(key)) return null; + + return this._listing.has(key); + } + + /** + * Get the last modified time for a file from the S3 listing. + * Returns Date if found, null if not tracked or VFS not available. + * @param {string} localPath + */ + async getMtime(localPath) { + if (!(await this._ensureLoaded())) return null; + + const key = this._localPathToKey(localPath); + if (key === null) return null; + if (!this._isTrackedKey(key)) return null; + + const entry = this._listing.get(key); + return entry ? entry.lastModified : null; + } + + /** + * Convert a local path (e.g. 'dist/assets/splatnet/foo.png') + * to an S3 key (e.g. 'assets/splatnet/foo.png'). + * @param {string} localPath + */ + _localPathToKey(localPath) { + const prefix = this._localPrefix + '/'; + if (localPath.startsWith(prefix)) { + return localPath.slice(prefix.length); + } + return null; + } + + _isTrackedKey(key) { + return prefixes.some(prefix => key.startsWith(prefix)); + } +} + +const vfs = new VirtualFileSystem(); +export default vfs; diff --git a/app/data/updaters/DataUpdater.mjs b/app/data/updaters/DataUpdater.mjs index 46c546c..9afb683 100644 --- a/app/data/updaters/DataUpdater.mjs +++ b/app/data/updaters/DataUpdater.mjs @@ -89,13 +89,13 @@ export default class DataUpdater await this.updateLocalizations(this.defaultLocale, data); // Download any new images - const images = await this.downloadImages(data); + await this.downloadImages(data); // Write the data to disk await this.saveData(data); // Update iCal data - await this.updateCalendarEvents(data, images); + await this.updateCalendarEvents(data); this.console.info('Done'); } @@ -223,11 +223,11 @@ export default class DataUpdater // Calendar output - async updateCalendarEvents(data, images) { + async updateCalendarEvents(data) { const events = this.getCalendarEntries(data); if (!events) return; - const ical = await this.getiCalData(events, images); + const ical = await this.getiCalData(events); await this.writeFile(this.getCalendarPath(this.calendarFilename ?? this.filename), ical); } @@ -239,7 +239,7 @@ export default class DataUpdater // } - async getiCalData(events, images) { + async getiCalData(events) { // Create a calendar object const calendar = ical({ name: this.calendarName ?? this.name, @@ -252,9 +252,6 @@ export default class DataUpdater timezone: 'UTC', }); - // Create a map of image URLs to image data - const imageData = {}; - // Add event entries for (let event of events) { let calEvent = calendar.createEvent({ @@ -265,30 +262,11 @@ export default class DataUpdater url: event.url, }); calEvent.createAttachment(event.imageUrl); - - const filename = images[event.imageUrl]; - if (filename) { - const data = await fs.readFile(this.imageProcessor.localPath(filename)); - imageData[event.imageUrl] = data; - } } // Convert the calendar to an ICS string let ics = calendar.toString(); - // Embed image attachments - ics = ics.replaceAll(/^ATTACH:((.|\r\n )*)$/gm, (match, url) => { - url = url.replaceAll('\r\n ', ''); - - const filename = images[url]; - const data = imageData[url]; - if (!filename || !data) return match; - - const ical = `ATTACH;ENCODING=BASE64;VALUE=BINARY;X-APPLE-FILENAME=${path.basename(filename)}:${data.toString('base64')}`; - - return ical.replace(/(.{72})/g, '$1\r\n ').trim(); - }); - return ics; } } diff --git a/app/sync/S3Syncer.mjs b/app/sync/S3Syncer.mjs index 911eac6..1e0c070 100644 --- a/app/sync/S3Syncer.mjs +++ b/app/sync/S3Syncer.mjs @@ -10,7 +10,7 @@ export default class S3Syncer return Promise.all([ dist && this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, { - filters: this.filters, + filters: this.downloadFilters, }), storage && this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`, { filters: this.privateFilters, @@ -76,6 +76,20 @@ export default class S3Syncer ]; } + // Download filters skip files that are handled by the VFS layer + // (existence/mtime checks only, no content reads needed) + get downloadFilters() { + return [ + { exclude: () => true }, // Exclude everything by default + // assets/splatnet/ - handled by VFS (existence checks only) + // status-screenshots/ - regenerated by social posting + { include: (key) => key.startsWith('data/') }, + { exclude: (key) => key.startsWith('data/archive/') }, + { exclude: (key) => key.startsWith('data/xrank/') }, + { exclude: (key) => key.startsWith('data/festivals.ranking.') }, + ]; + } + get privateFilters() { return [ { exclude: (key) => key.startsWith('archive/') },