Merge pull request #105 from misenhower/vfs

Add VFS to improve deployment

app/common/fs.mjs

@@ -1,4 +1,5 @@
 import fs from 'fs/promises';
+import vfs from './vfs.mjs';
 
 export function mkdirp(dir) {
   return fs.mkdir(dir, { recursive: true });
@@ -9,8 +10,9 @@ export async function exists(file) {
     await fs.access(file);
 
     return true;
-  } catch (e) {
-    return false;
+  } catch {
+    // Not on local disk; check VFS (S3 listing) as fallback
+    return (await vfs.has(file)) ?? false;
   }
 }
 
@@ -20,7 +22,13 @@ export async function olderThan(file, cutoff) {
     let stat = await fs.stat(file);
 
     return stat.mtime < cutoff;
-  } catch (e) {
+  } catch {
+    // Not on local disk; check VFS (S3 listing) as fallback
+    const mtime = await vfs.getMtime(file);
+    if (mtime !== null) {
+      return mtime < cutoff;
+    }
+
     return true;
   }
 }

app/common/vfs.mjs

@@ -0,0 +1,131 @@
+import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3';
+import prefixedConsole from './prefixedConsole.mjs';
+
+const prefixes = ['assets/splatnet/', 'data/'];
+
+class VirtualFileSystem {
+  // Map of S3 key to { lastModified: Date, size: number }
+  _listing = new Map();
+  _loadPromise = null;
+  _localPrefix = 'dist';
+
+  get _console() {
+    return this.__console ??= prefixedConsole('VFS');
+  }
+
+  get _canUseS3() {
+    return !!(
+      process.env.AWS_ACCESS_KEY_ID &&
+      process.env.AWS_SECRET_ACCESS_KEY &&
+      process.env.AWS_S3_BUCKET
+    );
+  }
+
+  get _s3Client() {
+    return this.__s3Client ??= new S3Client({
+      endpoint: process.env.AWS_S3_ENDPOINT,
+      region: process.env.AWS_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+      },
+    });
+  }
+
+  /**
+   * Ensure the S3 listing is loaded. Loads once on first call,
+   * subsequent calls return the same promise.
+   */
+  async _ensureLoaded() {
+    if (!this._canUseS3) return false;
+
+    this._loadPromise ??= this._loadFromS3();
+    await this._loadPromise;
+    return true;
+  }
+
+  async _loadFromS3() {
+    const bucket = process.env.AWS_S3_BUCKET;
+
+    this._console.info('Loading S3 listing...');
+
+    for (const prefix of prefixes) {
+      let continuationToken;
+      let count = 0;
+
+      do {
+        const response = await this._s3Client.send(new ListObjectsV2Command({
+          Bucket: bucket,
+          Prefix: prefix,
+          ContinuationToken: continuationToken,
+        }));
+
+        for (const obj of response.Contents ?? []) {
+          this._listing.set(obj.Key, {
+            lastModified: obj.LastModified,
+            size: obj.Size,
+          });
+          count++;
+        }
+
+        continuationToken = response.IsTruncated
+          ? response.NextContinuationToken
+          : undefined;
+      } while (continuationToken);
+
+      this._console.info(`Loaded ${count} entries for prefix "${prefix}"`);
+    }
+  }
+
+  /**
+   * Check if a local file path is known to exist in the VFS listing.
+   * Returns true/false if the path is within a tracked prefix,
+   * or null if VFS is not available.
+   * @param {string} localPath
+   */
+  async has(localPath) {
+    if (!(await this._ensureLoaded())) return null;
+
+    const key = this._localPathToKey(localPath);
+    if (key === null) return null;
+    if (!this._isTrackedKey(key)) return null;
+
+    return this._listing.has(key);
+  }
+
+  /**
+   * Get the last modified time for a file from the S3 listing.
+   * Returns Date if found, null if not tracked or VFS not available.
+   * @param {string} localPath
+   */
+  async getMtime(localPath) {
+    if (!(await this._ensureLoaded())) return null;
+
+    const key = this._localPathToKey(localPath);
+    if (key === null) return null;
+    if (!this._isTrackedKey(key)) return null;
+
+    const entry = this._listing.get(key);
+    return entry ? entry.lastModified : null;
+  }
+
+  /**
+   * Convert a local path (e.g. 'dist/assets/splatnet/foo.png')
+   * to an S3 key (e.g. 'assets/splatnet/foo.png').
+   * @param {string} localPath
+   */
+  _localPathToKey(localPath) {
+    const prefix = this._localPrefix + '/';
+    if (localPath.startsWith(prefix)) {
+      return localPath.slice(prefix.length);
+    }
+    return null;
+  }
+
+  _isTrackedKey(key) {
+    return prefixes.some(prefix => key.startsWith(prefix));
+  }
+}
+
+const vfs = new VirtualFileSystem();
+export default vfs;

app/data/updaters/DataUpdater.mjs

@@ -89,13 +89,13 @@ export default class DataUpdater
     await this.updateLocalizations(this.defaultLocale, data);
 
     // Download any new images
-    const images = await this.downloadImages(data);
+    await this.downloadImages(data);
 
     // Write the data to disk
     await this.saveData(data);
 
     // Update iCal data
-    await this.updateCalendarEvents(data, images);
+    await this.updateCalendarEvents(data);
 
     this.console.info('Done');
   }
@@ -223,11 +223,11 @@ export default class DataUpdater
 
   // Calendar output
 
-  async updateCalendarEvents(data, images) {
+  async updateCalendarEvents(data) {
     const events = this.getCalendarEntries(data);
     if (!events) return;
 
-    const ical = await this.getiCalData(events, images);
+    const ical = await this.getiCalData(events);
     await this.writeFile(this.getCalendarPath(this.calendarFilename ?? this.filename), ical);
   }
 
@@ -239,7 +239,7 @@ export default class DataUpdater
     //
   }
 
-  async getiCalData(events, images) {
+  async getiCalData(events) {
     // Create a calendar object
     const calendar = ical({
       name: this.calendarName ?? this.name,
@@ -252,9 +252,6 @@ export default class DataUpdater
       timezone: 'UTC',
     });
 
-    // Create a map of image URLs to image data
-    const imageData = {};
-
     // Add event entries
     for (let event of events) {
       let calEvent = calendar.createEvent({
@@ -265,30 +262,11 @@ export default class DataUpdater
         url: event.url,
       });
       calEvent.createAttachment(event.imageUrl);
-
-      const filename = images[event.imageUrl];
-      if (filename) {
-        const data = await fs.readFile(this.imageProcessor.localPath(filename));
-        imageData[event.imageUrl] = data;
-      }
     }
 
     // Convert the calendar to an ICS string
     let ics = calendar.toString();
 
-    // Embed image attachments
-    ics = ics.replaceAll(/^ATTACH:((.|\r\n )*)$/gm, (match, url) => {
-      url = url.replaceAll('\r\n ', '');
-
-      const filename = images[url];
-      const data = imageData[url];
-      if (!filename || !data) return match;
-
-      const ical = `ATTACH;ENCODING=BASE64;VALUE=BINARY;X-APPLE-FILENAME=${path.basename(filename)}:${data.toString('base64')}`;
-
-      return ical.replace(/(.{72})/g, '$1\r\n ').trim();
-    });
-
     return ics;
   }
 }

app/sync/S3Syncer.mjs

@@ -10,7 +10,7 @@ export default class S3Syncer
 
     return Promise.all([
       dist && this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, {
-        filters: this.filters,
+        filters: this.downloadFilters,
       }),
       storage && this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`, {
         filters: this.privateFilters,
@@ -76,6 +76,20 @@ export default class S3Syncer
     ];
   }
 
+  // Download filters skip files that are handled by the VFS layer
+  // (existence/mtime checks only, no content reads needed)
+  get downloadFilters() {
+    return [
+      { exclude: () => true }, // Exclude everything by default
+      // assets/splatnet/ - handled by VFS (existence checks only)
+      // status-screenshots/ - regenerated by social posting
+      { include: (key) => key.startsWith('data/') },
+      { exclude: (key) => key.startsWith('data/archive/') },
+      { exclude: (key) => key.startsWith('data/xrank/') },
+      { exclude: (key) => key.startsWith('data/festivals.ranking.') },
+    ];
+  }
+
   get privateFilters() {
     return [
       { exclude: (key) => key.startsWith('archive/') },