Merge pull request #83 from misenhower/develop

Release

.env.example

@@ -13,9 +13,20 @@ USER_AGENT=
 # Sentry
 SENTRY_DSN=
 
+# Archive all data (can use a lot of disk space)
+ARCHIVE_DATA=false
+
+# Browserless (for screenshots)
+BROWSERLESS_ENDPOINT=ws://localhost:3000
+SCREENSHOT_HOST=host.docker.internal
+BROWSERLESS_CONCURRENT=2
+
 # S3 parameters
 AWS_S3_ENDPOINT=
+AWS_REGION=
 AWS_S3_BUCKET=
+AWS_S3_ARCHIVE_BUCKET=
+AWS_S3_PRIVATE_BUCKET=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 

.github/workflows/deploy.yml

@@ -0,0 +1,87 @@
+name: Deploy
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+
+jobs:
+  deploy-frontend:
+    runs-on: ubuntu-latest
+
+    environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Build
+      run: npm run build
+
+    - name: Deploy
+      uses: jakejarvis/s3-sync-action@master
+      with:
+        args: --acl public-read
+      env:
+        AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
+        AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_REGION: ${{ secrets.AWS_REGION }}
+        SOURCE_DIR: 'dist'
+
+  deploy-backend:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: ${{ github.repository }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Extract metadata
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+    - name: Build and push Docker image
+      id: push
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./docker/app/Dockerfile
+        push: true
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+
+    - name: Generate artifact attestation
+      uses: actions/attest-build-provenance@v1
+      with:
+        subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+        subject-digest: ${{ steps.push.outputs.digest }}
+        push-to-registry: true

app/data/DataArchiver.mjs

@@ -37,8 +37,8 @@ export default class DataArchiver
 
   get canRun() {
     return process.env.AWS_S3_ENDPOINT
-      && process.env.AWS_S3_REGION
-      && process.env.AWS_S3_BUCKET
+      && process.env.AWS_REGION
+      && process.env.AWS_S3_ARCHIVE_BUCKET
       && process.env.AWS_ACCESS_KEY_ID
       && process.env.AWS_SECRET_ACCESS_KEY;
   }
@@ -46,7 +46,7 @@ export default class DataArchiver
   get s3Client() {
     return this._client ??= new S3Client({
       endpoint: process.env.AWS_S3_ENDPOINT,
-      region: process.env.AWS_S3_REGION,
+      region: process.env.AWS_REGION,
       credentials: {
         accessKeyId: process.env.AWS_ACCESS_KEY_ID,
         secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
@@ -88,7 +88,7 @@ export default class DataArchiver
 
   async uploadViaS3(file, destination) {
     return this.s3Client.send(new PutObjectCommand({
-      Bucket: process.env.AWS_S3_BUCKET,
+      Bucket: process.env.AWS_S3_ARCHIVE_BUCKET,
       Key: destination,
       Body: await fs.readFile(file),
       ACL: 'public-read',

app/data/ImageProcessor.mjs

@@ -1,10 +1,14 @@
 import fs from 'fs/promises';
 import path from 'path';
 import mkdirp from 'mkdirp';
+// eslint-disable-next-line import/no-unresolved
+import PQueue from 'p-queue';
 import prefixedConsole from '../common/prefixedConsole.mjs';
 import { normalizeSplatnetResourcePath } from '../common/util.mjs';
 import { exists } from '../common/fs.mjs';
 
+const queue = new PQueue({ concurrency: 4 });
+
 export default class ImageProcessor
 {
   destinationDirectory = 'dist';
@@ -15,17 +19,27 @@ export default class ImageProcessor
     this.siteUrl = process.env.SITE_URL;
   }
 
-  async process(url) {
+  async process(url, defer = true) {
     // Normalize the path
     let destination = this.normalize(url);
 
     // Download the image if necessary
-    await this.maybeDownload(url, destination);
+    let job = () => this.maybeDownload(url, destination);
+    // defer ? queue.add(job) : await job();
+    if (defer) {
+      queue.add(job);
+    } else {
+      await job();
+    }
 
     // Return the new public URL
     return [destination, this.publicUrl(destination)];
   }
 
+  static onIdle() {
+    return queue.onIdle();
+  }
+
   normalize(url) {
     return normalizeSplatnetResourcePath(url);
   }
@@ -53,6 +67,10 @@ export default class ImageProcessor
     try {
       let result = await fetch(url);
 
+      if (!result.ok) {
+        throw new Error(`Invalid image response code: ${result.status}`);
+      }
+
       await mkdirp(path.dirname(this.localPath(destination)));
       await fs.writeFile(this.localPath(destination), result.body);
     } catch (e) {

app/data/LocalizationProcessor.mjs

@@ -4,6 +4,9 @@ import mkdirp from 'mkdirp';
 import jsonpath from 'jsonpath';
 import get from 'lodash/get.js';
 import set from 'lodash/set.js';
+import pLimit from 'p-limit';
+
+const limit = pLimit(1);
 
 function makeArray(value) {
   return Array.isArray(value) ? value : [value];
@@ -55,6 +58,12 @@ export class LocalizationProcessor {
   }
 
   async updateLocalizations(data) {
+    // We're reading, modifying, and writing back to the same file,
+    // so we have to make sure the whole operation is atomic.
+    return limit(() => this._updateLocalizations(data));
+  }
+
+  async _updateLocalizations(data) {
     let localizations = await this.readData();
 
     for (let { path, value } of this.dataIterations(data)) {

app/data/index.mjs

@@ -1,10 +1,13 @@
 import * as Sentry from '@sentry/node';
+import S3Syncer from '../sync/S3Syncer.mjs';
+import { canSync } from '../sync/index.mjs';
 import GearUpdater from './updaters/GearUpdater.mjs';
 import StageScheduleUpdater from './updaters/StageScheduleUpdater.mjs';
 import CoopUpdater from './updaters/CoopUpdater.mjs';
 import FestivalUpdater from './updaters/FestivalUpdater.mjs';
 import XRankUpdater from './updaters/XRankUpdater.mjs';
 import StagesUpdater from './updaters/StagesUpdater.mjs';
+import ImageProcessor from './ImageProcessor.mjs';
 
 function updaters() {
   return [
@@ -40,7 +43,7 @@ export async function update(config = 'default') {
 
   let settings = configs[config];
 
-  for (let updater of updaters()) {
+  await Promise.all(updaters().map(async updater => {
     updater.settings = settings;
     try {
       await updater.updateIfNeeded();
@@ -48,6 +51,11 @@ export async function update(config = 'default') {
       console.error(e);
       Sentry.captureException(e);
     }
+  }));
+
+  if (canSync()) {
+    await ImageProcessor.onIdle();
+    await (new S3Syncer).upload();
   }
 
   console.info(`Done running ${config} updaters`);

app/data/updaters/DataUpdater.mjs

@@ -4,6 +4,7 @@ import { Console } from 'node:console';
 import mkdirp from 'mkdirp';
 import jsonpath from 'jsonpath';
 import ical from 'ical-generator';
+import pFilter from 'p-filter';
 import prefixedConsole from '../../common/prefixedConsole.mjs';
 import SplatNet3Client from '../../splatnet/SplatNet3Client.mjs';
 import ImageProcessor from '../ImageProcessor.mjs';
@@ -11,7 +12,6 @@ import NsoClient from '../../splatnet/NsoClient.mjs';
 import { locales, regionalLocales, defaultLocale } from '../../../src/common/i18n.mjs';
 import { LocalizationProcessor } from '../LocalizationProcessor.mjs';
 import { deriveId, getDateParts, getTopOfCurrentHour } from '../../common/util.mjs';
-
 export default class DataUpdater
 {
   name = null;
@@ -70,7 +70,6 @@ export default class DataUpdater
 
   async updateIfNeeded() {
     if (!(await this.shouldUpdate())) {
-      this.console.info('No need to update data');
       return;
     }
 
@@ -135,16 +134,18 @@ export default class DataUpdater
     }
 
     // Retrieve data for missing languages
-    for (let locale of this.locales.filter(l => l !== initialLocale)) {
-      processor = new LocalizationProcessor(locale, this.localizations);
+    let processors = this.locales.filter(l => l !== initialLocale)
+      .map(l => new LocalizationProcessor(l, this.localizations));
+    let missing = await pFilter(processors, p => p.hasMissingLocalizations(data));
 
-      if (await processor.hasMissingLocalizations(data)) {
-        this.console.info(`Retrieving localized data for ${locale.code}`);
-
-        let regionalData = await this.getData(locale);
+    if (missing.length > 0) {
+      await Promise.all(missing.map(async (processor) => {
+        let regionalData = await this.getData(processor.locale);
         this.deriveIds(regionalData);
         await processor.updateLocalizations(regionalData);
-      }
+
+        this.console.info(`Retrieved localized data for: ${processor.locale.code}`);
+      }));
     }
   }
 
@@ -166,6 +167,8 @@ export default class DataUpdater
       jsonpath.apply(data, expression, url => mapping[url]);
     }
 
+    await ImageProcessor.onIdle();
+
     return images;
   }
 
@@ -177,7 +180,9 @@ export default class DataUpdater
     await this.writeFile(this.getPath(this.filename), s);
 
     // Write a secondary file for archival
-    await this.writeFile(this.getArchivePath(this.filename), s);
+    if (process.env.ARCHIVE_DATA) {
+      await this.writeFile(this.getArchivePath(this.filename), s);
+    }
   }
 
   getPath(filename) {

app/data/updaters/FestivalRankingUpdater.mjs

@@ -43,7 +43,7 @@ export default class FestivalRankingUpdater extends DataUpdater
   async getData(locale) {
     const data = await this.splatnet(locale).getFestRankingData(this.festID);
 
-    for (const team of data.data.fest.teams) {
+    await Promise.all(data.data.fest.teams.map(async (team) => {
       let pageInfo = team.result?.rankingHolders?.pageInfo;
 
       while (pageInfo?.hasNextPage) {
@@ -62,7 +62,7 @@ export default class FestivalRankingUpdater extends DataUpdater
 
         pageInfo = page.data.node.result.rankingHolders.pageInfo;
       }
-    }
+    }));
 
     return data;
   }

app/data/updaters/FestivalUpdater.mjs

@@ -1,11 +1,14 @@
 import fs from 'fs/promises';
 import jsonpath from 'jsonpath';
+import pLimit from 'p-limit';
 import { getFestId } from '../../common/util.mjs';
 import ValueCache from '../../common/ValueCache.mjs';
 import { regionTokens } from '../../splatnet/NsoClient.mjs';
 import FestivalRankingUpdater from './FestivalRankingUpdater.mjs';
 import DataUpdater from './DataUpdater.mjs';
 
+const limit = pLimit(1);
+
 function generateFestUrl(id) {
   return process.env.DEBUG ?
     `https://s.nintendo.com/av5ja-lp1/znca/game/4834290508791808?p=/fest_record/${id}` :
@@ -79,7 +82,7 @@ export default class FestivalUpdater extends DataUpdater
     // Get the detailed data for each Splatfest
     // (unless we're getting localization-specific data)
     if (locale === this.defaultLocale) {
-      for (let node of result.data.festRecords.nodes) {
+      await Promise.all(result.data.festRecords.nodes.map(async node => {
         let detailResult = await this.getFestivalDetails(node);
 
         Object.assign(node, detailResult.data.fest);
@@ -93,7 +96,7 @@ export default class FestivalUpdater extends DataUpdater
             this.console.error(e);
           }
         }
-      }
+      }));
     }
 
     return result;
@@ -128,7 +131,11 @@ export default class FestivalUpdater extends DataUpdater
     return data;
   }
 
-  async formatDataForWrite(data) {
+  formatDataForWrite(data) {
+    return limit(() => this._formatDataForWrite(data));
+  }
+
+  async _formatDataForWrite(data) {
     // Combine this region's data with the other regions' data.
     let result = null;
     try {

app/data/updaters/XRankUpdater.mjs

@@ -46,10 +46,9 @@ export default class XRankUpdater extends DataUpdater
     let result = await this.splatnet(locale).getXRankingData(this.divisionKey);
     let seasons = this.getSeasons(result.data);
 
-    for (let season of seasons) {
-      this.deriveSeasonId(season);
-      await this.updateSeasonDetail(season);
-    }
+    seasons.forEach(s => this.deriveSeasonId(s));
+
+    await Promise.all(seasons.map(season => this.updateSeasonDetail(season)));
 
     return result;
   }
@@ -66,9 +65,9 @@ export default class XRankUpdater extends DataUpdater
   }
 
   async updateSeasonDetail(season) {
-    for (let type of this.splatnet().getXRankingDetailQueryTypes()) {
+    await Promise.all(this.splatnet().getXRankingDetailQueryTypes().map(type => {
       let updater = new XRankDetailUpdater(season.id, season.endTime, type);
-      await updater.updateIfNeeded();
-    }
+      return updater.updateIfNeeded();
+    }));
   }
 }

app/index.mjs

@@ -10,6 +10,7 @@ import BlueskyClient from './social/clients/BlueskyClient.mjs';
 import ThreadsClient from './social/clients/ThreadsClient.mjs';
 import { archiveData } from './data/DataArchiver.mjs';
 import { sentryInit } from './common/sentry.mjs';
+import { sync, syncUpload, syncDownload } from './sync/index.mjs';
 
 consoleStamp(console);
 dotenv.config();
@@ -26,6 +27,9 @@ const actions = {
   splatnet: update,
   warmCaches,
   dataArchive: archiveData,
+  sync,
+  syncUpload,
+  syncDownload,
 };
 
 const command = process.argv[2];

app/screenshots/ScreenshotHelper.mjs

@@ -1,5 +1,5 @@
 import { URL } from 'url';
-import puppeteer from 'puppeteer';
+import puppeteer from 'puppeteer-core';
 import HttpServer from './HttpServer.mjs';
 
 const defaultViewport = {
@@ -36,12 +36,9 @@ export default class ScreenshotHelper
     this.#httpServer = new HttpServer;
     await this.#httpServer.open();
 
-    // Launch a new Chrome instance
-    this.#browser = await puppeteer.launch({
-      args: [
-        '--no-sandbox', // Allow running as root inside the Docker container
-      ],
-      // headless: false, // For testing
+    // Connect to Browserless
+    this.#browser = await puppeteer.connect({
+      browserWSEndpoint: process.env.BROWSERLESS_ENDPOINT,
     });
 
     // Create a new page and set the viewport
@@ -66,7 +63,8 @@ export default class ScreenshotHelper
     await this.applyViewport(options.viewport);
 
     // Navigate to the URL
-    let url = new URL(`http://localhost:${this.#httpServer.port}/screenshots/`);
+    let host = process.env.SCREENSHOT_HOST || 'localhost';
+    let url = new URL(`http://${host}:${this.#httpServer.port}/screenshots/`);
     url.hash = path;
 
     let params = {

app/social/StatusGeneratorManager.mjs

@@ -12,9 +12,6 @@ export default class StatusGeneratorManager
   /** @type {Client[]} */
   clients;
 
-  /** @type {ScreenshotHelper} */
-  screenshotHelper;
-
   console(generator = null, client = null) {
     let prefixes = ['Social', generator?.name, client?.name].filter(s => s);
     return prefixedConsole(...prefixes);
@@ -23,45 +20,82 @@ export default class StatusGeneratorManager
   constructor(generators = [], clients = []) {
     this.generators = generators;
     this.clients = clients;
-    this.screenshotHelper = new ScreenshotHelper;
   }
 
   async sendStatuses(force = false) {
-    for (let generator of this.generators) {
-      try {
-        await this.#generateAndSend(generator, force);
-      } catch (e) {
-        this.console(generator).error(`Error generating status: ${e}`);
-        Sentry.captureException(e);
-      }
-    }
+    let availableClients = await this.#getAvailableClients();
 
-    await this.screenshotHelper.close();
+    // Create screenshots in parallel (via Browserless)
+    let statusPromises = this.#getStatuses(availableClients, force);
+
+    // Process each client in parallel (while maintaining post order)
+    await this.#sendStatusesToClients(statusPromises, availableClients);
   }
 
-  async #generateAndSend(generator, force) {
-    let clientsToPost = [];
+  async #getAvailableClients() {
+    let clients = [];
 
     for (let client of this.clients) {
       if (!(await client.canSend())) {
-        this.console(generator, client).warn('Client cannot send (missing credentials)');
+        this.console(client).warn('Client cannot send (missing credentials)');
         continue;
       }
 
-      if (force || await generator.shouldPost(client)) {
-        clientsToPost.push(client);
+      clients.push(client);
+    }
+
+    return clients;
+  }
+
+  #getStatuses(availableClients, force) {
+    return this.generators.map(generator => this.#getStatus(availableClients, generator, force));
+  }
+
+  async #getStatus(availableClients, generator, force) {
+    let screenshotHelper = new ScreenshotHelper;
+    try {
+      let clients = [];
+
+      for (let client of availableClients) {
+        if (force || await generator.shouldPost(client)) {
+          clients.push(client);
+        }
+      }
+
+      if (clients.length === 0) {
+        this.console(generator).info('No status to post, skipping');
+
+        return null;
+      }
+
+      await screenshotHelper.open();
+      let status = await generator.getStatus(screenshotHelper);
+
+      return { generator, status, clients };
+    } catch (e) {
+      this.console(generator).error(`Error generating status: ${e}`);
+      Sentry.captureException(e);
+    } finally {
+      await screenshotHelper.close();
+    }
+
+    return null;
+  }
+
+  #sendStatusesToClients(statusPromises, availableClients) {
+    return Promise.allSettled(availableClients.map(client => this.#sendStatusesToClient(statusPromises, client)));
+  }
+
+  async #sendStatusesToClient(statusPromises, client) {
+    for (let promise of statusPromises) {
+      let statusDetails = await promise;
+
+      if (statusDetails && statusDetails.clients.includes(client)) {
+        let { generator, status } = statusDetails;
+
+        await this.#sendToClient(generator, status, client);
       }
     }
-
-    if (clientsToPost.length === 0) {
-      this.console(generator).info('No status to post, skipping');
-
-      return;
-    }
-
-    let status = await generator.getStatus(this.screenshotHelper);
-
-    await Promise.all(clientsToPost.map(client => this.#sendToClient(generator, status, client)));
   }
 
   async #sendToClient(generator, status, client) {

app/social/clients/TwitterClient.mjs

@@ -33,7 +33,7 @@ export default class TwitterClient extends Client
     // Upload images
     let mediaIds = await Promise.all(
       status.media.map(async m => {
-        let id = await this.api().v1.uploadMedia(m.file, { mimeType: m.type });
+        let id = await this.api().v1.uploadMedia(Buffer.from(m.file), { mimeType: m.type });
 
         if (m.altText) {
           await this.api().v1.createMediaMetadata(id, { alt_text: { text: m.altText } });

app/social/generators/ChallengeStatus.mjs

@@ -13,7 +13,7 @@ export default class ChallengeStatus extends StatusGenerator
 
     let schedule = useEventSchedulesStore().activeSchedule;
 
-    if (schedule.activeTimePeriod) {
+    if (schedule?.activeTimePeriod) {
       return schedule;
     }
   }

app/social/generators/EggstraWorkStatus.mjs

@@ -17,7 +17,7 @@ export default class EggstraWorkStatus extends StatusGenerator
   async getDataTime() {
     let schedule = await this.getActiveSchedule();
 
-    return Date.parse(schedule.startTime);
+    return Date.parse(schedule?.startTime);
   }
 
   async _getStatus() {

app/social/index.mjs

@@ -1,3 +1,5 @@
+import S3Syncer from '../sync/S3Syncer.mjs';
+import { canSync } from '../sync/index.mjs';
 import FileWriter from './clients/FileWriter.mjs';
 import ImageWriter from './clients/ImageWriter.mjs';
 import MastodonClient from './clients/MastodonClient.mjs';
@@ -63,8 +65,12 @@ export function testStatusGeneratorManager(additionalClients) {
   );
 }
 
-export function sendStatuses() {
-  return defaultStatusGeneratorManager().sendStatuses();
+export async function sendStatuses() {
+  await defaultStatusGeneratorManager().sendStatuses();
+
+  if (canSync()) {
+    await (new S3Syncer).upload();
+  }
 }
 
 export function testStatuses(additionalClients = []) {

app/splatnet/NsoClient.mjs

@@ -1,9 +1,13 @@
 // eslint-disable-next-line import/no-unresolved
 import CoralApi from 'nxapi/coral';
 import { addUserAgent } from 'nxapi';
+import pLimit from 'p-limit';
 import ValueCache from '../common/ValueCache.mjs';
 import prefixedConsole from '../common/prefixedConsole.mjs';
 
+const coralLimit = pLimit(1);
+const webServiceLimit = pLimit(1);
+
 let _nxapiInitialized = false;
 
 function initializeNxapi() {
@@ -72,15 +76,17 @@ export default class NsoClient
   }
 
   async getCoralApi(useCache = true) {
-    let data = useCache
-      ? await this._getCoralCache().getData()
-      : null;
+    return coralLimit(async () => {
+      let data = useCache
+        ? await this._getCoralCache().getData()
+        : null;
 
-    if (!data) {
-      data = await this._createCoralSession();
-    }
+      if (!data) {
+        data = await this._createCoralSession();
+      }
 
-    return CoralApi.createWithSavedToken(data);
+      return CoralApi.createWithSavedToken(data);
+    });
   }
 
   async _createCoralSession() {
@@ -101,16 +107,18 @@ export default class NsoClient
   }
 
   async getWebServiceToken(id, useCache = true) {
-    let tokenCache = this._getWebServiceTokenCache(id);
-    let token = useCache
-      ? await tokenCache.getData()
-      : null;
+    return webServiceLimit(async () => {
+      let tokenCache = this._getWebServiceTokenCache(id);
+      let token = useCache
+        ? await tokenCache.getData()
+        : null;
 
-    if (!token) {
-      token = await this._createWebServiceToken(id, tokenCache);
-    }
+      if (!token) {
+        token = await this._createWebServiceToken(id, tokenCache);
+      }
 
-    return token.accessToken;
+      return token.accessToken;
+    });
   }
 
   async _createWebServiceToken(id, tokenCache) {

app/splatnet/SplatNet3Client.mjs

@@ -1,9 +1,13 @@
 import fs from 'fs/promises';
+import pLimit from 'p-limit';
 import ValueCache from '../common/ValueCache.mjs';
 import prefixedConsole from '../common/prefixedConsole.mjs';
 
 export const SPLATNET3_WEB_SERVICE_ID = '4834290508791808';
 
+// Concurrent request limit
+const limit = pLimit(5);
+
 export default class SplatNet3Client
 {
   baseUrl = 'https://api.lp1.av5ja.srv.nintendo.net';
@@ -108,17 +112,18 @@ export default class SplatNet3Client
 
   async getGraphQL(body = {}) {
     await this._maybeStartSession();
+    let webViewVersion = await this._webViewVersion();
 
-    let response = await fetch(this.baseUrl + '/api/graphql', {
+    let response = await limit(() => fetch(this.baseUrl + '/api/graphql', {
       method: 'POST',
       headers: {
         'Authorization': `Bearer ${this.bulletToken.bulletToken}`,
-        'X-Web-View-Ver': await this._webViewVersion(),
+        'X-Web-View-Ver': webViewVersion,
         'Content-Type': 'application/json',
         'Accept-Language': this.acceptLanguage,
       },
       body: JSON.stringify(body),
-    });
+    }));
 
     if (!response.ok) {
       throw new Error(`Invalid GraphQL response code: ${response.status}`);

app/sync/S3Syncer.mjs

@@ -0,0 +1,88 @@
+import path  from 'path';
+import { S3Client }  from '@aws-sdk/client-s3';
+import { S3SyncClient }  from 's3-sync-client';
+import mime  from 'mime-types';
+
+export default class S3Syncer
+{
+  download() {
+    this.log('Downloading files...');
+
+    return Promise.all([
+      this.syncClient.sync(this.publicBucket, `${this.localPath}/dist`, {
+        filters: this.filters,
+      }),
+      this.syncClient.sync(this.privateBucket, `${this.localPath}/storage`, {
+        filters: this.privateFilters,
+      }),
+    ]);
+  }
+
+  upload() {
+    this.log('Uploading files...');
+
+    return Promise.all([
+      this.syncClient.sync(`${this.localPath}/dist`, this.publicBucket, {
+        filters: this.filters,
+        commandInput: input => ({
+          ACL: 'public-read',
+          ContentType: mime.lookup(input.Key),
+          CacheControl: input.Key.startsWith('data/')
+            ? 'no-cache, stale-while-revalidate=5, stale-if-error=86400'
+            : undefined,
+        }),
+      }),
+      this.syncClient.sync(`${this.localPath}/storage`, this.privateBucket, {
+        filters: this.privateFilters,
+      }),
+    ]);
+  }
+
+  get s3Client() {
+    return this._s3Client ??= new S3Client({
+      endpoint: process.env.AWS_S3_ENDPOINT,
+      region: process.env.AWS_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+      },
+    });
+  }
+
+  /** @member {S3SyncClient} */
+  get syncClient() {
+    return this._syncClient ??= new S3SyncClient({ client: this.s3Client });
+  }
+
+  get publicBucket() {
+    return `s3://${process.env.AWS_S3_BUCKET}`;
+  }
+
+  get privateBucket() {
+    return `s3://${process.env.AWS_S3_PRIVATE_BUCKET}`;
+  }
+
+  get localPath() {
+    return path.resolve('.');
+  }
+
+  get filters() {
+    return [
+      { exclude: () => true }, // Exclude everything by default
+      { include: (key) => key.startsWith('assets/splatnet/') },
+      { include: (key) => key.startsWith('data/') },
+      { exclude: (key) => key.startsWith('data/archive/') },
+      { include: (key) => key.startsWith('status-screenshots/') },
+    ];
+  }
+
+  get privateFilters() {
+    return [
+      { exclude: (key) => key.startsWith('archive/') },
+    ];
+  }
+
+  log(message) {
+    console.log(`[S3] ${message}`);
+  }
+}

app/sync/index.mjs

@@ -0,0 +1,41 @@
+import S3Syncer from './S3Syncer.mjs';
+
+export function canSync() {
+  return !!(
+    process.env.AWS_ACCESS_KEY_ID &&
+    process.env.AWS_SECRET_ACCESS_KEY &&
+    process.env.AWS_S3_BUCKET &&
+    process.env.AWS_S3_PRIVATE_BUCKET
+  );
+}
+
+async function doSync(download, upload) {
+  if (!canSync()) {
+    console.warn('Missing S3 connection parameters');
+    return;
+  }
+
+  const syncer = new S3Syncer();
+
+  if (download) {
+    console.info('Downloading files...');
+    await syncer.download();
+  }
+
+  if (upload) {
+    console.info('Uploading files...');
+    await syncer.upload();
+  }
+}
+
+export function sync() {
+  return doSync(true, true);
+}
+
+export function syncUpload() {
+  return doSync(false, true);
+}
+
+export function syncDownload() {
+  return doSync(true, false);
+}

docker-compose.override.yml.dev.example

@@ -0,0 +1,21 @@
+# Example Docker Compose override file for local development
+
+services:
+  app:
+    platform: linux/amd64 # Needed for Apple Silicon
+    build:
+      dockerfile: docker/app/Dockerfile
+    init: true
+    restart: unless-stopped
+    environment:
+      BROWSERLESS_ENDPOINT: ws://browserless:3000
+      SCREENSHOT_HOST: app
+    depends_on:
+      - browserless
+    volumes:
+      - .:/app
+
+  browserless:
+    platform: linux/arm64 # Needed for Apple Silicon
+    ports:
+      - 3000:3000

docker-compose.override.yml.example

@@ -1,22 +0,0 @@
-version: "3.8"
-
-services:
-  app:
-    # This may be necessary on M1 Macs:
-    #platform: linux/amd64
-
-  nginx:
-    #ports:
-    #  - "127.0.0.1:8888:80"
-
-    environment:
-      VIRTUAL_HOST: splatoon3.ink,www.splatoon3.ink
-
-    networks:
-      - default
-      - nginx-proxy
-
-networks:
-  nginx-proxy:
-    external:
-      name: nginxproxy_default

docker-compose.override.yml.prod.example

@@ -0,0 +1,25 @@
+# Example Docker Compose override file for production
+
+services:
+  app:
+    image: ghcr.io/misenhower/splatoon3.ink:main
+    init: true
+    restart: unless-stopped
+    environment:
+      BROWSERLESS_ENDPOINT: ws://browserless:3000
+      SCREENSHOT_HOST: app
+    depends_on:
+      - browserless
+    env_file:
+      - .env
+    labels: [ "com.centurylinklabs.watchtower.scope=splatoon3ink" ]
+
+  browserless:
+    labels: [ "com.centurylinklabs.watchtower.scope=splatoon3ink" ]
+
+  watchtower:
+    image: containrrr/watchtower
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: --interval 30 --scope splatoon3ink
+    labels: [ "com.centurylinklabs.watchtower.scope=splatoon3ink" ]

docker-compose.yml

@@ -1,18 +1,9 @@
-version: "3.8"
+# See docker-compose.override.yml.* example files for dev/prod environments
 
 services:
-  app:
-    build: docker/app
-    init: true
+  browserless:
+    image: ghcr.io/browserless/chromium
     restart: unless-stopped
-    working_dir: /app
-    volumes:
-      - ./:/app
-    command: npm run cron
-
-  nginx:
-    image: nginx
-    restart: unless-stopped
-    volumes:
-      - ./docker/nginx/conf.d:/etc/nginx/conf.d:ro
-      - ./dist:/usr/share/nginx/html:ro
+    environment:
+      CONCURRENT: ${BROWSERLESS_CONCURRENT:-1}
+      QUEUED: ${BROWSERLESS_QUEUED:-100}

docker/app/Dockerfile

@@ -1,12 +1,15 @@
 FROM node:20
 
-# Puppeteer support
-# Adapted from: https://github.com/puppeteer/puppeteer/blob/2d50ec5b384f2ae8eb02a534843caceca9f58ffe/docker/Dockerfile
-RUN apt-get update \
-    && apt-get install -y wget gnupg \
-    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
-    && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
-    && apt-get update \
-    && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-freefont-ttf libxss1 \
-      --no-install-recommends \
-    && rm -rf /var/lib/apt/lists/*
+# App setup
+WORKDIR /app
+ENV PUPPETEER_SKIP_DOWNLOAD=true
+
+# Install NPM dependencies
+COPY package*.json ./
+RUN npm ci
+
+# Copy app files and build
+COPY . .
+RUN npm run build
+
+CMD ["npm", "run", "start"]

package.json

@@ -8,6 +8,7 @@
     "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs --ignore-path .gitignore",
     "lint-fix": "npm run lint -- --fix",
     "cron": "node app/index.mjs cron",
+    "start": "npm run sync:download && npm run splatnet:quick && npm run social && npm run cron",
     "social": "node app/index.mjs social",
     "social:test": "node app/index.mjs socialTest",
     "social:test:image": "node app/index.mjs socialTestImage",
@@ -18,7 +19,10 @@
     "splatnet": "node app/index.mjs splatnet default",
     "splatnet:all": "node app/index.mjs splatnet all",
     "warmCaches": "node app/index.mjs warmCaches",
-    "data:archive": "node app/index.mjs dataArchive"
+    "data:archive": "node app/index.mjs dataArchive",
+    "sync": "node app/index.mjs sync",
+    "sync:upload": "node app/index.mjs syncUpload",
+    "sync:download": "node app/index.mjs syncDownload"
   },
   "dependencies": {
     "@atproto/api": "^0.11.2",
@@ -37,8 +41,12 @@
     "masto": "^6.7.0",
     "mkdirp": "^1.0.4",
     "nxapi": "^1.4.0",
+    "p-filter": "^4.1.0",
+    "p-limit": "^6.1.0",
+    "p-queue": "^8.0.1",
     "pinia": "^2.0.22",
-    "puppeteer": "^18.0.3",
+    "puppeteer-core": "^23.8.0",
+    "s3-sync-client": "^4.3.1",
     "sharp": "^0.32.0",
     "threads-api": "^1.4.0",
     "twitter-api-v2": "^1.12.7",

public/data/index.html

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Splatoon3.ink Data</title>
+</head>
+
+<body>
+  <div id="navigation"></div>
+  <div id="listing"></div>
+  <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
+  <script type="text/javascript">
+    var S3BL_IGNORE_PATH = true;
+    var S3B_SORT = 'A2Z';
+    var BUCKET_URL = 'https://s3-data.splatoon3.ink';
+    var S3B_ROOT_DIR = 'data';
+    var BUCKET_WEBSITE_URL = 'https://splatoon3.ink';
+  </script>
+  <script type="text/javascript" src="https://rufuspollock.github.io/s3-bucket-listing/list.js"></script>
+</body>
+
+</html>

src/common/time.js

@@ -51,13 +51,18 @@ export function formatShortDuration(value) {
   const { t } = useI18n();
   let { negative, days, hours, minutes, seconds } = getDurationParts(value);
 
+  days = days && t('time.days', days);
+  hours = hours && t('time.hours', hours);
+  minutes = minutes && t('time.minutes', { n: minutes }, minutes);
+  seconds = t('time.seconds', { n: seconds }, seconds);
+
   if (days)
-    return t('time.days', { n: `${negative}${days}` }, days);
+    return hours ? `${negative}${days} ${hours}` : `${negative}${days}`;
   if (hours)
-    return t('time.hours', { n: `${negative}${hours}` }, hours);
+    return `${negative}${hours}`;
   if (minutes)
-    return t('time.minutes', { n: `${negative}${minutes}` }, minutes);
-  return t('time.seconds', { n: `${negative}${seconds}` }, seconds);
+    return `${negative}${minutes}`;
+  return `${negative}${seconds}`;
 }
 
 export function formatShortDurationFromNow(value) {

src/components/gear/GearCard.vue

@@ -72,7 +72,7 @@
       </div>
 
       <div class="inline-block text-xs bg-zinc-200 bg-opacity-30 rounded px-1 py-px font-semibold">
-        {{ $t('time.left', { time: formatShortDurationFromNow(props.gear.saleEndTime) }) }}
+        {{ $t('time.left', { time: formatDurationHoursFromNow(props.gear.saleEndTime) }) }}
       </div>
     </div>
   </div>
@@ -81,7 +81,7 @@
 <script setup>
 import { computed } from 'vue';
 import SquidTape from '@/components/SquidTape.vue';
-import { formatShortDurationFromNow } from '@/common/time';
+import { formatDurationHoursFromNow } from '@/common/time';
 import { getGesotownGearUrl } from '@/common/links';
 
 const props = defineProps({

src/components/gear/GearCardHorizontal.vue

@@ -9,7 +9,7 @@
     <div class="grow min-w-0 flex flex-col justify-evenly space-y-2">
       <div class="flex">
         <div class="inline-block text-xs bg-zinc-200 bg-opacity-30 rounded px-1 py-px font-semibold">
-          {{ $t('time.left', { time: formatShortDurationFromNow(props.gear.saleEndTime) }) }}
+          {{ $t('time.left', { time: formatDurationHoursFromNow(props.gear.saleEndTime) }) }}
         </div>
       </div>
 
@@ -95,7 +95,7 @@
 <script setup>
 import { computed } from 'vue';
 import SquidTape from '@/components/SquidTape.vue';
-import { formatShortDurationFromNow } from '@/common/time';
+import { formatDurationHoursFromNow } from '@/common/time';
 import { getGesotownGearUrl } from '@/common/links';
 
 const props = defineProps({

src/components/salmonrun/ExpandedSalmonRunRow.vue

@@ -14,7 +14,7 @@
 
         <div v-if="time.isUpcoming(schedule.startTime)" class="inline-block">
           {{ $t('salmonrun.opens') }}
-          {{ $t('time.in', { time: formatDurationHoursFromNow(schedule.startTime, true) }) }}
+          {{ $t('time.in', { time: formatShortDurationFromNow(schedule.startTime, true, true) }) }}
         </div>
         <div v-else class="inline-block">
           {{ $t('time.remaining', { time: formatDurationHoursFromNow(schedule.endTime) }) }}
@@ -57,7 +57,7 @@ import KingSalmonid from './KingSalmonid.vue';
 import SalmonRunWeapons from './SalmonRunWeapons.vue';
 import { useTimeStore } from '@/stores/time.mjs';
 import StageImage from '@/components/StageImage.vue';
-import { formatDurationFromNow, formatDurationHoursFromNow } from '@/common/time';
+import { formatDurationFromNow, formatDurationHoursFromNow, formatShortDurationFromNow } from '@/common/time';
 
 defineProps({
   schedule: Object,

src/stores/schedules.mjs

@@ -111,7 +111,7 @@ export const useSalmonRunSchedulesStore = defineScheduleStore('salmonRun', {
       .map(n => ({
         ...n,
         isMystery: n.setting.weapons.some(w => w.name === 'Random'),
-        isGrizzcoMystery: n.setting.weapons.some(w => w.__splatoon3ink_id === '6e17fbe20efecca9'),
+        isGrizzcoMystery: n.setting.weapons.some(w => ['6e17fbe20efecca9', '747937841598fff7'].includes(w.__splatoon3ink_id)),
       }));
 
     return sortBy(nodes, 'startTime');

src/views/SocialsView.vue

@@ -26,6 +26,13 @@
                     </ul>
                   </div>
 
+                  <div>
+                    You can also follow us on Twitter (now known as &ldquo;X&rdquo;) at
+                    <a href="https://twitter.com/splatoon3ink" target="_blank">@splatoon3ink</a>,
+                    but due to new restrictions on that platform the bot is no longer able to reliably post there.
+                    Use any of the other platforms above to continue getting updates!
+                  </div>
+
                   <div>
                     <router-link to="/">
                       &laquo; {{ $t('about.schedules') }}
@@ -60,11 +67,11 @@ import MainLayout from '@/layouts/MainLayout.vue';
 import ProductContainer from '@/components/ProductContainer.vue';
 
 const socials = [
-  {
-    title: 'Twitter',
-    handle: '@splatoon3ink',
-    url: 'https://twitter.com/splatoon3ink',
-  },
+  // {
+  //   title: 'Twitter',
+  //   handle: '@splatoon3ink',
+  //   url: 'https://twitter.com/splatoon3ink',
+  // },
   {
     title: 'Threads',
     handle: '@splatoon3ink',
@@ -77,7 +84,7 @@ const socials = [
   },
   {
     title: 'Bluesky',
-    handle: '@splatoon3ink',
+    handle: '@splatoon3.ink',
     url: 'https://bsky.app/profile/splatoon3.ink',
   },
 ];