Merge pull request #115 from misenhower/develop

Add Cloudflare screenshot rendering and data origin config
This commit is contained in:
Matt Isenhower
2026-08-30 10:10:05 -07:00
committed by GitHub
18 changed files with 549 additions and 86 deletions

View File

@@ -20,11 +20,18 @@ SENTRY_DSN=
# Archive all data (can use a lot of disk space)
ARCHIVE_DATA=false
# Browserless (for screenshots)
# Screenshots (choose cloudflare or browserless explicitly)
SCREENSHOT_PROVIDER=browserless
# Browserless (local development)
BROWSERLESS_ENDPOINT=ws://localhost:3000
SCREENSHOT_HOST=host.docker.internal
BROWSERLESS_CONCURRENT=2
# Cloudflare Browser Run Quick Actions (production)
CLOUDFLARE_ACCOUNT_ID=
CLOUDFLARE_BROWSER_RUN_API_TOKEN=
# S3 parameters
AWS_S3_ENDPOINT=
AWS_REGION=

View File

@@ -26,6 +26,8 @@ jobs:
- name: Build
run: npm run build
env:
VITE_DATA_FROM: ${{ vars.VITE_DATA_FROM }}
- name: Deploy
uses: jakejarvis/s3-sync-action@master

View File

@@ -35,7 +35,7 @@ npm start # Full production: sync → splatnet → social → cron
**Data pipeline**: `NsoClient` (Nintendo auth) → `SplatNet3Client` (GraphQL queries) → DataUpdaters (`app/data/updaters/`) → JSON files in `dist/data/` → Frontend Pinia stores → Vue components. Images are processed via sharp. Data is optionally archived and synced to S3.
**Social media**: StatusGenerators (`app/social/generators/`) create content from data. Clients (`app/social/clients/`) post to each platform. Screenshots are generated via puppeteer-core + a browserless service.
**Social media**: StatusGenerators (`app/social/generators/`) create content from data. Clients (`app/social/clients/`) post to each platform. `ScreenshotHelper` delegates rendering to drivers in `app/screenshots/drivers/`: production uses Cloudflare Browser Run Quick Actions against the public site, while local development can use puppeteer-core + Browserless.
**Scheduling**: Cron jobs (`app/cron.mjs`) run data updates and social posting at intervals.
@@ -56,4 +56,4 @@ Tests use Vitest. Test files live alongside source: `app/**/*.test.mjs` and `src
- Frontend: built to `dist/` and deployed to AWS S3 (static hosting)
- Backend: Docker container (`docker/app/Dockerfile`) pushed to GitHub Container Registry
- `dist/` is not emptied on build (preserves generated `dist/data/` from backend)
- Browserless runs as a separate Docker service for screenshot generation
- Browserless runs as a separate Docker service for local screenshot generation

View File

@@ -1,6 +1,4 @@
import { URL } from 'url';
import puppeteer from 'puppeteer-core';
import HttpServer from './HttpServer.mjs';
import createScreenshotDriver from './drivers/createScreenshotDriver.mjs';
const defaultViewport = {
// Using a 16:9 ratio here by default to match Twitter's image card dimensions
@@ -11,48 +9,29 @@ const defaultViewport = {
export default class ScreenshotHelper
{
/** @type {HttpServer} */
_httpServer = null;
/** @type {puppeteer.Browser} */
_browser = null;
/** @type {puppeteer.Page} */
_page = null;
_driver = null;
_driverDependencies;
_isOpen = false;
defaultParams = null;
get isOpen() {
return !!this._browser;
constructor(driverDependencies = {}) {
this._driverDependencies = driverDependencies;
}
/** @type {puppeteer.Page} */
get page() {
return this._page;
get isOpen() {
return this._isOpen;
}
async open() {
await this.close();
// Start the HTTP server
this._httpServer = new HttpServer;
await this._httpServer.open();
// Connect to Browserless
this._browser = await puppeteer.connect({
browserWSEndpoint: process.env.BROWSERLESS_ENDPOINT,
});
// Create a new page and set the viewport
this._page = await this._browser.newPage();
await this.applyViewport();
}
async applyViewport(viewport = {}) {
if (this._page) {
await this._page.setViewport({
...defaultViewport,
...viewport,
});
}
this._driver = createScreenshotDriver(
process.env.SCREENSHOT_PROVIDER,
this._driverDependencies,
);
await this._driver.open();
this._isOpen = true;
}
async capture(path, options = {}) {
@@ -60,51 +39,32 @@ export default class ScreenshotHelper
await this.open();
}
await this.applyViewport(options.viewport);
// Navigate to the URL
let host = process.env.SCREENSHOT_HOST || 'localhost';
let url = new URL(`http://${host}:${this._httpServer.port}/screenshots/`);
url.hash = path;
let params = {
...this.defaultParams,
...options.params,
};
let route = path;
if (params) {
// We can't use url.searchParams because they need to come after the hash
url.hash += '?';
url.hash += Object.keys(params)
if (Object.keys(params).length) {
route += '?';
route += Object.keys(params)
.map(key => `${key}=${params[key]}`)
.join('&');
}
await this._page.goto(url, {
waitUntil: 'networkidle0', // Wait until the network is idle
});
let viewport = {
...defaultViewport,
...options.viewport,
};
// Wait an additional 1000ms
await this._page.waitForNetworkIdle({ idleTime: 1000 });
// Take the screenshot
return await this._page.screenshot();
return await this._driver.capture(route, viewport);
}
async close() {
if (this._httpServer) {
await this._httpServer.close();
if (this._driver) {
await this._driver.close();
}
this._httpServer = null;
if (this._page) {
await this._page.close();
}
this._page = null;
if (this._browser) {
await this._browser.close();
}
this._browser = null;
this._driver = null;
this._isOpen = false;
}
}

View File

@@ -0,0 +1,156 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import ScreenshotHelper from './ScreenshotHelper.mjs';
afterEach(() => {
vi.unstubAllEnvs();
});
describe('ScreenshotHelper', () => {
it('captures a local screenshot through Browserless', async () => {
vi.stubEnv('SCREENSHOT_PROVIDER', 'browserless');
let png = Buffer.from([137, 80, 78, 71]);
let page = {
setViewport: vi.fn(),
goto: vi.fn(),
waitForSelector: vi.fn(),
screenshot: vi.fn().mockResolvedValue(png),
close: vi.fn(),
};
let browser = {
newPage: vi.fn().mockResolvedValue(page),
close: vi.fn(),
};
let puppeteerClient = {
connect: vi.fn().mockResolvedValue(browser),
};
let httpServer = {
port: 4321,
open: vi.fn(),
close: vi.fn(),
};
let helper = new ScreenshotHelper({
env: {
BROWSERLESS_ENDPOINT: 'ws://browserless:3000',
SCREENSHOT_HOST: 'app',
},
httpServerFactory: () => httpServer,
puppeteerClient,
});
let screenshot = await helper.capture('schedules', {
viewport: { height: 400 },
});
await helper.close();
expect(screenshot).toEqual(png);
expect(puppeteerClient.connect).toHaveBeenCalledWith({
browserWSEndpoint: 'ws://browserless:3000',
});
expect(page.setViewport).toHaveBeenCalledWith({
width: 1200,
height: 400,
deviceScaleFactor: 2,
});
expect(page.goto).toHaveBeenCalledWith(
new URL('http://app:4321/screenshots/#schedules'),
{ waitUntil: 'load' },
);
expect(page.waitForSelector).toHaveBeenCalledWith(
'[data-screenshot-ready="true"]',
{ timeout: 30_000 },
);
expect(httpServer.close).toHaveBeenCalledOnce();
expect(page.close).toHaveBeenCalledOnce();
expect(browser.close).toHaveBeenCalledOnce();
});
it('captures a public screenshot through Cloudflare Browser Run', async () => {
vi.stubEnv('SCREENSHOT_PROVIDER', 'cloudflare');
vi.stubEnv('SITE_URL', 'https://splatoon3.ink');
vi.stubEnv('CLOUDFLARE_ACCOUNT_ID', 'account-id');
vi.stubEnv('CLOUDFLARE_BROWSER_RUN_API_TOKEN', 'api-token');
let png = new Uint8Array([137, 80, 78, 71]);
let fetch = vi.fn().mockResolvedValue(new Response(png, {
headers: { 'Content-Type': 'image/png' },
}));
let helper = new ScreenshotHelper({ fetch });
helper.defaultParams = { time: 123 };
let screenshot = await helper.capture('schedules', {
params: { region: 'NA' },
viewport: { width: 600 },
});
expect(screenshot).toEqual(Buffer.from(png));
expect(fetch).toHaveBeenCalledWith(
'https://api.cloudflare.com/client/v4/accounts/account-id/browser-rendering/screenshot?cacheTTL=0',
{
method: 'POST',
headers: {
Authorization: 'Bearer api-token',
'Content-Type': 'application/json',
},
body: JSON.stringify({
url: 'https://splatoon3.ink/screenshots/index.html#schedules?time=123&region=NA',
viewport: {
width: 600,
height: 675,
deviceScaleFactor: 2,
},
gotoOptions: { waitUntil: 'load' },
waitForSelector: {
selector: '[data-screenshot-ready="true"]',
timeout: 30_000,
},
screenshotOptions: { type: 'png' },
}),
},
);
});
it('surfaces a Cloudflare rate limit response without retrying', async () => {
vi.stubEnv('SCREENSHOT_PROVIDER', 'cloudflare');
vi.stubEnv('SITE_URL', 'https://splatoon3.ink');
vi.stubEnv('CLOUDFLARE_ACCOUNT_ID', 'account-id');
vi.stubEnv('CLOUDFLARE_BROWSER_RUN_API_TOKEN', 'api-token');
let fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({
success: false,
errors: [{ code: 2001, message: 'Rate limit exceeded' }],
}), {
status: 429,
headers: { 'Content-Type': 'application/json' },
}));
let helper = new ScreenshotHelper({ fetch });
await expect(helper.capture('schedules')).rejects.toThrow(
'Cloudflare Browser Run screenshot failed (429): Rate limit exceeded',
);
expect(fetch).toHaveBeenCalledOnce();
});
it('requires the Cloudflare configuration before opening', async () => {
vi.stubEnv('SCREENSHOT_PROVIDER', 'cloudflare');
vi.stubEnv('SITE_URL', '');
vi.stubEnv('CLOUDFLARE_ACCOUNT_ID', '');
vi.stubEnv('CLOUDFLARE_BROWSER_RUN_API_TOKEN', '');
let helper = new ScreenshotHelper;
await expect(helper.open()).rejects.toThrow(
'Missing Cloudflare screenshot configuration: SITE_URL, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_BROWSER_RUN_API_TOKEN',
);
});
it('requires an explicitly supported screenshot provider', async () => {
vi.stubEnv('SCREENSHOT_PROVIDER', 'auto');
let helper = new ScreenshotHelper;
await expect(helper.open()).rejects.toThrow(
'SCREENSHOT_PROVIDER must be "cloudflare" or "browserless"',
);
});
});

View File

@@ -0,0 +1,71 @@
import { URL } from 'url';
import puppeteer from 'puppeteer-core';
import { screenshotReadySelector, screenshotReadyTimeout } from '../../../src/common/screenshot.mjs';
import HttpServer from '../HttpServer.mjs';
export default class BrowserlessScreenshotDriver
{
_browser = null;
_env;
_httpServer = null;
_httpServerFactory;
_page = null;
_puppeteer;
constructor({
env = process.env,
httpServerFactory = () => new HttpServer,
puppeteerClient = puppeteer,
} = {}) {
this._env = env;
this._httpServerFactory = httpServerFactory;
this._puppeteer = puppeteerClient;
}
async open() {
if (!this._env.BROWSERLESS_ENDPOINT) {
throw new Error('Missing Browserless screenshot configuration: BROWSERLESS_ENDPOINT');
}
this._httpServer = this._httpServerFactory();
await this._httpServer.open();
this._browser = await this._puppeteer.connect({
browserWSEndpoint: this._env.BROWSERLESS_ENDPOINT,
});
this._page = await this._browser.newPage();
}
async capture(route, viewport) {
let host = this._env.SCREENSHOT_HOST || 'localhost';
let url = new URL(`http://${host}:${this._httpServer.port}/screenshots/`);
url.hash = route;
await this._page.setViewport(viewport);
await this._page.goto(url, {
waitUntil: 'load',
});
await this._page.waitForSelector(screenshotReadySelector, {
timeout: screenshotReadyTimeout,
});
return await this._page.screenshot();
}
async close() {
if (this._httpServer) {
await this._httpServer.close();
}
this._httpServer = null;
if (this._page) {
await this._page.close();
}
this._page = null;
if (this._browser) {
await this._browser.close();
}
this._browser = null;
}
}

View File

@@ -0,0 +1,88 @@
import { URL } from 'url';
import { screenshotReadySelector, screenshotReadyTimeout } from '../../../src/common/screenshot.mjs';
export default class CloudflareScreenshotDriver
{
_config = null;
_env;
_fetch;
constructor({ env = process.env, fetch = globalThis.fetch } = {}) {
this._env = env;
this._fetch = fetch;
}
async open() {
let names = [
'SITE_URL',
'CLOUDFLARE_ACCOUNT_ID',
'CLOUDFLARE_BROWSER_RUN_API_TOKEN',
];
let missing = names.filter(name => !this._env[name]);
if (missing.length) {
throw new Error(`Missing Cloudflare screenshot configuration: ${missing.join(', ')}`);
}
this._config = {
siteUrl: this._env.SITE_URL,
accountId: this._env.CLOUDFLARE_ACCOUNT_ID,
apiToken: this._env.CLOUDFLARE_BROWSER_RUN_API_TOKEN,
};
}
async capture(route, viewport) {
let url = new URL('/screenshots/index.html', this._config.siteUrl);
url.hash = route;
let endpoint = new URL(
`/client/v4/accounts/${this._config.accountId}/browser-rendering/screenshot`,
'https://api.cloudflare.com',
);
endpoint.searchParams.set('cacheTTL', '0');
let response = await this._fetch(endpoint.toString(), {
method: 'POST',
headers: {
Authorization: `Bearer ${this._config.apiToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
url: url.toString(),
viewport,
gotoOptions: { waitUntil: 'load' },
waitForSelector: {
selector: screenshotReadySelector,
timeout: screenshotReadyTimeout,
},
screenshotOptions: { type: 'png' },
}),
});
if (!response.ok) {
let message = await this._errorMessage(response);
throw new Error(`Cloudflare Browser Run screenshot failed (${response.status}): ${message}`);
}
return Buffer.from(await response.arrayBuffer());
}
async _errorMessage(response) {
let body = await response.text();
try {
let result = JSON.parse(body);
let messages = result.errors?.map(error => error.message).filter(Boolean);
if (messages?.length) {
return messages.join('; ');
}
} catch {
// Use the response body as-is when Cloudflare does not return JSON.
}
return body || response.statusText || 'Unknown error';
}
async close() {
this._config = null;
}
}

View File

@@ -0,0 +1,14 @@
import BrowserlessScreenshotDriver from './BrowserlessScreenshotDriver.mjs';
import CloudflareScreenshotDriver from './CloudflareScreenshotDriver.mjs';
export default function createScreenshotDriver(name, dependencies = {}) {
if (name === 'browserless') {
return new BrowserlessScreenshotDriver(dependencies);
}
if (name === 'cloudflare') {
return new CloudflareScreenshotDriver(dependencies);
}
throw new Error('SCREENSHOT_PROVIDER must be "cloudflare" or "browserless"');
}

View File

@@ -25,7 +25,7 @@ export default class StatusGeneratorManager
async sendStatuses(force = false) {
let availableClients = await this._getAvailableClients();
// Create screenshots in parallel (via Browserless)
// Create screenshots in parallel
let statusPromises = this._getStatuses(availableClients, force);
// Process each client in parallel (while maintaining post order)

View File

@@ -8,6 +8,7 @@ services:
init: true
restart: unless-stopped
environment:
SCREENSHOT_PROVIDER: browserless
BROWSERLESS_ENDPOINT: ws://browserless:3000
SCREENSHOT_HOST: app
depends_on:
@@ -16,6 +17,11 @@ services:
- .:/app
browserless:
image: ghcr.io/browserless/chromium
platform: linux/arm64 # Needed for Apple Silicon
restart: unless-stopped
environment:
CONCURRENT: ${BROWSERLESS_CONCURRENT:-1}
QUEUED: ${BROWSERLESS_QUEUED:-100}
ports:
- 3000:3000

View File

@@ -6,17 +6,11 @@ services:
init: true
restart: unless-stopped
environment:
BROWSERLESS_ENDPOINT: ws://browserless:3000
SCREENSHOT_HOST: app
depends_on:
- browserless
SCREENSHOT_PROVIDER: cloudflare
env_file:
- .env
labels: [ "com.centurylinklabs.watchtower.scope=splatoon3ink" ]
browserless:
labels: [ "com.centurylinklabs.watchtower.scope=splatoon3ink" ]
watchtower:
image: containrrr/watchtower
volumes:

View File

@@ -1,9 +1,3 @@
# See docker-compose.override.yml.* example files for dev/prod environments
services:
browserless:
image: ghcr.io/browserless/chromium
restart: unless-stopped
environment:
CONCURRENT: ${BROWSERLESS_CONCURRENT:-1}
QUEUED: ${BROWSERLESS_QUEUED:-100}
services: {}

View File

@@ -27,6 +27,33 @@ npm run dev
npm run build
```
### Screenshot Generation
Set `SCREENSHOT_PROVIDER` explicitly for social-media screenshots. Use `browserless` for local development with the Docker Compose development configuration. Use `cloudflare` in production to call Cloudflare Browser Run Quick Actions against `${SITE_URL}/screenshots/`.
The Cloudflare provider requires `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_BROWSER_RUN_API_TOKEN`. Create the API token with the **Browser Rendering Write** permission. The provider does not automatically fall back to Browserless when a Cloudflare request fails.
Wrangler can run a local browser for Puppeteer, Playwright, and CDP-based Workers, but Quick Actions are not supported by its local browser binding. Quick Actions require remote mode, so testing this provider still requires Cloudflare to reach the rendered page.
To test the real Cloudflare provider against a local build, build and serve `dist` in one terminal:
```sh
npm run build
npm run preview
```
Expose that server through a temporary Wrangler tunnel in a second terminal:
```sh
npx wrangler@latest tunnel quick-start http://localhost:5050
```
In a third terminal, use the printed `https://*.trycloudflare.com` URL for that run:
```sh
SCREENSHOT_PROVIDER=cloudflare SITE_URL=https://example.trycloudflare.com npm run social:test
```
### Lint with [ESLint](https://eslint.org/)
```sh

24
src/common/screenshot.mjs Normal file
View File

@@ -0,0 +1,24 @@
export const screenshotReadyAttribute = 'data-screenshot-ready';
export const screenshotReadySelector = `[${screenshotReadyAttribute}="true"]`;
export const screenshotReadyTimeout = 30_000;
function nextFrame(requestAnimationFrame) {
return new Promise(resolve => requestAnimationFrame(resolve));
}
export async function markScreenshotReady({
document = globalThis.document,
isCurrent = () => true,
requestAnimationFrame = globalThis.requestAnimationFrame,
} = {}) {
await document.fonts?.ready;
await Promise.allSettled(
[...document.images].map(image => image.decode()),
);
await nextFrame(requestAnimationFrame);
await nextFrame(requestAnimationFrame);
if (isCurrent()) {
document.documentElement.setAttribute(screenshotReadyAttribute, 'true');
}
}

View File

@@ -0,0 +1,66 @@
import { describe, expect, it, vi } from 'vitest';
import { markScreenshotReady } from './screenshot.mjs';
describe('markScreenshotReady', () => {
it('marks the page ready after fonts, images, and layout settle', async () => {
let resolveFonts;
let resolveImage;
let fontsReady = new Promise(resolve => { resolveFonts = resolve; });
let imageReady = new Promise(resolve => { resolveImage = resolve; });
let setAttribute = vi.fn();
let document = {
documentElement: { setAttribute },
fonts: { ready: fontsReady },
images: [{ complete: true, decode: () => imageReady }],
};
let frames = [];
let requestAnimationFrame = callback => frames.push(callback);
let ready = markScreenshotReady({ document, requestAnimationFrame });
await Promise.resolve();
expect(setAttribute).not.toHaveBeenCalled();
resolveFonts();
await Promise.resolve();
expect(setAttribute).not.toHaveBeenCalled();
resolveImage();
await Promise.resolve();
await Promise.resolve();
expect(frames).toHaveLength(1);
expect(setAttribute).not.toHaveBeenCalled();
frames.shift()();
await Promise.resolve();
expect(frames).toHaveLength(1);
expect(setAttribute).not.toHaveBeenCalled();
frames.shift()();
await ready;
expect(setAttribute).toHaveBeenCalledWith('data-screenshot-ready', 'true');
});
it('does not mark a readiness run that became stale while assets settled', async () => {
let resolveImage;
let imageReady = new Promise(resolve => { resolveImage = resolve; });
let setAttribute = vi.fn();
let document = {
documentElement: { setAttribute },
fonts: { ready: Promise.resolve() },
images: [{ decode: () => imageReady }],
};
let isCurrent = true;
let ready = markScreenshotReady({
document,
isCurrent: () => isCurrent,
requestAnimationFrame: callback => callback(),
});
isCurrent = false;
resolveImage();
await ready;
expect(setAttribute).not.toHaveBeenCalled();
});
});

View File

@@ -40,8 +40,10 @@
</template>
<script setup>
import { watchEffect } from 'vue';
import { nextTick, onUnmounted, watch, watchEffect } from 'vue';
import { useRoute } from 'vue-router';
import { markScreenshotReady, screenshotReadyAttribute } from '@/common/screenshot.mjs';
import { useDataStore } from '@/stores/data';
import { useTimeStore } from '@/stores/time';
import TimeOffsetSelector from '@/components/Debug/TimeOffsetSelector.vue';
@@ -52,7 +54,28 @@ const props = defineProps({
});
const route = useRoute();
const data = useDataStore();
const time = useTimeStore();
let readinessVersion = 0;
watch(
[() => data.isLoaded, () => data.isUpdating, () => route.fullPath],
async ([isLoaded, isUpdating]) => {
let version = ++readinessVersion;
document.documentElement.removeAttribute(screenshotReadyAttribute);
if (!isLoaded || isUpdating) return;
await nextTick();
await markScreenshotReady({ isCurrent: () => version === readinessVersion });
},
{ immediate: true, flush: 'post' },
);
onUnmounted(() => {
readinessVersion++;
document.documentElement.removeAttribute(screenshotReadyAttribute);
});
watchEffect(() => {
if (route.query.time) {

View File

@@ -60,6 +60,7 @@ export const useDataStore = defineStore('data', () => {
}
const isUpdating = computed(() => Object.values(stores).some(s => s.isUpdating));
const isLoaded = computed(() => Object.values(stores).every(s => s.data !== null));
function refresh() {
const now = Date.now();
@@ -108,6 +109,7 @@ export const useDataStore = defineStore('data', () => {
return {
updateAll,
isLoaded,
isUpdating,
startUpdating,
stopUpdating,

29
src/stores/data.test.mjs Normal file
View File

@@ -0,0 +1,29 @@
import { beforeEach, describe, expect, it } from 'vitest';
import { createPinia, setActivePinia } from 'pinia';
import {
useCoopDataStore,
useDataStore,
useFestivalsDataStore,
useGearDataStore,
useSchedulesDataStore,
} from './data.mjs';
describe('useDataStore', () => {
beforeEach(() => {
setActivePinia(createPinia());
});
it('becomes loaded only after every initial data source has content', () => {
let data = useDataStore();
expect(data.isLoaded).toBe(false);
useSchedulesDataStore().setData({ data: {} });
useGearDataStore().setData({ data: {} });
useCoopDataStore().setData({ data: {} });
expect(data.isLoaded).toBe(false);
useFestivalsDataStore().setData({});
expect(data.isLoaded).toBe(true);
});
});