E2E tests against built site + add to pipeline (#2563)

.github/workflows/e2e-tests.yml

@@ -0,0 +1,42 @@
+name: E2E Tests
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'content/**'
+      - 'public/**'
+      - 'locales/**'
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - 'content/**'
+      - 'public/**'
+      - 'locales/**'
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps
+
+      - name: Run E2E tests
+        run: npm run test:e2e
+
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

AGENTS.md

@@ -8,6 +8,8 @@
 - `npm run typecheck` runs TypeScript type checking
 - `npm run biome:fix` runs Biome code formatter and linter
 - `npm run test:unit` runs all unit tests
+- `npm run test:e2e` runs all e2e tests
+- `npm run test:e2e:flaky-detect` runs all e2e tests and repeats each 10 times
 - `npm run i18n:sync` syncs translation jsons with English and should always be run after adding new text to an English translation file 
 
 ## Typescript

app/features/admin/core/dev-controls.ts

@@ -0,0 +1,5 @@
+import { IS_E2E_TEST_RUN } from "~/utils/e2e";
+
+/** Should the user be able to access dev controls? Seeding & user impersonation without auth. */
+export const DANGEROUS_CAN_ACCESS_DEV_CONTROLS =
+	process.env.NODE_ENV === "development" || IS_E2E_TEST_RUN;

app/features/admin/loaders/admin.server.ts

@@ -4,10 +4,10 @@ import * as UserRepository from "~/features/user-page/UserRepository.server";
 import { requireRole } from "~/modules/permissions/guards.server";
 import { parseSafeSearchParams } from "~/utils/remix.server";
 import { adminActionSearchParamsSchema } from "../admin-schemas";
+import { DANGEROUS_CAN_ACCESS_DEV_CONTROLS } from "../core/dev-controls";
 
 export const loader = async ({ request }: LoaderFunctionArgs) => {
-	// allow unauthorized access in development mode to access impersonation controls
-	if (process.env.NODE_ENV === "production") {
+	if (!DANGEROUS_CAN_ACCESS_DEV_CONTROLS) {
 		const user = await requireUser(request);
 		requireRole(user, "STAFF");
 	}

app/features/admin/routes/admin.tsx

@@ -35,6 +35,7 @@ import {
 	userPage,
 } from "~/utils/urls";
 import { action } from "../actions/admin.server";
+import { DANGEROUS_CAN_ACCESS_DEV_CONTROLS } from "../core/dev-controls";
 import { loader } from "../loaders/admin.server";
 export { loader, action };
 
@@ -111,10 +112,8 @@ function AdminActions() {
 
 	return (
 		<div className="stack lg">
-			{process.env.NODE_ENV !== "production" && <Seed />}
-			{process.env.NODE_ENV !== "production" || isAdmin ? (
-				<Impersonate />
-			) : null}
+			{DANGEROUS_CAN_ACCESS_DEV_CONTROLS && <Seed />}
+			{DANGEROUS_CAN_ACCESS_DEV_CONTROLS || isAdmin ? <Impersonate /> : null}
 
 			{isStaff ? <LinkPlayer /> : null}
 			{isStaff ? <GiveArtist /> : null}

app/features/api-private/routes/seed.ts

@@ -1,6 +1,7 @@
 import type { ActionFunction } from "@remix-run/node";
 import { z } from "zod/v4";
 import { seed } from "~/db/seed";
+import { DANGEROUS_CAN_ACCESS_DEV_CONTROLS } from "~/features/admin/core/dev-controls";
 import { SEED_VARIATIONS } from "~/features/api-private/constants";
 import { parseRequestPayload } from "~/utils/remix.server";
 
@@ -13,7 +14,7 @@ export type SeedVariation = NonNullable<
 >;
 
 export const action: ActionFunction = async ({ request }) => {
-	if (process.env.NODE_ENV === "production") {
+	if (!DANGEROUS_CAN_ACCESS_DEV_CONTROLS) {
 		throw new Response(null, { status: 400 });
 	}
 

app/features/auth/core/routes.server.ts

@@ -2,6 +2,7 @@ import type { ActionFunction, LoaderFunction } from "@remix-run/node";
 import { redirect } from "@remix-run/node";
 import { isbot } from "isbot";
 import { z } from "zod/v4";
+import { DANGEROUS_CAN_ACCESS_DEV_CONTROLS } from "~/features/admin/core/dev-controls";
 import * as UserRepository from "~/features/user-page/UserRepository.server";
 import { requireRole } from "~/modules/permissions/guards.server";
 import { logger } from "~/utils/logger";
@@ -74,7 +75,7 @@ export const logInAction: ActionFunction = async ({ request }) => {
 };
 
 export const impersonateAction: ActionFunction = async ({ request }) => {
-	if (process.env.NODE_ENV === "production") {
+	if (!DANGEROUS_CAN_ACCESS_DEV_CONTROLS) {
 		const user = await requireUser(request);
 		requireRole(user, "ADMIN");
 	}

app/features/auth/core/session.server.ts

@@ -1,4 +1,5 @@
 import { createCookieSessionStorage } from "@remix-run/node";
+import { IS_E2E_TEST_RUN } from "~/utils/e2e";
 import invariant from "~/utils/invariant";
 
 const ONE_YEAR_IN_SECONDS = 31_536_000;
@@ -11,11 +12,14 @@ export const authSessionStorage = createCookieSessionStorage({
 		name: "__session",
 		sameSite: "lax",
 		// need to specify domain so that sub-domains can access it
-		domain: process.env.NODE_ENV === "production" ? "sendou.ink" : undefined,
+		domain:
+			process.env.NODE_ENV === "production" && !IS_E2E_TEST_RUN
+				? "sendou.ink"
+				: undefined,
 		path: "/",
 		httpOnly: true,
 		secrets: [process.env.SESSION_SECRET ?? "secret"],
-		secure: process.env.NODE_ENV === "production",
+		secure: process.env.NODE_ENV === "production" && !IS_E2E_TEST_RUN,
 		maxAge: ONE_YEAR_IN_SECONDS,
 	},
 });

app/features/chat/ChatSystemMessage.server.ts

@@ -1,4 +1,5 @@
 import { nanoid } from "nanoid";
+import { IS_E2E_TEST_RUN } from "~/utils/e2e";
 import invariant from "~/utils/invariant";
 import { logger } from "~/utils/logger";
 import type { ChatMessage } from "./chat-types";
@@ -13,13 +14,24 @@ interface ChatSystemMessageService {
 	send: (msg: PartialChatMessage | PartialChatMessage[]) => undefined;
 }
 
-invariant(
-	process.env.SKALOP_SYSTEM_MESSAGE_URL,
-	"Missing env var: SKALOP_SYSTEM_MESSAGE_URL",
-);
-invariant(process.env.SKALOP_TOKEN, "Missing env var: SKALOP_TOKEN");
+let systemMessagesDisabled = false;
+
+if (!IS_E2E_TEST_RUN) {
+	invariant(
+		process.env.SKALOP_SYSTEM_MESSAGE_URL,
+		"Missing env var: SKALOP_SYSTEM_MESSAGE_URL",
+	);
+	invariant(process.env.SKALOP_TOKEN, "Missing env var: SKALOP_TOKEN");
+} else if (
+	!process.env.SKALOP_SYSTEM_MESSAGE_URL ||
+	!process.env.SKALOP_TOKEN
+) {
+	systemMessagesDisabled = true;
+}
 
 export const send: ChatSystemMessageService["send"] = (partialMsg) => {
+	if (systemMessagesDisabled) return;
+
 	const msgArr = Array.isArray(partialMsg) ? partialMsg : [partialMsg];
 
 	const fullMessages: ChatMessage[] = msgArr.map((partialMsg) => {

app/features/notifications/core/notify.server.ts

@@ -1,6 +1,7 @@
 import type { TFunction } from "i18next";
 import pLimit from "p-limit";
 import { WebPushError } from "web-push";
+import { IS_E2E_TEST_RUN } from "~/utils/e2e";
 import type { NotificationSubscription } from "../../../db/tables";
 import i18next from "../../../modules/i18n/i18next.server";
 import { logger } from "../../../utils/logger";
@@ -74,7 +75,7 @@ const sentNotifications = new Set<string>();
 // deduplicates notifications as a failsafe & anti-abuse mechanism
 function isNotificationAlreadySent(notification: Notification) {
 	// e2e tests should not be affected by this
-	if (process.env.NODE_ENV !== "production") {
+	if (IS_E2E_TEST_RUN) {
 		return false;
 	}
 

app/features/sendouq/QRepository.server.ts

@@ -8,6 +8,7 @@ import type {
 	UserMapModePreferences,
 } from "~/db/tables";
 import { databaseTimestampNow, dateToDatabaseTimestamp } from "~/utils/dates";
+import { IS_E2E_TEST_RUN } from "~/utils/e2e";
 import { shortNanoid } from "~/utils/id";
 import { COMMON_USER_FIELDS } from "~/utils/kysely.server";
 import { userIsBanned } from "../ban/core/banned.server";
@@ -27,7 +28,7 @@ export function mapModePreferencesByGroupId(groupId: number) {
 
 // groups visible for longer to make development easier
 const SECONDS_TILL_STALE =
-	process.env.NODE_ENV === "development" ? 1_000_000 : 1_800;
+	process.env.NODE_ENV === "development" || IS_E2E_TEST_RUN ? 1_000_000 : 1_800;
 
 export async function findLookingGroups({
 	minGroupSize,

app/features/theme/core/session.server.ts

@@ -1,5 +1,5 @@
 import { createCookieSessionStorage } from "@remix-run/node";
-
+import { IS_E2E_TEST_RUN } from "~/utils/e2e";
 import invariant from "~/utils/invariant";
 import type { Theme } from "./provider";
 import { isTheme } from "./provider";
@@ -14,7 +14,7 @@ const sessionSecret = process.env.SESSION_SECRET ?? "secret";
 const themeStorage = createCookieSessionStorage({
 	cookie: {
 		name: "theme",
-		secure: process.env.NODE_ENV === "production",
+		secure: process.env.NODE_ENV === "production" && !IS_E2E_TEST_RUN,
 		secrets: [sessionSecret],
 		sameSite: "lax",
 		path: "/",

app/features/tournament/core/streams.server.ts

@@ -1,9 +1,13 @@
 import type { TournamentData } from "~/features/tournament-bracket/core/Tournament.server";
 import { getStreams } from "~/modules/twitch";
+import { IS_E2E_TEST_RUN } from "~/utils/e2e";
 
 export async function streamsByTournamentId(tournament: TournamentData["ctx"]) {
 	// prevent error logs in development
-	if (process.env.NODE_ENV === "development" && !process.env.TWITCH_CLIENT_ID) {
+	if (
+		(process.env.NODE_ENV === "development" && !process.env.TWITCH_CLIENT_ID) ||
+		IS_E2E_TEST_RUN
+	) {
 		return [];
 	}
 	const twitchUsersOfTournament = tournament.teams

app/features/user-search/loaders/u.server.ts

@@ -1,5 +1,6 @@
 import type { LoaderFunctionArgs, SerializeFrom } from "@remix-run/node";
 import { z } from "zod/v4";
+import { DANGEROUS_CAN_ACCESS_DEV_CONTROLS } from "~/features/admin/core/dev-controls";
 import { getUserId } from "~/features/auth/core/user.server";
 import * as UserRepository from "~/features/user-page/UserRepository.server";
 import { parseSearchParams } from "~/utils/remix.server";
@@ -13,7 +14,7 @@ const searchParamsSchema = z.object({
 });
 
 export const loader = async ({ request }: LoaderFunctionArgs) => {
-	if (process.env.NODE_ENV === "production") {
+	if (!DANGEROUS_CAN_ACCESS_DEV_CONTROLS) {
 		const user = await getUserId(request);
 		if (!user) {
 			return null;

app/modules/permissions/hooks.ts

@@ -1,5 +1,6 @@
 import { useUser } from "~/features/auth/core/user";
 import type { EntityWithPermissions, Role } from "~/modules/permissions/types";
+import { IS_E2E_TEST_RUN } from "~/utils/e2e";
 import { isAdmin } from "./utils";
 
 /**
@@ -29,7 +30,11 @@ export function useHasPermission<
 	if (!user) return false;
 
 	// admin can do anything in production but not in development for better testing
-	if (process.env.NODE_ENV === "production" && isAdmin(user)) {
+	if (
+		process.env.NODE_ENV === "production" &&
+		!IS_E2E_TEST_RUN &&
+		isAdmin(user)
+	) {
 		return true;
 	}
 

app/root.tsx

@@ -48,6 +48,7 @@ import { useIsMounted } from "./hooks/useIsMounted";
 import { DEFAULT_LANGUAGE } from "./modules/i18n/config";
 import i18next, { i18nCookie } from "./modules/i18n/i18next.server";
 import type { Namespace } from "./modules/i18n/resources.server";
+import { IS_E2E_TEST_RUN } from "./utils/e2e";
 import { isRevalidation, metaTags } from "./utils/remix";
 import { SUSPENDED_PAGE } from "./utils/urls";
 
@@ -176,7 +177,7 @@ function Document({
 				<Fonts />
 			</head>
 			<body style={customizedCSSVars}>
-				{process.env.NODE_ENV === "development" && <HydrationTestIndicator />}
+				{IS_E2E_TEST_RUN && <HydrationTestIndicator />}
 				<React.StrictMode>
 					<RouterProvider navigate={navigate} useHref={useHref}>
 						<I18nProvider locale={i18n.language}>

app/utils/e2e.ts

@@ -0,0 +1 @@
+export const IS_E2E_TEST_RUN = import.meta.env.VITE_E2E_TEST_RUN === "true";

app/utils/playwright.ts

@@ -81,7 +81,7 @@ export function modalClickConfirmButton(page: Page) {
 }
 
 export async function fetchSendouInk<T>(url: string) {
-	const res = await fetch(`http://localhost:5173${url}`);
+	const res = await fetch(`http://localhost:6173${url}`);
 	if (!res.ok) throw new Error("Response not successful");
 
 	return res.json() as T;

app/utils/urls-img.ts

@@ -1,8 +1,11 @@
 // TODO: separating this file from urls.ts is a temporary solution. The reason is that import.meta.env cannot currently be used in files that are consumed by plain Node.js
 
+import { IS_E2E_TEST_RUN } from "./e2e";
+
 const USER_SUBMITTED_IMAGE_ROOT =
-	process.env.NODE_ENV === "development" &&
-	import.meta.env.VITE_PROD_MODE !== "true"
+	(process.env.NODE_ENV === "development" &&
+		import.meta.env.VITE_PROD_MODE !== "true") ||
+	IS_E2E_TEST_RUN
 		? "http://127.0.0.1:9000/sendou"
 		: "https://sendou.nyc3.cdn.digitaloceanspaces.com";
 

biome.json

@@ -6,7 +6,8 @@
 			"!scripts/dicts/**/*",
 			"!scripts/output/**/*",
 			"!app/db/seed/placements.json",
-			"!build/**/*"
+			"!build/**/*",
+			"!test-results/**/*"
 		]
 	},
 	"linter": {

e2e/associations.spec.ts

@@ -63,7 +63,7 @@ test.describe("Associations", () => {
 		await impersonate(page, NZAP_TEST_ID);
 		await navigate({
 			page,
-			url: inviteLink.replace("https://sendou.ink", "http://localhost:5173"),
+			url: inviteLink.replace("https://sendou.ink", "http://localhost:6173"),
 		});
 
 		await submit(page);

e2e/tournament-bracket.spec.ts

@@ -273,7 +273,7 @@ test.describe("Tournament bracket", () => {
 		);
 		const inviteLink = inviteLinkProd.replace(
 			"https://sendou.ink",
-			"http://localhost:5173",
+			"http://localhost:6173",
 		);
 
 		await impersonate(page, NZAP_TEST_ID);

e2e/user-page.spec.ts

@@ -37,7 +37,7 @@ test.describe("User page", () => {
 
 		// test changing the big badge
 		await page.getByAltText("Lobster Crossfire").click();
-		expect(page.getByAltText("Lobster Crossfire")).toHaveAttribute(
+		await expect(page.getByAltText("Lobster Crossfire")).toHaveAttribute(
 			"width",
 			"125",
 		);

playwright.config.ts

@@ -35,7 +35,7 @@ const config: PlaywrightTestConfig = {
 		/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
 		actionTimeout: 0,
 		/* Base URL to use in actions like `await page.goto('/')`. */
-		baseURL: "http://localhost:5173",
+		baseURL: "http://localhost:6173",
 
 		trace: "retain-on-failure",
 
@@ -99,9 +99,19 @@ const config: PlaywrightTestConfig = {
 
 	/* Run your local dev server before starting the tests */
 	webServer: {
-		command: "npm run dev",
-		port: 5173,
-		reuseExistingServer: !process.env.CI,
+		env: {
+			DB_PATH: "db-test-e2e.sqlite3",
+			DISCORD_CLIENT_ID: "123",
+			DISCORD_CLIENT_SECRET: "secret",
+			SESSION_SECRET: "secret",
+			PORT: "6173",
+			VITE_SITE_DOMAIN: "http://localhost:6173",
+			VITE_E2E_TEST_RUN: "true",
+		},
+		command: "npm run build && npm start",
+		port: 6173,
+		reuseExistingServer: false,
+		timeout: 60_000 * 2, // 2 minutes
 	},
 	build: {
 		external: ["**/*.json"],