diff --git a/.claude/skills/e2e/SKILL.md b/.claude/skills/e2e/SKILL.md new file mode 100644 index 000000000..7b005d613 --- /dev/null +++ b/.claude/skills/e2e/SKILL.md @@ -0,0 +1,115 @@ +--- +name: e2e +description: Run, debug, and manage Playwright e2e tests. Use when running e2e tests, debugging test failures, regenerating seed databases, or investigating test infrastructure issues. +--- + +# E2E Test Runner + +## Architecture overview + +- Tests live in `e2e/*.spec.ts`, config in `playwright.config.ts` +- Global setup (`e2e/global-setup.ts`) builds the app, creates per-worker databases, and starts one server per worker +- Port calculation: `E2E_BASE_PORT = PORT (from .env) + 500`. Default PORT is typically 4001, so base port = 4501. Workers use ports base+0 through base+3 +- Worker databases: `db-test-e2e-0.sqlite3` through `db-test-e2e-3.sqlite3` in the project root +- Seed databases (pre-seeded snapshots): `e2e/seeds/db-seed-*.sqlite3` +- MinIO (S3-compatible storage) is started via Docker Compose if not already running + +## Pre-flight checks (run before every test execution) + +Before running tests, check for these common issues: + +1. **Stale worker databases** — Files matching `db-test-e2e-*.sqlite3` in the project root can cause "table already exists" migration errors if the schema has changed since they were created. Run `npm run test:e2e:generate-seeds` to regenerate these from the seed databases. + +2. **Port conflicts** — Check if anything is already listening on the e2e ports (base port through base+3): + ``` + lsof -i :4501-4504 2>/dev/null + ``` + If ports are occupied by leftover e2e servers, kill them. If occupied by something else, warn the user. + +3. **Seed databases exist** — Verify `e2e/seeds/` contains the expected seed files. If missing, run `npm run test:e2e:generate-seeds`. + +4. **Docker running** — MinIO requires Docker. Check with `docker info` if there are storage-related failures. + +## Running tests + +### Run all tests +```bash +npm run test:e2e +``` + +### Run a specific test file +```bash +npx playwright test e2e/.spec.ts +``` + +### Flaky detection (repeats each test 10 times, stops on first failure) +```bash +npm run test:e2e:flaky-detect +``` + +### Regenerate seed databases (after schema/migration changes) +```bash +npm run test:e2e:generate-seeds +``` + +## Debugging failures + +Follow this funnel when tests fail: + +### Step 1: Read the error output +- Look for the actual assertion or timeout that failed +- Check if it's an infrastructure error (server didn't start, migration failed) vs. a test logic error + +### Step 2: Check infrastructure issues +Common infrastructure errors and fixes: +- **"table already exists"** → Stale worker DBs. Run `rm -f db-test-e2e-*.sqlite3` +- **"Server on port X did not start within timeout"** → Port conflict or app build error. Check ports with `lsof -i :` and check for build errors +- **"MinIO failed to start"** → Docker not running or compose issue. Check `docker info` +- **Seed-related errors** → Run `npm run test:e2e:generate-seeds` + +### Step 3: Reduce to single debug worker +If the error is unclear, re-run with debug output and a single worker to see server logs: +```bash +E2E_DEBUG=true E2E_WORKERS=1 npx playwright test e2e/.spec.ts +``` +This shows stdout/stderr from the test server, which is hidden by default. + +### Step 4: Examine trace artifacts +Playwright is configured with `trace: "retain-on-failure"`. After a failure, view the trace: +```bash +npx playwright show-trace test-results//trace.zip +``` + +## Test pattern reference + +Every test follows this pattern — use these imports from `~/utils/playwright`, NOT raw Playwright APIs: + +```typescript +import { expect, impersonate, navigate, seed, test } from "~/utils/playwright"; + +test.describe("Feature", () => { + test("does something", async ({ page }) => { + await seed(page); // Reset DB to a known seed state + await impersonate(page, USER_ID); // Log in as a specific user (default: admin) + await navigate({ page, url: "..." });// Navigate (waits for hydration) + // ... interact with the page ... + await submit(page); // Submit a form (waits for POST response) + }); +}); +``` + +Key rules: +- Use `navigate()` instead of `page.goto()` — it waits for hydration +- Use `submit()` instead of clicking submit buttons directly — it waits for the POST response +- Use `seed(page, variation?)` to reset the database. Available variations: DEFAULT, NO_TOURNAMENT_TEAMS, REG_OPEN, SMALL_SOS, NZAP_IN_TEAM, NO_SCRIMS, NO_SQ_GROUPS +- Use `impersonate(page, userId?)` to authenticate. Default is admin (ADMIN_ID) +- Avoid `page.waitForTimeout` — use assertions or `waitFor` patterns instead +- Import `test` from `~/utils/playwright` (not from `@playwright/test`) — it includes worker port fixtures + +## Environment variables + +| Variable | Purpose | Default | +|----------|---------|---------| +| `E2E_WORKERS` | Number of parallel workers | 4 | +| `E2E_DEBUG` | Show server stdout/stderr when "true" | unset | +| `PORT` | Base port for dev server (e2e adds 500) | 5173 | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f73ac387a..e09f6e977 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,4 +43,6 @@ jobs: - name: Check homemade badges run: npm run check-homemade-badges - name: Check articles - run: npm run check-articles + run: npm run check-articles + - name: Check test DB migrations + run: npm run check-test-db-migrations diff --git a/AGENTS.md b/AGENTS.md index 62a2f34bf..dc43316b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,13 +58,6 @@ - `db-test.sqlite3` is the unit test database (should be blank sans migrations ran) - `db-prod.sqlite3` is a copy of the production environment db which can be freely experimented with -## E2E testing - -- library used for E2E testing is Playwright -- `page.goto` is forbidden, use the `navigate` function to do a page navigation -- to submit a form you use the `submit` function -- `page.waitForTimeout` should be avoided - ## Unit testing - library used for unit testing is Vitest diff --git a/app/features/sendouq/core/SendouQ.server.test.ts b/app/features/sendouq/core/SendouQ.server.test.ts index d575c0d22..1ceb5a359 100644 --- a/app/features/sendouq/core/SendouQ.server.test.ts +++ b/app/features/sendouq/core/SendouQ.server.test.ts @@ -821,6 +821,7 @@ describe("SendouQ", () => { describe("skill-based sorting", () => { beforeEach(async () => { + refreshUserSkills(1); await dbInsertUsers(10); }); diff --git a/app/utils/playwright.ts b/app/utils/playwright.ts index faf2f0bef..212ebad75 100644 --- a/app/utils/playwright.ts +++ b/app/utils/playwright.ts @@ -10,7 +10,7 @@ import type { SeedVariation } from "~/features/api-private/routes/seed"; import { tournamentBracketsPage } from "./urls"; dotenv.config(); -export const E2E_BASE_PORT = Number(process.env.PORT || 5173) + 1000; +export const E2E_BASE_PORT = Number(process.env.PORT || 5173) + 500; type WorkerFixtures = { workerPort: number; diff --git a/db-test.sqlite3 b/db-test.sqlite3 index 7861ee433..148c34270 100644 Binary files a/db-test.sqlite3 and b/db-test.sqlite3 differ diff --git a/e2e/builds.spec.ts b/e2e/builds.spec.ts index f772ce08b..8d8f21cbc 100644 --- a/e2e/builds.spec.ts +++ b/e2e/builds.spec.ts @@ -127,6 +127,7 @@ test.describe("Builds", () => { await page.getByLabel("Tower Control").click(); await expect(page.getByTestId("build-mode-TC")).toHaveCount(24); await page.getByTestId("delete-filter-button").click(); + await expect(page.getByTestId("build-card").first()).toBeVisible(); // // date filter diff --git a/e2e/scrims.spec.ts b/e2e/scrims.spec.ts index 6c887698a..02968ef63 100644 --- a/e2e/scrims.spec.ts +++ b/e2e/scrims.spec.ts @@ -196,17 +196,17 @@ test.describe("Scrims", () => { await selectUser({ labelName: "User 2", page, - userName: "a", + userName: "5", }); await selectUser({ labelName: "User 3", page, - userName: "b", + userName: "6", }); await selectUser({ labelName: "User 4", page, - userName: "c", + userName: "7", }); await page.getByLabel("Start time").selectOption({ index: 1 }); diff --git a/e2e/seeds/db-seed-DEFAULT.sqlite3 b/e2e/seeds/db-seed-DEFAULT.sqlite3 index 561cfb90e..4e046e611 100644 Binary files a/e2e/seeds/db-seed-DEFAULT.sqlite3 and b/e2e/seeds/db-seed-DEFAULT.sqlite3 differ diff --git a/e2e/seeds/db-seed-NO_SCRIMS.sqlite3 b/e2e/seeds/db-seed-NO_SCRIMS.sqlite3 index edf1ef09c..91ded76e2 100644 Binary files a/e2e/seeds/db-seed-NO_SCRIMS.sqlite3 and b/e2e/seeds/db-seed-NO_SCRIMS.sqlite3 differ diff --git a/e2e/seeds/db-seed-NO_SQ_GROUPS.sqlite3 b/e2e/seeds/db-seed-NO_SQ_GROUPS.sqlite3 index 089aae25a..009f95522 100644 Binary files a/e2e/seeds/db-seed-NO_SQ_GROUPS.sqlite3 and b/e2e/seeds/db-seed-NO_SQ_GROUPS.sqlite3 differ diff --git a/e2e/seeds/db-seed-NO_TOURNAMENT_TEAMS.sqlite3 b/e2e/seeds/db-seed-NO_TOURNAMENT_TEAMS.sqlite3 index 43004361a..478e53375 100644 Binary files a/e2e/seeds/db-seed-NO_TOURNAMENT_TEAMS.sqlite3 and b/e2e/seeds/db-seed-NO_TOURNAMENT_TEAMS.sqlite3 differ diff --git a/e2e/seeds/db-seed-NZAP_IN_TEAM.sqlite3 b/e2e/seeds/db-seed-NZAP_IN_TEAM.sqlite3 index f835e50bc..ec41f54e6 100644 Binary files a/e2e/seeds/db-seed-NZAP_IN_TEAM.sqlite3 and b/e2e/seeds/db-seed-NZAP_IN_TEAM.sqlite3 differ diff --git a/e2e/seeds/db-seed-REG_OPEN.sqlite3 b/e2e/seeds/db-seed-REG_OPEN.sqlite3 index f587484f5..a742d2f63 100644 Binary files a/e2e/seeds/db-seed-REG_OPEN.sqlite3 and b/e2e/seeds/db-seed-REG_OPEN.sqlite3 differ diff --git a/e2e/seeds/db-seed-SMALL_SOS.sqlite3 b/e2e/seeds/db-seed-SMALL_SOS.sqlite3 index b8ef33dfe..5a022e32b 100644 Binary files a/e2e/seeds/db-seed-SMALL_SOS.sqlite3 and b/e2e/seeds/db-seed-SMALL_SOS.sqlite3 differ diff --git a/package.json b/package.json index 57ebc3a7f..96eb0283a 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,9 @@ "test:unit:browser:ui": "cross-env VITE_SITE_DOMAIN=http://localhost:5173 vitest --silent=passed-only", "test:e2e": "npx playwright test", "test:e2e:flaky-detect": "npx playwright test --repeat-each=10 --max-failures=1", - "test:e2e:generate-seeds": "cross-env DB_PATH=db-test.sqlite3 npx vite-node scripts/generate-e2e-seed-dbs.ts", - "checks": "npm run biome:fix && npm run test:unit:browser && npm run check-translation-jsons && npm run typecheck && npm run knip", + "test:e2e:generate-seeds": "cross-env DB_PATH=db-test.sqlite3 npm run migrate up && cross-env DB_PATH=db-test.sqlite3 npx vite-node scripts/generate-e2e-seed-dbs.ts", + "check-test-db-migrations": "node --experimental-strip-types scripts/check-test-db-migrations.ts", + "checks": "npm run biome:fix && npm run test:unit:browser && npm run check-translation-jsons && npm run typecheck && npm run knip && npm run check-test-db-migrations", "setup": "cross-env DB_PATH=db.sqlite3 vite-node ./scripts/setup.ts", "i18n:sync": "i18next-locales-sync -e true -p en -s da de es-ES es-US fr-CA fr-EU he it ja ko nl pl pt-BR ru zh -l locales && npm run biome:fix", "knip": "knip" diff --git a/scripts/check-test-db-migrations.ts b/scripts/check-test-db-migrations.ts new file mode 100644 index 000000000..d7cfad3ad --- /dev/null +++ b/scripts/check-test-db-migrations.ts @@ -0,0 +1,65 @@ +/** biome-ignore-all lint/suspicious/noConsole: Biome v2 migration */ + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import Database from "better-sqlite3"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const ROOT_DIR = path.join(__dirname, ".."); + +const MIGRATIONS_DIR = path.join(ROOT_DIR, "migrations"); +const DB_FILES = [ + path.join(ROOT_DIR, "db-test.sqlite3"), + ...fs + .readdirSync(path.join(ROOT_DIR, "e2e", "seeds")) + .filter((f) => f.startsWith("db-seed-") && f.endsWith(".sqlite3")) + .map((f) => path.join(ROOT_DIR, "e2e", "seeds", f)), +]; + +const migrationFilesOnDisk = fs + .readdirSync(MIGRATIONS_DIR) + .filter((f) => f.endsWith(".js")) + .sort(); + +let hasErrors = false; + +for (const dbPath of DB_FILES) { + const relativePath = path.relative(ROOT_DIR, dbPath); + + if (!fs.existsSync(dbPath)) { + console.warn(`Warning: ${relativePath} does not exist, skipping`); + continue; + } + + const db = new Database(dbPath, { readonly: true }); + const rows = db + .prepare("SELECT name FROM migrations ORDER BY id ASC") + .all() as Array<{ name: string }>; + db.close(); + + const migrationsInDb = new Set(rows.map((r) => r.name)); + const missingMigrations = migrationFilesOnDisk.filter( + (name) => !migrationsInDb.has(name), + ); + + if (missingMigrations.length > 0) { + hasErrors = true; + console.error( + `\n${relativePath} is missing ${missingMigrations.length} migration(s):`, + ); + for (const name of missingMigrations) { + console.error(` - ${name}`); + } + } +} + +if (hasErrors) { + console.error( + "\nRun `npm run test:e2e:generate-seeds` to regenerate test databases.", + ); + process.exit(1); +} else { + console.log("All test databases have the latest migrations."); +} diff --git a/scripts/generate-e2e-seed-dbs.ts b/scripts/generate-e2e-seed-dbs.ts index 574b3667a..56bbb9694 100644 --- a/scripts/generate-e2e-seed-dbs.ts +++ b/scripts/generate-e2e-seed-dbs.ts @@ -7,10 +7,20 @@ import { SEED_VARIATIONS } from "../app/features/api-private/constants"; const E2E_SEEDS_DIR = "e2e/seeds"; const BASE_TEST_DB = "db-test.sqlite3"; +const E2E_WORKER_DB_PATTERN = /^db-test-e2e-\d+\.sqlite3$/; + async function generatePreSeededDatabases() { // biome-ignore lint/suspicious/noConsole: CLI script output console.log("Generating pre-seeded databases for e2e tests...\n"); + for (const file of fs.readdirSync(".")) { + if (E2E_WORKER_DB_PATTERN.test(file)) { + fs.unlinkSync(file); + // biome-ignore lint/suspicious/noConsole: CLI script output + console.log(`Deleted stale worker db: ${file}`); + } + } + if (!fs.existsSync(E2E_SEEDS_DIR)) { fs.mkdirSync(E2E_SEEDS_DIR, { recursive: true }); }