From 27a468a8c721d5a2591c70853a1fd0fce022bf23 Mon Sep 17 00:00:00 2001 From: Kalle <38327916+Sendouc@users.noreply.github.com> Date: Wed, 14 Jan 2026 17:57:40 +0200 Subject: [PATCH] Misc perf improvements (#2723) --- app/features/builds/BuildRepository.server.ts | 102 +++++-- .../seasonPopularUsersWeapon.server.ts | 7 +- app/modules/in-game-lists/weapon-ids.ts | 13 - docs/load-testing.md | 263 ++++++++++++++++++ .../112-group-match-reported-at-index.js | 5 + migrations/113-season-query-indexes.js | 9 + 6 files changed, 360 insertions(+), 39 deletions(-) create mode 100644 docs/load-testing.md create mode 100644 migrations/112-group-match-reported-at-index.js create mode 100644 migrations/113-season-query-indexes.js diff --git a/app/features/builds/BuildRepository.server.ts b/app/features/builds/BuildRepository.server.ts index 961e5965c..6091d1412 100644 --- a/app/features/builds/BuildRepository.server.ts +++ b/app/features/builds/BuildRepository.server.ts @@ -9,10 +9,7 @@ import type { MainWeaponId, ModeShort, } from "~/modules/in-game-lists/types"; -import { - weaponIdHasAlts, - weaponIdToArrayWithAlts, -} from "~/modules/in-game-lists/weapon-ids"; +import { weaponIdToArrayWithAlts } from "~/modules/in-game-lists/weapon-ids"; import invariant from "~/utils/invariant"; import { COMMON_USER_FIELDS } from "~/utils/kysely.server"; import { sortAbilities } from "./core/ability-sorting.server"; @@ -252,8 +249,80 @@ export async function allByWeaponId( options: { limit: number; sortAbilities?: boolean }, ) { const { limit, sortAbilities: shouldSortAbilities = false } = options; + const weaponIds = weaponIdToArrayWithAlts(weaponId); - let query = db + let rows: Awaited>; + + if (weaponIds.length === 1) { + rows = await buildsByWeaponIdQuery(weaponIds[0], limit); + } else { + // For weapons with alts, run separate queries and merge. + // This allows each query to use the covering index for ordering, + // which is ~6x faster than using IN with multiple values. + const allResults = await Promise.all( + weaponIds.map((id) => buildsByWeaponIdQuery(id, limit)), + ); + + const seenBuildIds = new Set(); + type BuildRow = Awaited>[number]; + const merged: BuildRow[] = []; + + // Merge results maintaining sort order (tier asc, isTop500 desc, updatedAt desc) + // Since each query returns sorted results, we can interleave them + const pointers = allResults.map(() => 0); + + while (merged.length < limit) { + let bestIdx = -1; + let bestRow: BuildRow | null = null; + + for (let i = 0; i < allResults.length; i++) { + while ( + pointers[i] < allResults[i].length && + seenBuildIds.has(allResults[i][pointers[i]].id) + ) { + pointers[i]++; + } + + if (pointers[i] >= allResults[i].length) continue; + + const row = allResults[i][pointers[i]]; + + if ( + !bestRow || + row.bwTier < bestRow.bwTier || + (row.bwTier === bestRow.bwTier && + row.bwIsTop500 > bestRow.bwIsTop500) || + (row.bwTier === bestRow.bwTier && + row.bwIsTop500 === bestRow.bwIsTop500 && + row.bwUpdatedAt > bestRow.bwUpdatedAt) + ) { + bestIdx = i; + bestRow = row; + } + } + + if (bestIdx === -1 || !bestRow) break; + + seenBuildIds.add(bestRow.id); + merged.push(bestRow); + pointers[bestIdx]++; + } + + rows = merged; + } + + return rows.map((row) => { + const abilities = dbAbilitiesToArrayOfArrays(row.abilities); + + return { + ...row, + abilities: shouldSortAbilities ? sortAbilities(abilities) : abilities, + }; + }); +} + +function buildsByWeaponIdQuery(weaponSplId: MainWeaponId, limit: number) { + return db .selectFrom("BuildWeapon") .innerJoin("Build", "Build.id", "BuildWeapon.buildId") .leftJoin("PlusTier", "PlusTier.userId", "Build.ownerId") @@ -268,6 +337,9 @@ export async function allByWeaponId( "Build.updatedAt", "Build.private", "PlusTier.tier as plusTier", + "BuildWeapon.tier as bwTier", + "BuildWeapon.isTop500 as bwIsTop500", + "BuildWeapon.updatedAt as bwUpdatedAt", withAbilities(eb), jsonArrayFrom( eb @@ -284,26 +356,12 @@ export async function allByWeaponId( ).as("owner"), ]) .where("Build.private", "=", 0) - .where("BuildWeapon.weaponSplId", "in", weaponIdToArrayWithAlts(weaponId)) + .where("BuildWeapon.weaponSplId", "=", weaponSplId) .orderBy("BuildWeapon.tier", "asc") .orderBy("BuildWeapon.isTop500", "desc") .orderBy("BuildWeapon.updatedAt", "desc") - .limit(limit); - - if (weaponIdHasAlts(weaponId)) { - query = query.groupBy("BuildWeapon.buildId"); - } - - const rows = await query.execute(); - - return rows.map((row) => { - const abilities = dbAbilitiesToArrayOfArrays(row.abilities); - - return { - ...row, - abilities: shouldSortAbilities ? sortAbilities(abilities) : abilities, - }; - }); + .limit(limit) + .execute(); } function withAbilities(eb: ExpressionBuilder) { diff --git a/app/features/leaderboards/queries/seasonPopularUsersWeapon.server.ts b/app/features/leaderboards/queries/seasonPopularUsersWeapon.server.ts index 62a6c6f73..2aa25aa93 100644 --- a/app/features/leaderboards/queries/seasonPopularUsersWeapon.server.ts +++ b/app/features/leaderboards/queries/seasonPopularUsersWeapon.server.ts @@ -12,16 +12,15 @@ const stm = sql.prepare(/* sql */ ` "ReportedWeapon"."weaponSplId", count(*) as "count" from "ReportedWeapon" - left join "GroupMatchMap" on "ReportedWeapon"."groupMatchMapId" = "GroupMatchMap"."id" - left join "GroupMatch" on "GroupMatchMap"."matchId" = "GroupMatch"."id" + inner join "GroupMatchMap" on "ReportedWeapon"."groupMatchMapId" = "GroupMatchMap"."id" + inner join "GroupMatch" on "GroupMatchMap"."matchId" = "GroupMatch"."id" where "GroupMatch"."createdAt" between @starts and @ends group by "ReportedWeapon"."userId", "ReportedWeapon"."weaponSplId" - order by "count" desc ) select "q1"."userId", "q1"."weaponSplId", - "q1"."count" + max("q1"."count") as "count" from "q1" group by "q1"."userId" `); diff --git a/app/modules/in-game-lists/weapon-ids.ts b/app/modules/in-game-lists/weapon-ids.ts index ea1be325e..e5de10125 100644 --- a/app/modules/in-game-lists/weapon-ids.ts +++ b/app/modules/in-game-lists/weapon-ids.ts @@ -148,19 +148,6 @@ export const weaponIdToType = (weaponId: MainWeaponId) => { return "ALT_KIT"; }; -/** Returns true if the weapon ID has alternate skins - * - * * @example - * // Splattershot, Hero Shot, Order Shot... - * weaponIdHasAlts(40); // -> true - * weaponIdHasAlts(41); // -> true - * - * // Sploosh-o-matic has no alt skins - * weaponIdHasAlts(0); // -> false - */ -export const weaponIdHasAlts = (weaponId: MainWeaponId) => - weaponIdToAltId.has(weaponId) || altWeaponIdToId.has(weaponId); - export const SPLAT_BOMB_ID = 0; export const SUCTION_BOMB_ID = 1; export const BURST_BOMB_ID = 2; diff --git a/docs/load-testing.md b/docs/load-testing.md new file mode 100644 index 000000000..994299c99 --- /dev/null +++ b/docs/load-testing.md @@ -0,0 +1,263 @@ +# Load Testing Guide + +This guide explains how to profile and optimize slow routes in sendou.ink. + +## Prerequisites + +Install autocannon globally: + +```bash +npm install -g autocannon +``` + +## Step 1: Identify Slow Routes + +### Data URLs vs Full Routes + +When benchmarking a route like `/leaderboards`, the response includes server-side rendering (HTML generation). To isolate just the data loader performance, use React Router 7's data URLs: + +```bash +# Full route (includes SSR) +autocannon -c 10 -d 10 http://localhost:301/leaderboards + +# Data only (loader function only) +autocannon -c 10 -d 10 "http://localhost:301/leaderboards.data?_routes=features%2Fleaderboards%2Froutes%2Fleaderboards" +``` + +The data URL format is: `{route}.data?_routes={encoded_route_path}` + +To discover the exact data URL for a route, open Chrome DevTools Network tab, navigate to the page, then filter by "Fetch/XHR" - you'll see the `.data` requests made during client-side navigation. + +### Running benchmarks + +Use autocannon to benchmark endpoints: + +```bash +# Basic benchmark (10 connections, 10 seconds) +autocannon -c 10 -d 10 http://localhost:301/leaderboards + +# With more connections to simulate load +autocannon -c 50 -d 30 http://localhost:301/leaderboards + +# Single request timing +curl -s -o /dev/null -w "Time: %{time_total}s\n" "http://localhost:301/leaderboards" +``` + +### Reading autocannon output + +``` +┌─────────┬────────┬─────────┬─────────┬─────────┬────────────┬────────────┬─────────┐ +│ Stat │ 2.5% │ 50% │ 97.5% │ 99% │ Avg │ Stdev │ Max │ +├─────────┼────────┼─────────┼─────────┼─────────┼────────────┼────────────┼─────────┤ +│ Latency │ 826 ms │ 1478 ms │ 7674 ms │ 8985 ms │ 1989.09 ms │ 1770.75 ms │ 8985 ms │ +└─────────┴────────┴─────────┴─────────┴─────────┴────────────┴────────────┴─────────┘ +``` + +- **50%** (median): Half of requests are faster than this +- **97.5%/99%**: Tail latency - worst case for most users +- **Avg**: Average response time +- **Max**: Slowest single request + +Target: median < 100ms, 99th percentile < 500ms + +## Step 2: Profile SQL Queries + +Create a profiling script to test queries directly against the prod database: + +```typescript +// profile-query.ts +import Database from "better-sqlite3"; + +const db = new Database("db-prod.sqlite3"); + +function timeQuery(name: string, fn: () => unknown) { + const runs: number[] = []; + let result: unknown; + + // Run 5 times for consistent measurement + for (let i = 0; i < 5; i++) { + const start = performance.now(); + result = fn(); + runs.push(performance.now() - start); + } + + const avg = runs.reduce((a, b) => a + b, 0) / runs.length; + const rowCount = Array.isArray(result) ? result.length : "N/A"; + console.log(`${name}: avg ${avg.toFixed(2)}ms (${rowCount} rows)`); + return { avg, result }; +} + +// Example: Test a query +const stm = db.prepare(` + SELECT * FROM "User" WHERE "id" = @id +`); + +timeQuery("User lookup", () => stm.all({ id: 1 })); + +db.close(); +``` + +Run with: + +```bash +npx tsx profile-query.ts +``` + +## Step 3: Analyze Query Plans + +Use `EXPLAIN QUERY PLAN` to understand how SQLite executes queries: + +```typescript +const plan = db.prepare(` + EXPLAIN QUERY PLAN + SELECT * FROM "User" + JOIN "Skill" ON "Skill"."userId" = "User"."id" + WHERE "Skill"."season" = @season +`).all({ season: 10 }); + +console.log(JSON.stringify(plan, null, 2)); +``` + +### Reading query plans + +- **SCAN**: Full table scan (slow for large tables) +- **SEARCH USING INDEX**: Index lookup (fast) +- **USE TEMP B-TREE**: Temporary sorting/grouping needed + +## Step 4: Check Indexes + +```typescript +// List indexes on a table +const indexes = db.prepare(`PRAGMA index_list("Skill")`).all(); +console.log("Indexes:", JSON.stringify(indexes, null, 2)); + +// Get columns in an index +for (const index of indexes) { + const info = db.prepare(`PRAGMA index_info("${index.name}")`).all(); + console.log(`${index.name}:`, JSON.stringify(info, null, 2)); +} + +// Table row counts +const count = db.prepare(`SELECT COUNT(*) as count FROM "Skill"`).get(); +console.log(`Skill rows: ${count.count}`); +``` + +## Common Optimizations + +### 1. LEFT JOIN → INNER JOIN + +If your `WHERE` clause filters on the joined table, `LEFT JOIN` is unnecessary and prevents index usage: + +```sql +-- Slow: scans entire ReportedWeapon table +SELECT * FROM "ReportedWeapon" +LEFT JOIN "GroupMatch" ON ... +WHERE "GroupMatch"."createdAt" > @date + +-- Fast: uses index on createdAt +SELECT * FROM "ReportedWeapon" +INNER JOIN "GroupMatch" ON ... +WHERE "GroupMatch"."createdAt" > @date +``` + +### 2. Add Missing Indexes + +If query plan shows `SCAN` on a large table with a `WHERE` clause, add an index: + +```sql +CREATE INDEX skill_season ON "Skill"("season"); +``` + +### 3. Avoid ORDER BY in CTEs + +```sql +-- Slow: sorts intermediate results +WITH q1 AS ( + SELECT ... ORDER BY count DESC +) +SELECT ... FROM q1 + +-- Fast: only sort final results +WITH q1 AS ( + SELECT ... +) +SELECT ... FROM q1 ORDER BY count DESC +``` + +### 4. Use HAVING Instead of Subquery Filter + +```sql +-- Filter aggregates directly +SELECT "userId", count(*) as cnt +FROM "ReportedWeapon" +GROUP BY "userId" +HAVING count(*) >= 7 +``` + +## Step 5: Verify Fix + +Always verify optimizations produce identical results: + +```typescript +const original = originalStm.all(params); +const optimized = optimizedStm.all(params); + +console.log(`Original: ${original.length} rows`); +console.log(`Optimized: ${optimized.length} rows`); + +// Compare results +const originalSet = new Set(original.map(r => JSON.stringify(r))); +const optimizedSet = new Set(optimized.map(r => JSON.stringify(r))); + +const match = originalSet.size === optimizedSet.size && + [...originalSet].every(x => optimizedSet.has(x)); + +console.log(match ? "✓ Results match" : "✗ Results differ"); +``` + +## Step 6: Re-benchmark + +After applying fixes, restart the dev server and re-run autocannon: + +```bash +# Restart server to pick up changes +# (Ctrl+C and npm run dev) + +# Re-benchmark +autocannon -c 10 -d 10 http://localhost:301/leaderboards +``` + +## Example: Full Profiling Session + +```bash +# 1. Benchmark the slow route +autocannon -c 10 -d 10 http://localhost:301/leaderboards + +# 2. Create and run profiling script +cat > profile.ts << 'EOF' +import Database from "better-sqlite3"; +const db = new Database("db-prod.sqlite3"); + +const start = performance.now(); +const result = db.prepare(`YOUR_QUERY_HERE`).all({ /* params */ }); +console.log(`Time: ${(performance.now() - start).toFixed(2)}ms`); +console.log(`Rows: ${result.length}`); + +db.close(); +EOF + +npx tsx profile.ts + +# 3. Clean up +rm profile.ts +``` + +## Route File Locations + +| Route | Code Location | +|-------|---------------| +| `/leaderboards` | `app/features/leaderboards/` | +| `/q/*` | `app/features/sendouq/` | +| `/to/*` | `app/features/tournament/` | +| `/u/*` | `app/features/user-page/` | +| `/builds/*` | `app/features/builds/` | diff --git a/migrations/112-group-match-reported-at-index.js b/migrations/112-group-match-reported-at-index.js new file mode 100644 index 000000000..ff39cec75 --- /dev/null +++ b/migrations/112-group-match-reported-at-index.js @@ -0,0 +1,5 @@ +export function up(db) { + db.prepare( + /* sql */ `create index group_match_reported_at on "GroupMatch"("reportedAt")`, + ).run(); +} diff --git a/migrations/113-season-query-indexes.js b/migrations/113-season-query-indexes.js new file mode 100644 index 000000000..1de2fa736 --- /dev/null +++ b/migrations/113-season-query-indexes.js @@ -0,0 +1,9 @@ +export function up(db) { + db.prepare( + /* sql */ `create index map_result_user_id_season ON "MapResult"("userId", "season")`, + ).run(); + + db.prepare( + /* sql */ `create index player_result_owner_user_id_season ON "PlayerResult"("ownerUserId", "season")`, + ).run(); +}