diff --git a/app/db/tables.ts b/app/db/tables.ts index bb0d8d285..999d55166 100644 --- a/app/db/tables.ts +++ b/app/db/tables.ts @@ -1105,6 +1105,17 @@ export interface UserSubmittedImage { validatedAt: number | null; } +/** FTS5 trigram index over User's searchable columns (external content table, + * kept in sync with triggers). Only meant for reading: filter with + * `match` and join `rowid` to `User.id`. */ +export interface UserSearch { + rowid: GeneratedAlways; + username: GeneratedAlways; + inGameName: GeneratedAlways; + discordUniqueName: GeneratedAlways; + customUrl: GeneratedAlways; +} + export interface UserWeapon { createdAt: Generated; isFavorite: Generated; @@ -1462,6 +1473,7 @@ export interface DB { UnvalidatedUserSubmittedImage: UnvalidatedUserSubmittedImage; UnvalidatedVideo: UnvalidatedVideo; User: User; + UserSearch: UserSearch; UserResultHighlight: UserResultHighlight; UserSubmittedImage: UserSubmittedImage; UserWeapon: UserWeapon; diff --git a/app/features/user-page/UserRepository.server.ts b/app/features/user-page/UserRepository.server.ts index 55c4202c9..be77109c5 100644 --- a/app/features/user-page/UserRepository.server.ts +++ b/app/features/user-page/UserRepository.server.ts @@ -812,6 +812,11 @@ export async function search({ const includeExactMatches = query.length > 1; + // the trigram index needs at least 3 characters and can't replicate + // LIKE wildcard semantics, those queries fall back to scanning User + const canUseSearchIndex = + query.length >= 3 && !query.includes("%") && !query.includes("_"); + let dbQuery = db .selectFrom("User") .leftJoin("PlusTier", "PlusTier.userId", "User.id") @@ -824,6 +829,16 @@ export async function search({ ), ); + if (canUseSearchIndex) { + // UserSearch match prefilters candidates via the trigram index (it + // matches a superset of the LIKE conditions, which stay above as the + // source of truth so results are identical to the fallback path) + const ftsPhrase = `"${query.replaceAll('"', '""')}"`; + dbQuery = dbQuery + .innerJoin("UserSearch", "UserSearch.rowid", "User.id") + .where(sql`"UserSearch" match ${ftsPhrase}`); + } + if (includeExactMatches) { dbQuery = dbQuery.orderBy( (eb) => @@ -837,19 +852,23 @@ export async function search({ ); } - return dbQuery - .orderBy( - (eb) => - eb - .case() - .when("PlusTier.tier", "is", null) - .then(4) - .else(eb.ref("PlusTier.tier")) - .end(), - "asc", - ) - .limit(limit) - .execute(); + return ( + dbQuery + .orderBy( + (eb) => + eb + .case() + .when("PlusTier.tier", "is", null) + .then(4) + .else(eb.ref("PlusTier.tier")) + .end(), + "asc", + ) + // deterministic order for ties so both query paths return the same rows + .orderBy("User.id", "asc") + .limit(limit) + .execute() + ); } export function searchExact(args: { diff --git a/app/utils/Test.ts b/app/utils/Test.ts index cb0038fb4..83ce6563e 100644 --- a/app/utils/Test.ts +++ b/app/utils/Test.ts @@ -208,9 +208,20 @@ async function authHeader( * }); */ export const dbReset = () => { + // virtual tables and their shadow tables (e.g. UserSearch_data) can not be + // deleted from directly; the fts index stays in sync via the User triggers const tables = sql .prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE 'migrations';", + `SELECT name FROM sqlite_master + WHERE type='table' + AND name NOT LIKE 'sqlite_%' + AND name NOT LIKE 'migrations' + AND sql NOT LIKE 'CREATE VIRTUAL TABLE%' + AND NOT EXISTS ( + SELECT 1 FROM sqlite_master AS vt + WHERE vt.sql LIKE 'CREATE VIRTUAL TABLE%' + AND sqlite_master.name LIKE vt.name || '_%' + );`, ) .all() as { name: string }[]; diff --git a/db-test.sqlite3 b/db-test.sqlite3 index 805389fe3..40e455134 100644 Binary files a/db-test.sqlite3 and b/db-test.sqlite3 differ diff --git a/e2e/seeds/db-seed-AB_RR.sqlite3 b/e2e/seeds/db-seed-AB_RR.sqlite3 index d1bb5ad5c..c67682344 100644 Binary files a/e2e/seeds/db-seed-AB_RR.sqlite3 and b/e2e/seeds/db-seed-AB_RR.sqlite3 differ diff --git a/e2e/seeds/db-seed-DEFAULT.sqlite3 b/e2e/seeds/db-seed-DEFAULT.sqlite3 index 81d6e08ed..671ee9217 100644 Binary files a/e2e/seeds/db-seed-DEFAULT.sqlite3 and b/e2e/seeds/db-seed-DEFAULT.sqlite3 differ diff --git a/e2e/seeds/db-seed-FINALIZED_BRACKET.sqlite3 b/e2e/seeds/db-seed-FINALIZED_BRACKET.sqlite3 index 33fb07531..393196449 100644 Binary files a/e2e/seeds/db-seed-FINALIZED_BRACKET.sqlite3 and b/e2e/seeds/db-seed-FINALIZED_BRACKET.sqlite3 differ diff --git a/e2e/seeds/db-seed-IN_SQ_MATCH.sqlite3 b/e2e/seeds/db-seed-IN_SQ_MATCH.sqlite3 index 8cde179d3..2d154a377 100644 Binary files a/e2e/seeds/db-seed-IN_SQ_MATCH.sqlite3 and b/e2e/seeds/db-seed-IN_SQ_MATCH.sqlite3 differ diff --git a/e2e/seeds/db-seed-NO_SCRIMS.sqlite3 b/e2e/seeds/db-seed-NO_SCRIMS.sqlite3 index 31720ce8a..19832d814 100644 Binary files a/e2e/seeds/db-seed-NO_SCRIMS.sqlite3 and b/e2e/seeds/db-seed-NO_SCRIMS.sqlite3 differ diff --git a/e2e/seeds/db-seed-NO_SQ_GROUPS.sqlite3 b/e2e/seeds/db-seed-NO_SQ_GROUPS.sqlite3 index 09edd4561..de76dc3fd 100644 Binary files a/e2e/seeds/db-seed-NO_SQ_GROUPS.sqlite3 and b/e2e/seeds/db-seed-NO_SQ_GROUPS.sqlite3 differ diff --git a/e2e/seeds/db-seed-NO_TOURNAMENT_TEAMS.sqlite3 b/e2e/seeds/db-seed-NO_TOURNAMENT_TEAMS.sqlite3 index 0fa21e349..8cb418984 100644 Binary files a/e2e/seeds/db-seed-NO_TOURNAMENT_TEAMS.sqlite3 and b/e2e/seeds/db-seed-NO_TOURNAMENT_TEAMS.sqlite3 differ diff --git a/e2e/seeds/db-seed-NZAP_IN_TEAM.sqlite3 b/e2e/seeds/db-seed-NZAP_IN_TEAM.sqlite3 index 60bb69544..90b73ff5f 100644 Binary files a/e2e/seeds/db-seed-NZAP_IN_TEAM.sqlite3 and b/e2e/seeds/db-seed-NZAP_IN_TEAM.sqlite3 differ diff --git a/e2e/seeds/db-seed-REG_OPEN.sqlite3 b/e2e/seeds/db-seed-REG_OPEN.sqlite3 index 48ca2e280..1c99fec5c 100644 Binary files a/e2e/seeds/db-seed-REG_OPEN.sqlite3 and b/e2e/seeds/db-seed-REG_OPEN.sqlite3 differ diff --git a/e2e/seeds/db-seed-SMALL_SOS.sqlite3 b/e2e/seeds/db-seed-SMALL_SOS.sqlite3 index a9ffa0ce1..d64a70bcb 100644 Binary files a/e2e/seeds/db-seed-SMALL_SOS.sqlite3 and b/e2e/seeds/db-seed-SMALL_SOS.sqlite3 differ diff --git a/e2e/seeds/db-seed-TEAM_MAP_PREFS.sqlite3 b/e2e/seeds/db-seed-TEAM_MAP_PREFS.sqlite3 index f81d08724..221aeff0e 100644 Binary files a/e2e/seeds/db-seed-TEAM_MAP_PREFS.sqlite3 and b/e2e/seeds/db-seed-TEAM_MAP_PREFS.sqlite3 differ diff --git a/migrations/148-user-search-fts.js b/migrations/148-user-search-fts.js new file mode 100644 index 000000000..8b7603dcc --- /dev/null +++ b/migrations/148-user-search-fts.js @@ -0,0 +1,50 @@ +export function up(db) { + db.transaction(() => { + // trigram-tokenized full text search index over the columns user search + // matches against, so substring (LIKE '%query%') searches can use an + // index instead of scanning the whole User table. + // external content table: rows are not stored twice, the index reads + // from User and is kept in sync by the triggers below. + db.prepare( + /* sql */ `create virtual table "UserSearch" using fts5( + "username", + "inGameName", + "discordUniqueName", + "customUrl", + content='User', + content_rowid='id', + tokenize='trigram' + )`, + ).run(); + + db.prepare( + /* sql */ `insert into "UserSearch"("UserSearch") values ('rebuild')`, + ).run(); + + db.prepare( + /* sql */ `create trigger "user_search_after_insert" after insert on "User" begin + insert into "UserSearch"(rowid, "username", "inGameName", "discordUniqueName", "customUrl") + values (new."id", new."username", new."inGameName", new."discordUniqueName", new."customUrl"); + end`, + ).run(); + + db.prepare( + /* sql */ `create trigger "user_search_after_delete" after delete on "User" begin + insert into "UserSearch"("UserSearch", rowid, "username", "inGameName", "discordUniqueName", "customUrl") + values ('delete', old."id", old."username", old."inGameName", old."discordUniqueName", old."customUrl"); + end`, + ).run(); + + // "username" is a generated column (coalesce of customName/discordName) + // and generated columns can not be listed in "update of", so the + // trigger watches its source columns instead + db.prepare( + /* sql */ `create trigger "user_search_after_update" after update of "customName", "discordName", "inGameName", "discordUniqueName", "customUrl" on "User" begin + insert into "UserSearch"("UserSearch", rowid, "username", "inGameName", "discordUniqueName", "customUrl") + values ('delete', old."id", old."username", old."inGameName", old."discordUniqueName", old."customUrl"); + insert into "UserSearch"(rowid, "username", "inGameName", "discordUniqueName", "customUrl") + values (new."id", new."username", new."inGameName", new."discordUniqueName", new."customUrl"); + end`, + ).run(); + })(); +}