Optimize user search by introducing fts5 index

app/db/tables.ts

@@ -1105,6 +1105,17 @@ export interface UserSubmittedImage {
 	validatedAt: number | null;
 }
 
+/** FTS5 trigram index over User's searchable columns (external content table,
+ * kept in sync with triggers). Only meant for reading: filter with
+ * `match` and join `rowid` to `User.id`. */
+export interface UserSearch {
+	rowid: GeneratedAlways<number>;
+	username: GeneratedAlways<string | null>;
+	inGameName: GeneratedAlways<string | null>;
+	discordUniqueName: GeneratedAlways<string | null>;
+	customUrl: GeneratedAlways<string | null>;
+}
+
 export interface UserWeapon {
 	createdAt: Generated<number>;
 	isFavorite: Generated<DBBoolean>;
@@ -1462,6 +1473,7 @@ export interface DB {
 	UnvalidatedUserSubmittedImage: UnvalidatedUserSubmittedImage;
 	UnvalidatedVideo: UnvalidatedVideo;
 	User: User;
+	UserSearch: UserSearch;
 	UserResultHighlight: UserResultHighlight;
 	UserSubmittedImage: UserSubmittedImage;
 	UserWeapon: UserWeapon;

app/features/user-page/UserRepository.server.ts

@@ -812,6 +812,11 @@ export async function search({
 
 	const includeExactMatches = query.length > 1;
 
+	// the trigram index needs at least 3 characters and can't replicate
+	// LIKE wildcard semantics, those queries fall back to scanning User
+	const canUseSearchIndex =
+		query.length >= 3 && !query.includes("%") && !query.includes("_");
+
 	let dbQuery = db
 		.selectFrom("User")
 		.leftJoin("PlusTier", "PlusTier.userId", "User.id")
@@ -824,6 +829,16 @@ export async function search({
 			),
 		);
 
+	if (canUseSearchIndex) {
+		// UserSearch match prefilters candidates via the trigram index (it
+		// matches a superset of the LIKE conditions, which stay above as the
+		// source of truth so results are identical to the fallback path)
+		const ftsPhrase = `"${query.replaceAll('"', '""')}"`;
+		dbQuery = dbQuery
+			.innerJoin("UserSearch", "UserSearch.rowid", "User.id")
+			.where(sql<boolean>`"UserSearch" match ${ftsPhrase}`);
+	}
+
 	if (includeExactMatches) {
 		dbQuery = dbQuery.orderBy(
 			(eb) =>
@@ -837,19 +852,23 @@ export async function search({
 		);
 	}
 
-	return dbQuery
-		.orderBy(
-			(eb) =>
-				eb
-					.case()
-					.when("PlusTier.tier", "is", null)
-					.then(4)
-					.else(eb.ref("PlusTier.tier"))
-					.end(),
-			"asc",
-		)
-		.limit(limit)
-		.execute();
+	return (
+		dbQuery
+			.orderBy(
+				(eb) =>
+					eb
+						.case()
+						.when("PlusTier.tier", "is", null)
+						.then(4)
+						.else(eb.ref("PlusTier.tier"))
+						.end(),
+				"asc",
+			)
+			// deterministic order for ties so both query paths return the same rows
+			.orderBy("User.id", "asc")
+			.limit(limit)
+			.execute()
+	);
 }
 
 export function searchExact(args: {

app/utils/Test.ts

@@ -208,9 +208,20 @@ async function authHeader(
  * });
  */
 export const dbReset = () => {
+	// virtual tables and their shadow tables (e.g. UserSearch_data) can not be
+	// deleted from directly; the fts index stays in sync via the User triggers
 	const tables = sql
 		.prepare(
-			"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE 'migrations';",
+			`SELECT name FROM sqlite_master
+			WHERE type='table'
+			AND name NOT LIKE 'sqlite_%'
+			AND name NOT LIKE 'migrations'
+			AND sql NOT LIKE 'CREATE VIRTUAL TABLE%'
+			AND NOT EXISTS (
+				SELECT 1 FROM sqlite_master AS vt
+				WHERE vt.sql LIKE 'CREATE VIRTUAL TABLE%'
+				AND sqlite_master.name LIKE vt.name || '_%'
+			);`,
 		)
 		.all() as { name: string }[];
 

migrations/148-user-search-fts.js

@@ -0,0 +1,50 @@
+export function up(db) {
+	db.transaction(() => {
+		// trigram-tokenized full text search index over the columns user search
+		// matches against, so substring (LIKE '%query%') searches can use an
+		// index instead of scanning the whole User table.
+		// external content table: rows are not stored twice, the index reads
+		// from User and is kept in sync by the triggers below.
+		db.prepare(
+			/* sql */ `create virtual table "UserSearch" using fts5(
+				"username",
+				"inGameName",
+				"discordUniqueName",
+				"customUrl",
+				content='User',
+				content_rowid='id',
+				tokenize='trigram'
+			)`,
+		).run();
+
+		db.prepare(
+			/* sql */ `insert into "UserSearch"("UserSearch") values ('rebuild')`,
+		).run();
+
+		db.prepare(
+			/* sql */ `create trigger "user_search_after_insert" after insert on "User" begin
+				insert into "UserSearch"(rowid, "username", "inGameName", "discordUniqueName", "customUrl")
+				values (new."id", new."username", new."inGameName", new."discordUniqueName", new."customUrl");
+			end`,
+		).run();
+
+		db.prepare(
+			/* sql */ `create trigger "user_search_after_delete" after delete on "User" begin
+				insert into "UserSearch"("UserSearch", rowid, "username", "inGameName", "discordUniqueName", "customUrl")
+				values ('delete', old."id", old."username", old."inGameName", old."discordUniqueName", old."customUrl");
+			end`,
+		).run();
+
+		// "username" is a generated column (coalesce of customName/discordName)
+		// and generated columns can not be listed in "update of", so the
+		// trigger watches its source columns instead
+		db.prepare(
+			/* sql */ `create trigger "user_search_after_update" after update of "customName", "discordName", "inGameName", "discordUniqueName", "customUrl" on "User" begin
+				insert into "UserSearch"("UserSearch", rowid, "username", "inGameName", "discordUniqueName", "customUrl")
+				values ('delete', old."id", old."username", old."inGameName", old."discordUniqueName", old."customUrl");
+				insert into "UserSearch"(rowid, "username", "inGameName", "discordUniqueName", "customUrl")
+				values (new."id", new."username", new."inGameName", new."discordUniqueName", new."customUrl");
+			end`,
+		).run();
+	})();
+}