sendou.ink/app/db/sql.ts

import { DatabaseSync } from "node:sqlite";
import { styleText } from "node:util";
import * as Sentry from "@sentry/react-router";
import { Kysely, type LogEvent } from "kysely";
import { format } from "sql-formatter";
import { Config } from "~/config";
import { ServerConfig } from "~/config.server";
import { logger } from "~/utils/logger";
import { roundToNDecimalPlaces } from "~/utils/number";
import { EmptyValuesNoopPlugin } from "./empty-values-noop-plugin";
import { NodeSqliteDialect } from "./node-sqlite-dialect";
import { FastParseJSONResultsPlugin } from "./parse-json-results-plugin";
import type { DB } from "./tables";
import { WriteTrackerPlugin } from "./write-tracker";

const sql = new DatabaseSync(
	ServerConfig.isTest ? ":memory:" : ServerConfig.dbPath,
);

if (ServerConfig.isTest) {
	applyMigratedSchema(sql);
}

sql.exec("PRAGMA journal_mode = WAL");
// The synchronous=NORMAL setting provides the best balance between performance and safety for most applications running in WAL mode.
// You lose durability across power lose with synchronous NORMAL in WAL mode, but that is not important for most applications.
// Transactions are still atomic, consistent, and isolated, which are the most important characteristics in most use cases.
// Source: https://sqlite.org/pragma.html
sql.exec("PRAGMA synchronous = NORMAL");
sql.exec("PRAGMA foreign_keys = ON");
sql.exec("PRAGMA busy_timeout = 5000");
// 64MB page cache (default is 2MB)
sql.exec("PRAGMA cache_size = -65536");
// lets reads come straight from the OS page cache without read() syscalls
// Source: https://sqlite.org/mmap.html
sql.exec("PRAGMA mmap_size = 3221225472");
// see https://sqlite.org/pragma.html#pragma_optimize — recommended for long-lived
// connections; pair with a periodic `PRAGMA optimize;` (see OptimizeDatabase routine)
sql.exec("PRAGMA optimize = 0x10002");

// Strips diacritics so accent-insensitive name searches are possible
// (e.g. "cafe" matches "Café"). Combined with LIKE's built-in ASCII
// case-insensitivity this also folds case for the resulting latin letters.
sql.function("unaccent", { deterministic: true }, (value) =>
	typeof value === "string"
		? value.normalize("NFD").replace(/\p{M}/gu, "")
		: value,
);

export const db = new Kysely<DB>({
	dialect: new NodeSqliteDialect({
		database: sql,
		cacheStatements: true,
	}),
	log,
	plugins: [
		new EmptyValuesNoopPlugin(),
		new FastParseJSONResultsPlugin(),
		new WriteTrackerPlugin(),
	],
});

// Every test worker gets its own in-memory database, built by replaying the
// schema of the migrated (and otherwise empty) file that scripts/ensure-test-db.ts
// creates in vitest's globalSetup. Replaying the DDL rather than copying the file
// is what `node:sqlite` allows: unlike better-sqlite3 it cannot deserialize a
// database into memory.
function applyMigratedSchema(target: DatabaseSync) {
	const source = new DatabaseSync("db-test.sqlite3", {
		readOnly: true,
		timeout: 5000,
	});

	try {
		// virtual tables create their own shadow tables (e.g. UserSearch_data), so
		// replaying those would fail on a table that already exists
		const statements = source
			.prepare(`
				SELECT sql FROM sqlite_master m
				WHERE sql IS NOT NULL
				AND name NOT LIKE 'sqlite_%'
				AND NOT EXISTS (
					SELECT 1 FROM sqlite_master AS vt
					WHERE vt.sql LIKE 'CREATE VIRTUAL TABLE%'
					AND m.name LIKE vt.name || '_%'
				)
				ORDER BY m.rowid
			`)
			.all() as Array<{ sql: string }>;

		for (const statement of statements) {
			target.exec(statement.sql);
		}
	} finally {
		source.close();
	}
}

function log(event: LogEvent) {
	if (Config.sentry.enabled && event.level === "query") {
		// Backdated span so the query nests under the active loader/action span
		// in Sentry's waterfall. `onlyIfParent: true` skips emission when there's
		// no active trace (e.g. cron routines), avoiding orphan root spans.
		Sentry.startInactiveSpan({
			name: event.query.sql,
			op: "db.sql.query",
			startTime: new Date(Date.now() - event.queryDurationMillis),
			onlyIfParent: true,
		}).end();
	}

	if (ServerConfig.sqlLog === "trunc" || ServerConfig.sqlLog === "full") {
		logQuery(event);
	} else {
		logError(event);
	}
}

function logQuery(event: LogEvent) {
	const isSelectQuery = Boolean((event.query.query as any).from?.froms);

	if (event.level === "query" && isSelectQuery) {
		const from = () =>
			(event.query.query as any).from.froms.map(
				// plain tables have the name under table, aliased tables and
				// subqueries under alias
				(f: any) => f.table?.identifier?.name ?? f.alias?.name ?? "unknown",
			);
		// biome-ignore lint/suspicious/noConsole: dev only
		console.log(styleText("blue", `-- SQLITE QUERY to "${from()}" --`));
		// biome-ignore lint/suspicious/noConsole: dev only
		console.log(
			styleText(
				millisToColor(event.queryDurationMillis),
				`${roundToNDecimalPlaces(event.queryDurationMillis, 1)}ms`,
			),
		);
		// biome-ignore lint/suspicious/noConsole: dev only
		console.log(formatSql(event.query.sql, event.query.parameters));
	} else {
		logError(event);
	}
}

function logError(event: LogEvent) {
	if (
		event.level === "error" &&
		// it seems that this error happens everytime something goes wrong inside transaction
		// my guess is that the transaction is already implicitly rolled back in the case of error
		// but kysely also does it explicitly -> fails because there is no transaction to rollback.
		// this `logError` function at least makes it so that due to that the error doesn't get logged
		// but of course the best solution would also avoid useless rollbacks, something for the future
		// btw this particular check is here just to avoid the double "no transaction is active" log
		!(event.error as any).message.includes("no transaction is active")
	) {
		logger.error(event.error);
	}
}

function millisToColor(millis: number) {
	if (millis < 1) {
		return "bgGreen";
	}
	if (millis < 5) {
		return "green";
	}
	if (millis < 50) {
		return "yellow";
	}
	return "red";
}

function formatSql(sql: string, params: readonly unknown[]) {
	const formatted = format(sql);

	const lines = formatted.split("\n");

	if (ServerConfig.sqlLog === "full" || lines.length <= 11) {
		return addParams(formatted, params);
	}

	const linesNotShown = lines.length - 10;

	return `${lines.slice(0, 10).join("\n")}\n... (${linesNotShown} more lines) ...\n`;
}

function addParams(sql: string, params: readonly unknown[]) {
	const coloredParams = params.map((param) =>
		styleText("yellow", JSON.stringify(param)),
	);

	return sql.replace(/\?/g, () => coloredParams.shift() || "");
}