Vacuum DB routine

app/entry.server.tsx

@@ -17,6 +17,7 @@ import {
 	everyHourAt00,
 	everyHourAt30,
 	everyTwoMinutes,
+	weekly,
 } from "./routines/list.server";
 import { loadAllDateFnsLocales } from "./utils/dates";
 import { IS_E2E_TEST_RUN } from "./utils/e2e";
@@ -108,6 +109,18 @@ if (!global.appStartSignal && ServerConfig.isProduction && !IS_E2E_TEST_RUN) {
 		}
 	});
 
+	// 9:00 AM Finnish time on Wednesdays, a quiet hour picked because vacuuming blocks
+	// writes for longer than the 5s busy_timeout
+	cron.schedule(
+		"0 9 * * 3",
+		async () => {
+			for (const routine of weekly) {
+				await routine.run();
+			}
+		},
+		{ timezone: "Europe/Helsinki" },
+	);
+
 	cron.schedule("*/2 * * * *", async () => {
 		for (const routine of everyTwoMinutes) {
 			await routine.run();

app/features/api-private/routes/run-routine.ts

@@ -8,13 +8,13 @@ export const action = async ({ request }: ActionFunctionArgs) => {
 
 	const routineName = (await request.formData()).get("name");
 
-	const { everyHourAt00, everyHourAt30, daily, everyTwoMinutes } = await import(
-		"~/routines/list.server"
-	);
+	const { everyHourAt00, everyHourAt30, daily, weekly, everyTwoMinutes } =
+		await import("~/routines/list.server");
 	const routine = [
 		...everyHourAt00,
 		...everyHourAt30,
 		...daily,
+		...weekly,
 		...everyTwoMinutes,
 	].find((routine) => routine.name === routineName);
 

app/routines/list.server.ts

@@ -21,6 +21,7 @@ import { SyncLiveStreamsRoutine } from "./syncLiveStreams";
 import { SyncSplatoonRotationsRoutine } from "./syncSplatoonRotations";
 import { SyncTournamentVodsRoutine } from "./syncTournamentVods";
 import { UpdatePatreonDataRoutine } from "./updatePatreonData";
+import { VacuumDatabaseRoutine } from "./vacuumDatabase";
 
 /** List of Routines that should occur hourly at XX:00 */
 export const everyHourAt00 = [
@@ -55,6 +56,9 @@ export const daily = [
 	OptimizeDatabaseRoutine,
 ];
 
+/** List of Routines that should occur weekly */
+export const weekly = [VacuumDatabaseRoutine];
+
 /** List of Routines that should occur every 2 minutes */
 export const everyTwoMinutes = [
 	SyncLiveStreamsRoutine,

app/routines/vacuumDatabase.ts

@@ -0,0 +1,48 @@
+import { CompiledQuery, sql } from "kysely";
+import { db } from "../db/sql";
+import { logger } from "../utils/logger";
+import { roundToNDecimalPlaces } from "../utils/number";
+import { Routine } from "./routine.server";
+
+const BYTES_IN_MB = 1024 * 1024;
+
+/**
+ * Rewrites the database file so the space that deletes and dropped columns leave behind as
+ * partially filled pages is given back to the disk. Ordinary traffic frees very little of it,
+ * the freelist stays in the single digit megabytes, so what this actually collects is the
+ * fragmentation left by migrations that drop a wide column or a large index.
+ *
+ * Readers are unaffected because WAL serves them the pre-vacuum snapshot, but writers block
+ * for the whole rewrite and `busy_timeout` is 5s, which the rewrite outlasts. It is scheduled
+ * for a quiet hour for that reason, and the write errors it can cause there are the reason it
+ * is weekly rather than daily.
+ */
+export const VacuumDatabaseRoutine = new Routine({
+	name: "VacuumDatabase",
+	func: async () => {
+		const sizeBefore = await databaseSizeInBytes();
+
+		await db.executeQuery(CompiledQuery.raw("VACUUM"));
+
+		const sizeAfter = await databaseSizeInBytes();
+
+		logger.info(
+			`VACUUM reclaimed ${inMb(sizeBefore - sizeAfter)}MB (${inMb(sizeBefore)}MB -> ${inMb(sizeAfter)}MB)`,
+		);
+	},
+});
+
+async function databaseSizeInBytes() {
+	const { rows } = await sql<{
+		page_count: number;
+		page_size: number;
+	}>`select (select * from pragma_page_count()) as page_count, (select * from pragma_page_size()) as page_size`.execute(
+		db,
+	);
+
+	return rows[0].page_count * rows[0].page_size;
+}
+
+function inMb(bytes: number) {
+	return roundToNDecimalPlaces(bytes / BYTES_IN_MB, 1);
+}

scripts/vacuum-db.ts

@@ -0,0 +1,9 @@
+import { ServerConfig } from "~/config.server";
+import { VacuumDatabaseRoutine } from "~/routines/vacuumDatabase";
+import { logger } from "~/utils/logger";
+
+logger.info(
+	`Vacuuming ${ServerConfig.dbPath}, writes are blocked until it finishes`,
+);
+
+await VacuumDatabaseRoutine.run();