feat: add scheduled action system + halffinished SPR data clean procedure

src/database.ts

@@ -216,3 +216,22 @@ export async function getRandomCECData(pids: number[], gameID: number): Promise<
 
 	return null;
 }
+
+export async function deleteOldCECData(olderThan: Date, limit: number): Promise<{ _id: string; file_key: string }[]> {
+	verifyConnected();
+
+	const toDelete = await CECData.find({
+		created: {
+			$lt: olderThan.getTime()
+		}
+	}, { file_key: 1 }, { limit });
+	const ids = toDelete.map(v => v.data);
+
+	await CECData.deleteMany({
+		_id: {
+			$in: ids
+		}
+	});
+
+	return toDelete.map(v => ({ _id: v._id.toString(), file_key: v.file_key }));
+}

src/scheduled.ts

@@ -0,0 +1,44 @@
+import { CronJob } from 'cron';
+import { logger } from './logger';
+import { deleteOldCECData } from './database';
+
+async function runCleanSprData(): Promise<void> {
+	const maxAgeMs = 14 * 24 * 60 * 60 * 1000; // 14 days
+	const timestampInPast = new Date(Date.now() - maxAgeMs);
+	const processingLimit = 1000; // S3 only allows 1k objects at a time
+	let totalRemoved = 0;
+
+	logger.info('Starting SPR data cleanup');
+	let hasDataToDelete = true;
+	while (hasDataToDelete) {
+		const deletedData = await deleteOldCECData(timestampInPast, processingLimit);
+		logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`);
+
+		// TODO CDN removal
+
+		totalRemoved += deletedData.length;
+		hasDataToDelete = deletedData.length < processingLimit;
+	}
+
+	logger.success(`Completed cleanup of ${totalRemoved}`);
+}
+
+function registerSchedule(schedule: string, name: string, fn: () => void | Promise<void>): void {
+	CronJob.from({
+		cronTime: schedule,
+		onTick: async () => {
+			try {
+				const result = fn();
+				await result;
+			} catch (err) {
+				logger.error(`Error in schedule ${name}: ${err}`);
+			}
+		},
+		start: true
+	});
+	logger.info(`Added schedule ${name} for ${schedule}`);
+}
+
+export async function setupScheduler(): Promise<void> {
+	registerSchedule('0 2 * * *', 'clean-spr-data', runCleanSprData);
+}

src/server.ts

@@ -12,6 +12,7 @@ import npdi from '@/services/npdi';
 import npfl from '@/services/npfl';
 import npdl from '@/services/npdl';
 import spr from '@/services/spr';
+import { setupScheduler } from './scheduled';
 
 process.title = 'Pretendo - BOSS';
 process.on('SIGTERM', () => {
@@ -73,6 +74,9 @@ async function main(): Promise<void> {
 	await connectDatabase();
 	logger.success('Database connected');
 
+	await setupScheduler();
+	logger.success('Scheduler started');
+
 	await startGRPCServer();
 	logger.success(`gRPC server started at address ${config.grpc.boss.address}:${config.grpc.boss.port}`);