From c19feefcb79548ba55cabd9d5e1900b6235b950c Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 16 Sep 2025 19:43:19 +0200 Subject: [PATCH] feat: add scheduled action system + halffinished SPR data clean procedure --- package-lock.json | 48 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/database.ts | 19 +++++++++++++++++++ src/scheduled.ts | 44 +++++++++++++++++++++++++++++++++++++++++++ src/server.ts | 4 ++++ 5 files changed, 116 insertions(+) create mode 100644 src/scheduled.ts diff --git a/package-lock.json b/package-lock.json index 095f457..498240b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@typegoose/auto-increment": "^3.6.1", "boss-js": "github:PretendoNetwork/boss-js", "commander": "^14.0.0", + "cron": "^4.3.3", "dicer": "^0.3.1", "dotenv": "^16.4.7", "express": "^5.1.0", @@ -3237,6 +3238,12 @@ "@types/node": "*" } }, + "node_modules/@types/luxon": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz", + "integrity": "sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==", + "license": "MIT" + }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -4588,6 +4595,19 @@ "node": ">=6.6.0" } }, + "node_modules/cron": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/cron/-/cron-4.3.3.tgz", + "integrity": "sha512-B/CJj5yL3sjtlun6RtYHvoSB26EmQ2NUmhq9ZiJSyKIM4K/fqfh9aelDFlIayD2YMeFZqWLi9hHV+c+pq2Djkw==", + "license": "MIT", + "dependencies": { + "@types/luxon": "~3.7.0", + "luxon": "~3.7.0" + }, + "engines": { + "node": ">=18.x" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -7132,6 +7152,15 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "dev": true }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.18", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.18.tgz", @@ -12111,6 +12140,11 @@ "@types/node": "*" } }, + "@types/luxon": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz", + "integrity": "sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==" + }, "@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -12965,6 +12999,15 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==" }, + "cron": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/cron/-/cron-4.3.3.tgz", + "integrity": "sha512-B/CJj5yL3sjtlun6RtYHvoSB26EmQ2NUmhq9ZiJSyKIM4K/fqfh9aelDFlIayD2YMeFZqWLi9hHV+c+pq2Djkw==", + "requires": { + "@types/luxon": "~3.7.0", + "luxon": "~3.7.0" + } + }, "cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -14710,6 +14753,11 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "dev": true }, + "luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==" + }, "magic-string": { "version": "0.30.18", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.18.tgz", diff --git a/package.json b/package.json index 66ff6ef..fe9acb9 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@typegoose/auto-increment": "^3.6.1", "boss-js": "github:PretendoNetwork/boss-js", "commander": "^14.0.0", + "cron": "^4.3.3", "dicer": "^0.3.1", "dotenv": "^16.4.7", "express": "^5.1.0", diff --git a/src/database.ts b/src/database.ts index 60d83ce..919f69c 100644 --- a/src/database.ts +++ b/src/database.ts @@ -216,3 +216,22 @@ export async function getRandomCECData(pids: number[], gameID: number): Promise< return null; } + +export async function deleteOldCECData(olderThan: Date, limit: number): Promise<{ _id: string; file_key: string }[]> { + verifyConnected(); + + const toDelete = await CECData.find({ + created: { + $lt: olderThan.getTime() + } + }, { file_key: 1 }, { limit }); + const ids = toDelete.map(v => v.data); + + await CECData.deleteMany({ + _id: { + $in: ids + } + }); + + return toDelete.map(v => ({ _id: v._id.toString(), file_key: v.file_key })); +} diff --git a/src/scheduled.ts b/src/scheduled.ts new file mode 100644 index 0000000..f8e37a9 --- /dev/null +++ b/src/scheduled.ts @@ -0,0 +1,44 @@ +import { CronJob } from 'cron'; +import { logger } from './logger'; +import { deleteOldCECData } from './database'; + +async function runCleanSprData(): Promise { + const maxAgeMs = 14 * 24 * 60 * 60 * 1000; // 14 days + const timestampInPast = new Date(Date.now() - maxAgeMs); + const processingLimit = 1000; // S3 only allows 1k objects at a time + let totalRemoved = 0; + + logger.info('Starting SPR data cleanup'); + let hasDataToDelete = true; + while (hasDataToDelete) { + const deletedData = await deleteOldCECData(timestampInPast, processingLimit); + logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`); + + // TODO CDN removal + + totalRemoved += deletedData.length; + hasDataToDelete = deletedData.length < processingLimit; + } + + logger.success(`Completed cleanup of ${totalRemoved}`); +} + +function registerSchedule(schedule: string, name: string, fn: () => void | Promise): void { + CronJob.from({ + cronTime: schedule, + onTick: async () => { + try { + const result = fn(); + await result; + } catch (err) { + logger.error(`Error in schedule ${name}: ${err}`); + } + }, + start: true + }); + logger.info(`Added schedule ${name} for ${schedule}`); +} + +export async function setupScheduler(): Promise { + registerSchedule('0 2 * * *', 'clean-spr-data', runCleanSprData); +} diff --git a/src/server.ts b/src/server.ts index b4acf0f..307721e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -12,6 +12,7 @@ import npdi from '@/services/npdi'; import npfl from '@/services/npfl'; import npdl from '@/services/npdl'; import spr from '@/services/spr'; +import { setupScheduler } from './scheduled'; process.title = 'Pretendo - BOSS'; process.on('SIGTERM', () => { @@ -73,6 +74,9 @@ async function main(): Promise { await connectDatabase(); logger.success('Database connected'); + await setupScheduler(); + logger.success('Scheduler started'); + await startGRPCServer(); logger.success(`gRPC server started at address ${config.grpc.boss.address}:${config.grpc.boss.port}`);