Leahnaya/BOSS
1
0
Fork 0
mirror of https://github.com/PretendoNetwork/BOSS.git synced 2026-03-21 17:34:19 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add scheduled action system + halffinished SPR data clean procedure

Browse Source
This commit is contained in:
mrjvs 2025-09-16 19:43:19 +02:00
parent da6ace244f
commit c19feefcb7
5 changed files with 116 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
package-lock.json generated
View File

@ -15,6 +15,7 @@
"@typegoose/auto-increment": "^3.6.1",
"boss-js": "github:PretendoNetwork/boss-js",
"commander": "^14.0.0",
"cron": "^4.3.3",
"dicer": "^0.3.1",
"dotenv": "^16.4.7",
"express": "^5.1.0",
@ -3237,6 +3238,12 @@
"@types/node": "*"
}
},
"node_modules/@types/luxon": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz",
"integrity": "sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==",
"license": "MIT"
},
"node_modules/@types/mime": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
@ -4588,6 +4595,19 @@
"node": ">=6.6.0"
}
},
"node_modules/cron": {
"version": "4.3.3",
"resolved": "https://registry.npmjs.org/cron/-/cron-4.3.3.tgz",
"integrity": "sha512-B/CJj5yL3sjtlun6RtYHvoSB26EmQ2NUmhq9ZiJSyKIM4K/fqfh9aelDFlIayD2YMeFZqWLi9hHV+c+pq2Djkw==",
"license": "MIT",
"dependencies": {
"@types/luxon": "~3.7.0",
"luxon": "~3.7.0"
},
"engines": {
"node": ">=18.x"
}
},
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@ -7132,6 +7152,15 @@
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"dev": true
},
"node_modules/luxon": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz",
"integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==",
"license": "MIT",
"engines": {
"node": ">=12"
}
},
"node_modules/magic-string": {
"version": "0.30.18",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.18.tgz",
@ -12111,6 +12140,11 @@
"@types/node": "*"
}
},
"@types/luxon": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz",
"integrity": "sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg=="
},
"@types/mime": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
@ -12965,6 +12999,15 @@
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz",
"integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="
},
"cron": {
"version": "4.3.3",
"resolved": "https://registry.npmjs.org/cron/-/cron-4.3.3.tgz",
"integrity": "sha512-B/CJj5yL3sjtlun6RtYHvoSB26EmQ2NUmhq9ZiJSyKIM4K/fqfh9aelDFlIayD2YMeFZqWLi9hHV+c+pq2Djkw==",
"requires": {
"@types/luxon": "~3.7.0",
"luxon": "~3.7.0"
}
},
"cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@ -14710,6 +14753,11 @@
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"dev": true
},
"luxon": {
"version": "3.7.2",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz",
"integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="
},
"magic-string": {
"version": "0.30.18",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.18.tgz",

1
package.json
View File

@ -19,6 +19,7 @@
"@typegoose/auto-increment": "^3.6.1",
"boss-js": "github:PretendoNetwork/boss-js",
"commander": "^14.0.0",
"cron": "^4.3.3",
"dicer": "^0.3.1",
"dotenv": "^16.4.7",
"express": "^5.1.0",

19
src/database.ts
View File

@ -216,3 +216,22 @@ export async function getRandomCECData(pids: number[], gameID: number): Promise<
return null;
}
export async function deleteOldCECData(olderThan: Date, limit: number): Promise<{ _id: string; file_key: string }[]> {
verifyConnected();
const toDelete = await CECData.find({
created: {
$lt: olderThan.getTime()
}
}, { file_key: 1 }, { limit });
const ids = toDelete.map(v => v.data);
await CECData.deleteMany({
_id: {
$in: ids
}
});
return toDelete.map(v => ({ _id: v._id.toString(), file_key: v.file_key }));
}

44
src/scheduled.ts Normal file
View File

@ -0,0 +1,44 @@
import { CronJob } from 'cron';
import { logger } from './logger';
import { deleteOldCECData } from './database';
async function runCleanSprData(): Promise<void> {
const maxAgeMs = 14 * 24 * 60 * 60 * 1000; // 14 days
const timestampInPast = new Date(Date.now() - maxAgeMs);
const processingLimit = 1000; // S3 only allows 1k objects at a time
let totalRemoved = 0;
logger.info('Starting SPR data cleanup');
let hasDataToDelete = true;
while (hasDataToDelete) {
const deletedData = await deleteOldCECData(timestampInPast, processingLimit);
logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`);
// TODO CDN removal
totalRemoved += deletedData.length;
hasDataToDelete = deletedData.length < processingLimit;
}
logger.success(`Completed cleanup of ${totalRemoved}`);
}
function registerSchedule(schedule: string, name: string, fn: () => void | Promise<void>): void {
CronJob.from({
cronTime: schedule,
onTick: async () => {
try {
const result = fn();
await result;
} catch (err) {
logger.error(`Error in schedule ${name}: ${err}`);
}
},
start: true
});
logger.info(`Added schedule ${name} for ${schedule}`);
}
export async function setupScheduler(): Promise<void> {
registerSchedule('0 2 * * *', 'clean-spr-data', runCleanSprData);
}

4
src/server.ts
View File

@ -12,6 +12,7 @@ import npdi from '@/services/npdi';
import npfl from '@/services/npfl';
import npdl from '@/services/npdl';
import spr from '@/services/spr';
import { setupScheduler } from './scheduled';
process.title = 'Pretendo - BOSS';
process.on('SIGTERM', () => {
@ -73,6 +74,9 @@ async function main(): Promise<void> {
await connectDatabase();
logger.success('Database connected');
await setupScheduler();
logger.success('Scheduler started');
await startGRPCServer();
logger.success(`gRPC server started at address ${config.grpc.boss.address}:${config.grpc.boss.port}`);