From c19feefcb79548ba55cabd9d5e1900b6235b950c Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 16 Sep 2025 19:43:19 +0200 Subject: [PATCH 01/16] feat: add scheduled action system + halffinished SPR data clean procedure --- package-lock.json | 48 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/database.ts | 19 +++++++++++++++++++ src/scheduled.ts | 44 +++++++++++++++++++++++++++++++++++++++++++ src/server.ts | 4 ++++ 5 files changed, 116 insertions(+) create mode 100644 src/scheduled.ts diff --git a/package-lock.json b/package-lock.json index 095f457..498240b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@typegoose/auto-increment": "^3.6.1", "boss-js": "github:PretendoNetwork/boss-js", "commander": "^14.0.0", + "cron": "^4.3.3", "dicer": "^0.3.1", "dotenv": "^16.4.7", "express": "^5.1.0", @@ -3237,6 +3238,12 @@ "@types/node": "*" } }, + "node_modules/@types/luxon": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz", + "integrity": "sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==", + "license": "MIT" + }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -4588,6 +4595,19 @@ "node": ">=6.6.0" } }, + "node_modules/cron": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/cron/-/cron-4.3.3.tgz", + "integrity": "sha512-B/CJj5yL3sjtlun6RtYHvoSB26EmQ2NUmhq9ZiJSyKIM4K/fqfh9aelDFlIayD2YMeFZqWLi9hHV+c+pq2Djkw==", + "license": "MIT", + "dependencies": { + "@types/luxon": "~3.7.0", + "luxon": "~3.7.0" + }, + "engines": { + "node": ">=18.x" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -7132,6 +7152,15 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "dev": true }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.18", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.18.tgz", @@ -12111,6 +12140,11 @@ "@types/node": "*" } }, + "@types/luxon": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz", + "integrity": "sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==" + }, "@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -12965,6 +12999,15 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==" }, + "cron": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/cron/-/cron-4.3.3.tgz", + "integrity": "sha512-B/CJj5yL3sjtlun6RtYHvoSB26EmQ2NUmhq9ZiJSyKIM4K/fqfh9aelDFlIayD2YMeFZqWLi9hHV+c+pq2Djkw==", + "requires": { + "@types/luxon": "~3.7.0", + "luxon": "~3.7.0" + } + }, "cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -14710,6 +14753,11 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "dev": true }, + "luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==" + }, "magic-string": { "version": "0.30.18", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.18.tgz", diff --git a/package.json b/package.json index 66ff6ef..fe9acb9 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@typegoose/auto-increment": "^3.6.1", "boss-js": "github:PretendoNetwork/boss-js", "commander": "^14.0.0", + "cron": "^4.3.3", "dicer": "^0.3.1", "dotenv": "^16.4.7", "express": "^5.1.0", diff --git a/src/database.ts b/src/database.ts index 60d83ce..919f69c 100644 --- a/src/database.ts +++ b/src/database.ts @@ -216,3 +216,22 @@ export async function getRandomCECData(pids: number[], gameID: number): Promise< return null; } + +export async function deleteOldCECData(olderThan: Date, limit: number): Promise<{ _id: string; file_key: string }[]> { + verifyConnected(); + + const toDelete = await CECData.find({ + created: { + $lt: olderThan.getTime() + } + }, { file_key: 1 }, { limit }); + const ids = toDelete.map(v => v.data); + + await CECData.deleteMany({ + _id: { + $in: ids + } + }); + + return toDelete.map(v => ({ _id: v._id.toString(), file_key: v.file_key })); +} diff --git a/src/scheduled.ts b/src/scheduled.ts new file mode 100644 index 0000000..f8e37a9 --- /dev/null +++ b/src/scheduled.ts @@ -0,0 +1,44 @@ +import { CronJob } from 'cron'; +import { logger } from './logger'; +import { deleteOldCECData } from './database'; + +async function runCleanSprData(): Promise { + const maxAgeMs = 14 * 24 * 60 * 60 * 1000; // 14 days + const timestampInPast = new Date(Date.now() - maxAgeMs); + const processingLimit = 1000; // S3 only allows 1k objects at a time + let totalRemoved = 0; + + logger.info('Starting SPR data cleanup'); + let hasDataToDelete = true; + while (hasDataToDelete) { + const deletedData = await deleteOldCECData(timestampInPast, processingLimit); + logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`); + + // TODO CDN removal + + totalRemoved += deletedData.length; + hasDataToDelete = deletedData.length < processingLimit; + } + + logger.success(`Completed cleanup of ${totalRemoved}`); +} + +function registerSchedule(schedule: string, name: string, fn: () => void | Promise): void { + CronJob.from({ + cronTime: schedule, + onTick: async () => { + try { + const result = fn(); + await result; + } catch (err) { + logger.error(`Error in schedule ${name}: ${err}`); + } + }, + start: true + }); + logger.info(`Added schedule ${name} for ${schedule}`); +} + +export async function setupScheduler(): Promise { + registerSchedule('0 2 * * *', 'clean-spr-data', runCleanSprData); +} diff --git a/src/server.ts b/src/server.ts index b4acf0f..307721e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -12,6 +12,7 @@ import npdi from '@/services/npdi'; import npfl from '@/services/npfl'; import npdl from '@/services/npdl'; import spr from '@/services/spr'; +import { setupScheduler } from './scheduled'; process.title = 'Pretendo - BOSS'; process.on('SIGTERM', () => { @@ -73,6 +74,9 @@ async function main(): Promise { await connectDatabase(); logger.success('Database connected'); + await setupScheduler(); + logger.success('Scheduler started'); + await startGRPCServer(); logger.success(`gRPC server started at address ${config.grpc.boss.address}:${config.grpc.boss.port}`); From 1d0bacfb9d30d7506b874b649ea16ecf6ffbb300 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 16 Sep 2025 19:45:58 +0200 Subject: [PATCH 02/16] feat: add config option to enable automatic cleanup --- README.md | 63 ++++++++++++++++++++++--------------------- src/config-manager.ts | 3 ++- src/scheduled.ts | 5 +++- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 1e0e814..a3cd8c6 100644 --- a/README.md +++ b/README.md @@ -11,37 +11,38 @@ Handles all BOSS (Background Online Storage Service) related tasks for the Prete Configurations are loaded through environment variables. `.env` files are supported. -| Environment variable | Description | Default | -| -------------------------------------------- | ----------------------------------------------------------------- | --------------------------------------------- | -| `PN_BOSS_CONFIG_HTTP_PORT` | The HTTP port the server listens on | None | -| `PN_BOSS_CONFIG_LOG_FORMAT` | What logging format to use, possible options: `pretty` or `json` | `pretty` | -| `PN_BOSS_CONFIG_LOG_LEVEL` | What log level to use | `info` | -| `PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY` | The BOSS WiiU AES key, needs to be dumped from a console | None | -| `PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY` | The BOSS WiiU HMAC key, needs to be dumped from a console | None | -| `PN_BOSS_CONFIG_BOSS_3DS_AES_KEY` | The BOSS 3DS AES key, needs to be dumped from a console | None | -| `PN_BOSS_CONFIG_MONGO_CONNECTION_STRING` | MongoDB connection string | None | -| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_ADDRESS` | Address for the GRPC server to listen on | None | -| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_PORT` | Port for the GRPC server to listen on | None | -| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_API_KEY` | API key that services will use to connect to the BOSS GRPC server | None | -| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_ADDRESS` | Address of the account GRPC server | None | -| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_PORT` | Port of the account GRPC server | None | -| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_API_KEY` | API key of the account GRPC server | None | -| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_ADDRESS` | Address of the friends GRPC server | None | -| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_PORT` | Port of the friends GRPC server | None | -| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_API_KEY` | API key of the friends GRPC server | None | -| `PN_BOSS_CONFIG_S3_ENDPOINT` | S3 server endpoint | None | -| `PN_BOSS_CONFIG_S3_REGION` | S3 server region | None | -| `PN_BOSS_CONFIG_S3_BUCKET` | S3 server bucket | None | -| `PN_BOSS_CONFIG_S3_ACCESS_KEY` | S3 access key | None | -| `PN_BOSS_CONFIG_S3_ACCESS_SECRET` | S3 access key secret | None | -| `PN_BOSS_CONFIG_CDN_DISK_PATH` | Storage path for the CDN, use as alternative for S3 | None | -| `PN_BOSS_CONFIG_STREETPASS_RELAY_ENABLED` | Should Streetpass Relay be enabled? | `false` | -| `PN_BOSS_CONFIG_DOMAINS_NPDI` | What domain should the NPDI component use? | `npdi.cdn.pretendo.cc` | -| `PN_BOSS_CONFIG_DOMAINS_NPDL` | What domain should the NPDL component use? | `npdl.cdn.pretendo.cc` | -| `PN_BOSS_CONFIG_DOMAINS_NPFL` | What domain should the NPFL component use? | `npfl.c.app.pretendo.cc` | -| `PN_BOSS_CONFIG_DOMAINS_NPPL` | What domain should the NPPL component use? | `nppl.app.pretendo.cc,nppl.c.app.pretendo.cc` | -| `PN_BOSS_CONFIG_DOMAINS_NPTS` | What domain should the NPTS component use? | `npts.app.pretendo.cc` | -| `PN_BOSS_CONFIG_DOMAINS_SPR` | What domain should the SPR component use? | `service.spr.app.pretendo.cc` | +| Environment variable | Description | Default | +| ------------------------------------------------ | ----------------------------------------------------------------- | --------------------------------------------- | +| `PN_BOSS_CONFIG_HTTP_PORT` | The HTTP port the server listens on | None | +| `PN_BOSS_CONFIG_LOG_FORMAT` | What logging format to use, possible options: `pretty` or `json` | `pretty` | +| `PN_BOSS_CONFIG_LOG_LEVEL` | What log level to use | `info` | +| `PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY` | The BOSS WiiU AES key, needs to be dumped from a console | None | +| `PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY` | The BOSS WiiU HMAC key, needs to be dumped from a console | None | +| `PN_BOSS_CONFIG_BOSS_3DS_AES_KEY` | The BOSS 3DS AES key, needs to be dumped from a console | None | +| `PN_BOSS_CONFIG_MONGO_CONNECTION_STRING` | MongoDB connection string | None | +| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_ADDRESS` | Address for the GRPC server to listen on | None | +| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_PORT` | Port for the GRPC server to listen on | None | +| `PN_BOSS_CONFIG_GRPC_BOSS_SERVER_API_KEY` | API key that services will use to connect to the BOSS GRPC server | None | +| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_ADDRESS` | Address of the account GRPC server | None | +| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_PORT` | Port of the account GRPC server | None | +| `PN_BOSS_CONFIG_GRPC_ACCOUNT_SERVER_API_KEY` | API key of the account GRPC server | None | +| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_ADDRESS` | Address of the friends GRPC server | None | +| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_PORT` | Port of the friends GRPC server | None | +| `PN_BOSS_CONFIG_GRPC_FRIENDS_SERVER_API_KEY` | API key of the friends GRPC server | None | +| `PN_BOSS_CONFIG_S3_ENDPOINT` | S3 server endpoint | None | +| `PN_BOSS_CONFIG_S3_REGION` | S3 server region | None | +| `PN_BOSS_CONFIG_S3_BUCKET` | S3 server bucket | None | +| `PN_BOSS_CONFIG_S3_ACCESS_KEY` | S3 access key | None | +| `PN_BOSS_CONFIG_S3_ACCESS_SECRET` | S3 access key secret | None | +| `PN_BOSS_CONFIG_CDN_DISK_PATH` | Storage path for the CDN, use as alternative for S3 | None | +| `PN_BOSS_CONFIG_STREETPASS_RELAY_ENABLED` | Should Streetpass Relay be enabled? | `false` | +| `PN_BOSS_CONFIG_STREETPASS_RELAY_CLEAN_OLD_DATA` | Should old Streetpass Relay data be automatically cleaned up? | `false` | +| `PN_BOSS_CONFIG_DOMAINS_NPDI` | What domain should the NPDI component use? | `npdi.cdn.pretendo.cc` | +| `PN_BOSS_CONFIG_DOMAINS_NPDL` | What domain should the NPDL component use? | `npdl.cdn.pretendo.cc` | +| `PN_BOSS_CONFIG_DOMAINS_NPFL` | What domain should the NPFL component use? | `npfl.c.app.pretendo.cc` | +| `PN_BOSS_CONFIG_DOMAINS_NPPL` | What domain should the NPPL component use? | `nppl.app.pretendo.cc,nppl.c.app.pretendo.cc` | +| `PN_BOSS_CONFIG_DOMAINS_NPTS` | What domain should the NPTS component use? | `npts.app.pretendo.cc` | +| `PN_BOSS_CONFIG_DOMAINS_SPR` | What domain should the SPR component use? | `service.spr.app.pretendo.cc` | ## S3 server The S3 server is optional, you can set `PN_BOSS_CONFIG_CDN_DISK_PATH` if you want to use a local folder as CDN source instead. diff --git a/src/config-manager.ts b/src/config-manager.ts index e37fc1b..c8e5e23 100644 --- a/src/config-manager.ts +++ b/src/config-manager.ts @@ -75,7 +75,8 @@ export const config = { disk_path: process.env.PN_BOSS_CONFIG_CDN_DISK_PATH?.trim() || '' }, spr: { - enabled: process.env.PN_BOSS_CONFIG_STREETPASS_RELAY_ENABLED?.trim().toLowerCase() === 'true' + enabled: process.env.PN_BOSS_CONFIG_STREETPASS_RELAY_ENABLED?.trim().toLowerCase() === 'true', + cleanOldData: process.env.PN_BOSS_CONFIG_STREETPASS_RELAY_CLEAN_OLD_DATA?.trim().toLowerCase() === 'true' }, domains: { npdi: (process.env.PN_BOSS_CONFIG_DOMAINS_NPDI || 'npdi.cdn.pretendo.cc').split(','), diff --git a/src/scheduled.ts b/src/scheduled.ts index f8e37a9..fd16d56 100644 --- a/src/scheduled.ts +++ b/src/scheduled.ts @@ -1,6 +1,7 @@ import { CronJob } from 'cron'; import { logger } from './logger'; import { deleteOldCECData } from './database'; +import { config } from './config-manager'; async function runCleanSprData(): Promise { const maxAgeMs = 14 * 24 * 60 * 60 * 1000; // 14 days @@ -40,5 +41,7 @@ function registerSchedule(schedule: string, name: string, fn: () => void | Promi } export async function setupScheduler(): Promise { - registerSchedule('0 2 * * *', 'clean-spr-data', runCleanSprData); + if (config.spr.cleanOldData) { + registerSchedule('0 2 * * *', 'clean-spr-data', runCleanSprData); + } } From 651a34dc0717771b28522a9af43e1cb311027d67 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 16 Sep 2025 19:47:56 +0200 Subject: [PATCH 03/16] chore: add TODO for cleanup routine --- src/scheduled.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scheduled.ts b/src/scheduled.ts index fd16d56..6f5fc53 100644 --- a/src/scheduled.ts +++ b/src/scheduled.ts @@ -15,6 +15,8 @@ async function runCleanSprData(): Promise { const deletedData = await deleteOldCECData(timestampInPast, processingLimit); logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`); + // TODO remove references to CEC datas that have been removed + // TODO CDN removal totalRemoved += deletedData.length; From 193bbcee0741762c1d99067bbea54d72f10ce132 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Sep 2025 11:40:21 +0200 Subject: [PATCH 04/16] feat: implement bulk deletion of CDN items --- src/cdn.ts | 22 +++++++++++++++++++++- src/scheduled.ts | 6 +++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/cdn.ts b/src/cdn.ts index 164dfd2..023241c 100644 --- a/src/cdn.ts +++ b/src/cdn.ts @@ -2,7 +2,7 @@ import path from 'node:path'; import { Stream } from 'node:stream'; import { buffer as bufferConsumer } from 'node:stream/consumers'; import fs from 'fs-extra'; -import { DeleteObjectCommand, GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3'; +import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3'; import { config, disabledFeatures } from '@/config-manager'; import { fileStatOrNull } from './util'; import { logger } from './logger'; @@ -111,6 +111,26 @@ export async function deleteCDNFile(namespace: CDNNamespace, key: string): Promi })); } +export async function bulkDeleteCdnFiles(namespace: CDNNamespace, keys: string[]): Promise { + if (keys.length > 1000) { + throw new Error('Cannot bulk delete more than 1000 CDN files in one batch'); + } + + if (!s3) { + await Promise.allSettled(keys.map(v => deleteCDNFile(namespace, v))); + return; + } + + const fullKeys = keys.map(v => buildKey(namespace, v)); + await s3.send(new DeleteObjectsCommand({ + Delete: { + Objects: fullKeys.map(v => ({ Key: v })), + Quiet: true + }, + Bucket: config.cdn.s3.bucket + })); +} + export function streamFileToResponse(response: Response, stream: Readable, headers: Record = {}): void { response.setHeaders(new Headers(headers)); Stream.pipeline(stream, response, (err) => { diff --git a/src/scheduled.ts b/src/scheduled.ts index 6f5fc53..5e9fd4e 100644 --- a/src/scheduled.ts +++ b/src/scheduled.ts @@ -2,6 +2,7 @@ import { CronJob } from 'cron'; import { logger } from './logger'; import { deleteOldCECData } from './database'; import { config } from './config-manager'; +import { bulkDeleteCdnFiles } from './cdn'; async function runCleanSprData(): Promise { const maxAgeMs = 14 * 24 * 60 * 60 * 1000; // 14 days @@ -15,9 +16,8 @@ async function runCleanSprData(): Promise { const deletedData = await deleteOldCECData(timestampInPast, processingLimit); logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`); - // TODO remove references to CEC datas that have been removed - - // TODO CDN removal + await bulkDeleteCdnFiles('taskFile', deletedData.map(v => v.file_key)); + logger.info(`CDN removal processed!`); totalRemoved += deletedData.length; hasDataToDelete = deletedData.length < processingLimit; From 33bad4733237985ee8fd92bb50f8681c16d890d6 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Sep 2025 11:40:38 +0200 Subject: [PATCH 05/16] feat: remove CECSlot references while clearing old CECData --- src/database.ts | 16 ++++++++++++---- src/models/cec-slot.ts | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/database.ts b/src/database.ts index 919f69c..0c41186 100644 --- a/src/database.ts +++ b/src/database.ts @@ -217,15 +217,15 @@ export async function getRandomCECData(pids: number[], gameID: number): Promise< return null; } -export async function deleteOldCECData(olderThan: Date, limit: number): Promise<{ _id: string; file_key: string }[]> { +export async function deleteOldCECData(olderThan: Date, limit: number): Promise<{ id: string; file_key: string }[]> { verifyConnected(); const toDelete = await CECData.find({ created: { $lt: olderThan.getTime() } - }, { file_key: 1 }, { limit }); - const ids = toDelete.map(v => v.data); + }).limit(limit).sort({ created: 1 }).select({ file_key: 1 }); + const ids = toDelete.map(v => v.id); await CECData.deleteMany({ _id: { @@ -233,5 +233,13 @@ export async function deleteOldCECData(olderThan: Date, limit: number): Promise< } }); - return toDelete.map(v => ({ _id: v._id.toString(), file_key: v.file_key })); + // Remove slot if their newest data is what we've just deleted + // This is safe because everything older than the deleted data is also gone + await CECSlot.deleteMany({ + latest_data_id: { + $in: ids + } + }); + + return toDelete.map(v => ({ id: v.id, file_key: v.file_key })); } diff --git a/src/models/cec-slot.ts b/src/models/cec-slot.ts index 027a2c9..65fac50 100644 --- a/src/models/cec-slot.ts +++ b/src/models/cec-slot.ts @@ -8,5 +8,6 @@ const CECSlotSchema = new mongoose.Schema('CECSlot', CECSlotSchema); From 551897355789fdf69d918d5a453649f5326a3817 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Sep 2025 11:52:42 +0200 Subject: [PATCH 06/16] fix: add seeding folder to the docker image --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0eab34a..f6e25c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,7 @@ USER node COPY --chown=node:node update-rotation.mjs ${app_dir} COPY --chown=node:node ./boss ${app_dir} +COPY --chown=node:node seeding ${app_dir} COPY --chown=node:node package.json . COPY --from=dependencies --chown=node:node ${app_dir}/node_modules ${app_dir}/node_modules From 6b03e1c82f5787fbccf54854cc1a0e7faf8e5031 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Sep 2025 12:08:32 +0200 Subject: [PATCH 07/16] chore: remove update-rotations.mjs and replace it with documentation on how to use the CLI --- Dockerfile | 1 - README.md | 18 ++++- src/cli/files.cmd.ts | 8 ++- update-rotation.mjs | 157 ------------------------------------------- 4 files changed, 23 insertions(+), 161 deletions(-) delete mode 100644 update-rotation.mjs diff --git a/Dockerfile b/Dockerfile index f6e25c7..4b3d7d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,6 @@ RUN chown node:node ${app_dir} ENV NODE_ENV=production USER node -COPY --chown=node:node update-rotation.mjs ${app_dir} COPY --chown=node:node ./boss ${app_dir} COPY --chown=node:node seeding ${app_dir} COPY --chown=node:node package.json . diff --git a/README.md b/README.md index 8fd367b..365318c 100644 --- a/README.md +++ b/README.md @@ -68,4 +68,20 @@ Configurations are loaded through environment variables. `.env` files are suppor | `PN_BOSS_CLI_WIIU_HMAC_KEY` | The BOSS WiiU HMAC key, needs to be dumped from a console | Optional | | `PN_BOSS_CLI_NPDI_URL` | The URL of the NPDI part the BOSS HTTP server, only needed when downloading | Optional | | `PN_BOSS_CLI_NPDI_HOST` | The Host header for the NPDI requests. Use when you don't have NPDI exposed to the internet | Optional | - \ No newline at end of file + +## Common CLI operations + +```sh +# Download taskfile and decrypt +./boss file ls # View list of files and their IDs +./boss file view --decrypt > output.txt # Download file and decrypt +``` + +```sh +# Update splatoon rotations +# Run the following for all of these BOSS app ids: +# - bb6tOEckvgZ50ciH +# - rjVlM7hUXPxmYQJh +# - zvGSM4kOrXpkKnpT +./boss file create schdat2 --name VSSetting.byaml --type AppData --notify-new app --file +``` diff --git a/src/cli/files.cmd.ts b/src/cli/files.cmd.ts index 0dd73b7..51ac755 100644 --- a/src/cli/files.cmd.ts +++ b/src/cli/files.cmd.ts @@ -113,7 +113,9 @@ const createCmd = new Command('create') .option('--country ', 'Countries for this task file') .option('--lang ', 'Languages for this task file') .option('--name-as-id', 'Force the name as the data ID') - .action(async (appId: string, taskId: string, opts: { name: string; country: string[]; lang: string[]; nameAsId?: boolean; type: string; file: string }) => { + .option('--notify-new ', 'Add entry to NotifyNew') + .option('--notify-led', 'Enable NotifyLED') + .action(async (appId: string, taskId: string, opts: { name: string; country: string[]; notifyNew: string[]; notifyLed: boolean; lang: string[]; nameAsId?: boolean; type: string; file: string }) => { const fileBuf = await fs.readFile(opts.file); const ctx = getCliContext(); const { file } = await ctx.grpc.uploadFile({ @@ -124,7 +126,9 @@ const createCmd = new Command('create') supportedLanguages: opts.lang, type: opts.type, nameEqualsDataId: opts.nameAsId ?? false, - data: fileBuf + data: fileBuf, + notifyOnNew: opts.notifyNew, + notifyLed: opts.notifyLed }); if (!file) { console.log(`Failed to create file!`); diff --git a/update-rotation.mjs b/update-rotation.mjs deleted file mode 100644 index 6569fa1..0000000 --- a/update-rotation.mjs +++ /dev/null @@ -1,157 +0,0 @@ -/* eslint-disable no-undef -- Tis a script */ - -import path from 'path'; -import crypto from 'crypto'; -import readline from 'readline'; -import fs from 'fs-extra'; -import dotenv from 'dotenv'; -import xml from 'xml-js'; -import { encryptWiiU } from '@pretendonetwork/boss-crypto'; - -dotenv.config(); - -function md5(input) { - return crypto.createHash('md5').update(input).digest('hex'); -} - -const BOSS_WIIU_AES_KEY_MD5_HASH = '5202ce5099232c3d365e28379790a919'; -const BOSS_WIIU_HMAC_KEY_MD5_HASH = 'b4482fef177b0100090ce0dbeb8ce977'; - -const { PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY, PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY } = process.env; - -if (!PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY || md5(PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY) !== BOSS_WIIU_AES_KEY_MD5_HASH) { - console.error('PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY is not set or does not match the expected value'); - process.exit(1); -} -if (!PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY || md5(PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY) !== BOSS_WIIU_HMAC_KEY_MD5_HASH) { - console.error('PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY is not set or does not match the expected value'); - process.exit(1); -} - -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}); - -const askQuestion = (question) => { - return new Promise((resolve) => { - rl.question(question, (answer) => { - resolve(answer); - }); - }); -}; - -const rootDir = import.meta.dirname; -const sourceFile = path.join(rootDir, 'VSSetting.byaml'); -const checksumFile = path.join(rootDir, 'cdn/VSSetting.byaml.checksum'); - -const exists = await fs.exists(sourceFile); -if (!exists) { - console.error('Source VSSetting.byaml file does not exist'); - process.exit(1); -} - -const sourceFileContents = await fs.readFile(sourceFile); - -const stat = await fs.stat(sourceFile); -if (stat.mtime.toDateString() !== new Date().toDateString()) { - const answer = await askQuestion(`The source file was not updated today (Updated ${stat.mtime.toDateString()}). Do you want to continue? (y/n) `); - if (answer.toLowerCase() !== 'y') { - process.exit(0); - } -} - -const checksumExists = await fs.exists(checksumFile); -const newChecksum = crypto.createHash('sha256').update(sourceFileContents).digest('hex'); -console.log(`Checksum for source file is ${newChecksum}`); -if (checksumExists) { - const checksum = await fs.readFile(checksumFile, 'utf8'); - - if (checksum === newChecksum) { - const answer = await askQuestion('The source file has not changed since the last run. Do you want to continue? (y/n) '); - if (answer.toLowerCase() !== 'y') { - process.exit(0); - } - } -} - -const titles = [ - 'bb6tOEckvgZ50ciH', - 'rjVlM7hUXPxmYQJh', - 'zvGSM4kOrXpkKnpT' -]; - -async function backupFile(filePath) { - const copyFilePath = path.join(path.dirname(filePath), `${path.basename(filePath)}.bak`); - const exists = await fs.exists(filePath); - if (!exists) { - console.log(`File ${filePath} does not exist, skipping backup...`); - return; - } - await fs.copyFile(filePath, copyFilePath); - console.log(`Backup created of ${filePath} at ${copyFilePath}`); -} - -for (const title of titles) { - console.log(`\n --- Processing ${title} ---`); - const decryptedDir = path.join(rootDir, `cdn/content/decrypted/${title}`); - const encryptedDir = path.join(rootDir, `cdn/content/encrypted/${title}`); - const taskSheetDir = path.join(rootDir, `cdn/tasksheet/1/${title}`); - await fs.ensureDir(decryptedDir); - await fs.ensureDir(encryptedDir); - await fs.ensureDir(taskSheetDir); - - const decryptedFilePath = path.join(decryptedDir, 'VSSetting.byaml'); - await backupFile(decryptedFilePath); - await fs.copyFile(sourceFile, decryptedFilePath); - - const encryptedContents = encryptWiiU(decryptedFilePath, PN_BOSS_CONFIG_BOSS_WIIU_AES_KEY, PN_BOSS_CONFIG_BOSS_WIIU_HMAC_KEY); - const hash = crypto.createHash('md5').update(encryptedContents).digest('hex'); - console.log(`Hash for title ${title} is ${hash}`); - - const encryptedFilePath = path.join(encryptedDir, hash); - await fs.writeFile(encryptedFilePath, encryptedContents); - console.log(`Encrypted file created at ${encryptedFilePath}`); - - const taskSheetFilePath = path.join(taskSheetDir, 'schdat2'); - await backupFile(taskSheetFilePath); - - const tasksheetContents = await fs.readFile(taskSheetFilePath, 'utf8'); - const xmlContents = xml.xml2js(tasksheetContents, { compact: true }); - - const dataId = parseInt(xmlContents.TaskSheet.Files.File.DataId._text); - if (isNaN(dataId)) { - console.error(`DataId for title ${title} is not a number, skipping...`); - continue; - } - console.log(`DataId for title ${title} is ${dataId}`); - const newDataId = dataId + 1; - console.log(`New DataId for title ${title} is ${newDataId}`); - - xmlContents.TaskSheet.Files.File.DataId._text = newDataId.toString(); - - const size = encryptedContents.length; - const oldSize = parseInt(xmlContents.TaskSheet.Files.File.Size._text); - if (size === oldSize) { - console.log(`Size for title ${title} is already updated, skipping update...`); - } else { - console.log(`Old size for title ${title} is ${oldSize}`); - console.log(`New size for title ${title} is ${size}`); - xmlContents.TaskSheet.Files.File.Size._text = size.toString(); - } - - const oldUrl = xmlContents.TaskSheet.Files.File.Url._text; - const newUrl = `https://npdi.cdn.pretendo.cc/p01/data/1/${title}/${newDataId}/${hash}`; - console.log(`Old URL for title ${title} is ${oldUrl}`); - console.log(`New URL for title ${title} is ${newUrl}`); - xmlContents.TaskSheet.Files.File.Url._text = newUrl; - - const newXmlContents = xml.js2xml(xmlContents, { spaces: 2, compact: true }); - await fs.writeFile(taskSheetFilePath, newXmlContents); - console.log(`Tasksheet file updated at ${taskSheetFilePath}`); -} - -rl.close(); - -console.log('All tasks completed successfully!'); -await fs.writeFile(checksumFile, newChecksum); From f432d2d3dcedc1c091f64261752b250d5acdac96 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Sep 2025 12:15:41 +0200 Subject: [PATCH 08/16] chore: update mongoose and auto-increment to latest version --- package-lock.json | 378 +++++++++++++++++++-------------------------- package.json | 4 +- src/models/task.ts | 1 + 3 files changed, 159 insertions(+), 224 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4289cbb..dfbbb63 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,14 +12,14 @@ "@aws-sdk/client-s3": "^3.723.0", "@pretendonetwork/boss-crypto": "^1.0.0", "@pretendonetwork/grpc": "^1.0.6", - "@typegoose/auto-increment": "^3.6.1", + "@typegoose/auto-increment": "^4.13.0", "commander": "^14.0.0", "dicer": "^0.3.1", "dotenv": "^16.4.7", "express": "^5.1.0", "fs-extra": "^11.2.0", "moment": "^2.30.1", - "mongoose": "~7.6.1", + "mongoose": "^8.18.1", "nice-grpc": "^2.1.10", "pino": "^9.9.1", "pino-http": "^10.5.0", @@ -1896,10 +1896,10 @@ } }, "node_modules/@mongodb-js/saslprep": { - "version": "1.1.9", - "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.9.tgz", - "integrity": "sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==", - "optional": true, + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.3.0.tgz", + "integrity": "sha512-zlayKCsIjYb7/IdfqxorK5+xUMyi4vOKcFy10wKJYc63NSdKI8mNME+uJqfatkPmOSMMUiojrL58IePKBm3gvQ==", + "license": "MIT", "dependencies": { "sparse-bitfield": "^3.0.3" } @@ -3124,19 +3124,19 @@ } }, "node_modules/@typegoose/auto-increment": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/@typegoose/auto-increment/-/auto-increment-3.6.1.tgz", - "integrity": "sha512-WLSCc0TvVungjrOZSbCFlmTU14j4ob23i1/OI4JUKOILWn7AQFhAaDsCODB9ywcLdkXtZgWXJJsbtwtrfsdzpA==", + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/@typegoose/auto-increment/-/auto-increment-4.13.0.tgz", + "integrity": "sha512-saOwqB66duV+rntkME/027A8opjgzmV3pBY8+zoJ4mGSc3FVGad6CSr56x4oqd15p39XtWH1UNZaS5Bzp6O6Ow==", "license": "MIT", "dependencies": { - "loglevel": "^1.8.1", - "tslib": "^2.6.2" + "loglevel": "^1.9.2", + "tslib": "^2.8.1" }, "engines": { - "node": ">=14.17.0" + "node": ">=16.20.1" }, "peerDependencies": { - "mongoose": "~7.6.1" + "mongoose": "^8.13.0" } }, "node_modules/@types/body-parser": { @@ -3287,14 +3287,15 @@ "node_modules/@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", - "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==" + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "license": "MIT" }, "node_modules/@types/whatwg-url": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", - "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "license": "MIT", "dependencies": { - "@types/node": "*", "@types/webidl-conversions": "*" } }, @@ -4229,9 +4230,9 @@ } }, "node_modules/axios": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", - "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.2.tgz", + "integrity": "sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==", "dev": true, "license": "MIT", "dependencies": { @@ -4330,11 +4331,12 @@ } }, "node_modules/bson": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/bson/-/bson-5.5.1.tgz", - "integrity": "sha512-ix0EwukN2EpC0SRWIj/7B5+A6uQMQy6KMREI9qQqvgpkV2frH63T0UDVd1SYedL6dNCmDBYB3QtXi4ISk9YT+g==", + "version": "6.10.4", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz", + "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==", + "license": "Apache-2.0", "engines": { - "node": ">=14.20.1" + "node": ">=16.20.1" } }, "node_modules/bundle-require": { @@ -6427,23 +6429,6 @@ "node": ">= 0.4" } }, - "node_modules/ip-address": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", - "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", - "dependencies": { - "jsbn": "1.1.0", - "sprintf-js": "^1.1.3" - }, - "engines": { - "node": ">= 12" - } - }, - "node_modules/ip-address/node_modules/sprintf-js": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", - "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" - }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -6938,11 +6923,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/jsbn": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", - "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" - }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -7003,9 +6983,10 @@ } }, "node_modules/kareem": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.5.1.tgz", - "integrity": "sha512-7jFxRVm+jD+rkq3kY0iZDJfsO2/t4BBPeEb2qKn2lR/9KhuksYk5hxzfRYWMPV8P/x2d0kHD306YyWLzjjH+uA==", + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.6.3.tgz", + "integrity": "sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==", + "license": "Apache-2.0", "engines": { "node": ">=12.0.0" } @@ -7092,9 +7073,10 @@ "dev": true }, "node_modules/loglevel": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.8.1.tgz", - "integrity": "sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", + "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==", + "license": "MIT", "engines": { "node": ">= 0.6.0" }, @@ -7158,7 +7140,7 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "optional": true + "license": "MIT" }, "node_modules/merge-descriptors": { "version": "2.0.0", @@ -7273,26 +7255,26 @@ } }, "node_modules/mongodb": { - "version": "5.9.2", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.9.2.tgz", - "integrity": "sha512-H60HecKO4Bc+7dhOv4sJlgvenK4fQNqqUIlXxZYQNbfEWSALGAwGoyJd/0Qwk4TttFXUOHJ2ZJQe/52ScaUwtQ==", + "version": "6.18.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.18.0.tgz", + "integrity": "sha512-fO5ttN9VC8P0F5fqtQmclAkgXZxbIkYRTUi1j8JO6IYwvamkhtYDilJr35jOPELR49zqCJgXZWwCtW7B+TM8vQ==", + "license": "Apache-2.0", "dependencies": { - "bson": "^5.5.0", - "mongodb-connection-string-url": "^2.6.0", - "socks": "^2.7.1" + "@mongodb-js/saslprep": "^1.1.9", + "bson": "^6.10.4", + "mongodb-connection-string-url": "^3.0.0" }, "engines": { - "node": ">=14.20.1" - }, - "optionalDependencies": { - "@mongodb-js/saslprep": "^1.1.0" + "node": ">=16.20.1" }, "peerDependencies": { "@aws-sdk/credential-providers": "^3.188.0", - "@mongodb-js/zstd": "^1.0.0", - "kerberos": "^1.0.0 || ^2.0.0", - "mongodb-client-encryption": ">=2.3.0 <3", - "snappy": "^7.2.2" + "@mongodb-js/zstd": "^1.1.0 || ^2.0.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.2.2", + "socks": "^2.7.1" }, "peerDependenciesMeta": { "@aws-sdk/credential-providers": { @@ -7301,6 +7283,9 @@ "@mongodb-js/zstd": { "optional": true }, + "gcp-metadata": { + "optional": true + }, "kerberos": { "optional": true }, @@ -7309,33 +7294,38 @@ }, "snappy": { "optional": true + }, + "socks": { + "optional": true } } }, "node_modules/mongodb-connection-string-url": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz", - "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz", + "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==", + "license": "Apache-2.0", "dependencies": { - "@types/whatwg-url": "^8.2.1", - "whatwg-url": "^11.0.0" + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^14.1.0 || ^13.0.0" } }, "node_modules/mongoose": { - "version": "7.6.13", - "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-7.6.13.tgz", - "integrity": "sha512-2pFFgdP5tsXUz15odMKSvlnkEfzp1dKEd3Zs50xegCg0eRlANt6irW5RrF+HLHKoS5cwGZqUjUqgxJ50wWAReA==", + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.18.1.tgz", + "integrity": "sha512-K0RfrUXXufqNRZZjvAGdyjydB91SnbWxlwFYi5t7zN2DxVWFD3c6puia0/7xfBwZm6RCpYOVdYFlRFpoDWiC+w==", + "license": "MIT", "dependencies": { - "bson": "^5.5.0", - "kareem": "2.5.1", - "mongodb": "5.9.2", + "bson": "^6.10.4", + "kareem": "2.6.3", + "mongodb": "~6.18.0", "mpath": "0.9.0", "mquery": "5.0.0", "ms": "2.1.3", - "sift": "16.0.1" + "sift": "17.1.3" }, "engines": { - "node": ">=14.20.1" + "node": ">=16.20.1" }, "funding": { "type": "opencollective", @@ -8001,9 +7991,10 @@ } }, "node_modules/punycode": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "license": "MIT", "engines": { "node": ">=6" } @@ -8637,9 +8628,10 @@ } }, "node_modules/sift": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/sift/-/sift-16.0.1.tgz", - "integrity": "sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ==" + "version": "17.1.3", + "resolved": "https://registry.npmjs.org/sift/-/sift-17.1.3.tgz", + "integrity": "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==", + "license": "MIT" }, "node_modules/signal-exit": { "version": "4.1.0", @@ -8653,28 +8645,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/smart-buffer": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", - "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "engines": { - "node": ">= 6.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", - "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", - "dependencies": { - "ip-address": "^9.0.5", - "smart-buffer": "^4.2.0" - }, - "engines": { - "node": ">= 10.0.0", - "npm": ">= 3.0.0" - } - }, "node_modules/sonic-boom": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", @@ -8726,7 +8696,7 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", - "optional": true, + "license": "MIT", "dependencies": { "memory-pager": "^1.0.2" } @@ -9127,14 +9097,15 @@ } }, "node_modules/tr46": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", - "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", "dependencies": { - "punycode": "^2.1.1" + "punycode": "^2.3.1" }, "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/tree-kill": { @@ -9184,9 +9155,10 @@ } }, "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" }, "node_modules/tsup": { "version": "8.5.0", @@ -9589,20 +9561,22 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", "engines": { "node": ">=12" } }, "node_modules/whatwg-url": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", - "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", "dependencies": { - "tr46": "^3.0.0", + "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" }, "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/which": { @@ -11130,10 +11104,9 @@ "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==" }, "@mongodb-js/saslprep": { - "version": "1.1.9", - "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.9.tgz", - "integrity": "sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==", - "optional": true, + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.3.0.tgz", + "integrity": "sha512-zlayKCsIjYb7/IdfqxorK5+xUMyi4vOKcFy10wKJYc63NSdKI8mNME+uJqfatkPmOSMMUiojrL58IePKBm3gvQ==", "requires": { "sparse-bitfield": "^3.0.3" } @@ -12012,12 +11985,12 @@ } }, "@typegoose/auto-increment": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/@typegoose/auto-increment/-/auto-increment-3.6.1.tgz", - "integrity": "sha512-WLSCc0TvVungjrOZSbCFlmTU14j4ob23i1/OI4JUKOILWn7AQFhAaDsCODB9ywcLdkXtZgWXJJsbtwtrfsdzpA==", + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/@typegoose/auto-increment/-/auto-increment-4.13.0.tgz", + "integrity": "sha512-saOwqB66duV+rntkME/027A8opjgzmV3pBY8+zoJ4mGSc3FVGad6CSr56x4oqd15p39XtWH1UNZaS5Bzp6O6Ow==", "requires": { - "loglevel": "^1.8.1", - "tslib": "^2.6.2" + "loglevel": "^1.9.2", + "tslib": "^2.8.1" } }, "@types/body-parser": { @@ -12168,11 +12141,10 @@ "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==" }, "@types/whatwg-url": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", - "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", "requires": { - "@types/node": "*", "@types/webidl-conversions": "*" } }, @@ -12726,9 +12698,9 @@ } }, "axios": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", - "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.2.tgz", + "integrity": "sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==", "dev": true, "requires": { "follow-redirects": "^1.15.6", @@ -12805,9 +12777,9 @@ } }, "bson": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/bson/-/bson-5.5.1.tgz", - "integrity": "sha512-ix0EwukN2EpC0SRWIj/7B5+A6uQMQy6KMREI9qQqvgpkV2frH63T0UDVd1SYedL6dNCmDBYB3QtXi4ISk9YT+g==" + "version": "6.10.4", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz", + "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==" }, "bundle-require": { "version": "5.1.0", @@ -14243,22 +14215,6 @@ "side-channel": "^1.1.0" } }, - "ip-address": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", - "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", - "requires": { - "jsbn": "1.1.0", - "sprintf-js": "^1.1.3" - }, - "dependencies": { - "sprintf-js": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", - "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" - } - } - }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -14564,11 +14520,6 @@ "argparse": "^2.0.1" } }, - "jsbn": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", - "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" - }, "json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -14618,9 +14569,9 @@ } }, "kareem": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.5.1.tgz", - "integrity": "sha512-7jFxRVm+jD+rkq3kY0iZDJfsO2/t4BBPeEb2qKn2lR/9KhuksYk5hxzfRYWMPV8P/x2d0kHD306YyWLzjjH+uA==" + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.6.3.tgz", + "integrity": "sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==" }, "keyv": { "version": "4.5.4", @@ -14686,9 +14637,9 @@ "dev": true }, "loglevel": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.8.1.tgz", - "integrity": "sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg==" + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", + "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==" }, "long": { "version": "5.2.3", @@ -14732,8 +14683,7 @@ "memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "optional": true + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" }, "merge-descriptors": { "version": "2.0.0", @@ -14809,37 +14759,36 @@ "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==" }, "mongodb": { - "version": "5.9.2", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.9.2.tgz", - "integrity": "sha512-H60HecKO4Bc+7dhOv4sJlgvenK4fQNqqUIlXxZYQNbfEWSALGAwGoyJd/0Qwk4TttFXUOHJ2ZJQe/52ScaUwtQ==", + "version": "6.18.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.18.0.tgz", + "integrity": "sha512-fO5ttN9VC8P0F5fqtQmclAkgXZxbIkYRTUi1j8JO6IYwvamkhtYDilJr35jOPELR49zqCJgXZWwCtW7B+TM8vQ==", "requires": { - "@mongodb-js/saslprep": "^1.1.0", - "bson": "^5.5.0", - "mongodb-connection-string-url": "^2.6.0", - "socks": "^2.7.1" + "@mongodb-js/saslprep": "^1.1.9", + "bson": "^6.10.4", + "mongodb-connection-string-url": "^3.0.0" } }, "mongodb-connection-string-url": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz", - "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz", + "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==", "requires": { - "@types/whatwg-url": "^8.2.1", - "whatwg-url": "^11.0.0" + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^14.1.0 || ^13.0.0" } }, "mongoose": { - "version": "7.6.13", - "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-7.6.13.tgz", - "integrity": "sha512-2pFFgdP5tsXUz15odMKSvlnkEfzp1dKEd3Zs50xegCg0eRlANt6irW5RrF+HLHKoS5cwGZqUjUqgxJ50wWAReA==", + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.18.1.tgz", + "integrity": "sha512-K0RfrUXXufqNRZZjvAGdyjydB91SnbWxlwFYi5t7zN2DxVWFD3c6puia0/7xfBwZm6RCpYOVdYFlRFpoDWiC+w==", "requires": { - "bson": "^5.5.0", - "kareem": "2.5.1", - "mongodb": "5.9.2", + "bson": "^6.10.4", + "kareem": "2.6.3", + "mongodb": "~6.18.0", "mpath": "0.9.0", "mquery": "5.0.0", "ms": "2.1.3", - "sift": "16.0.1" + "sift": "17.1.3" }, "dependencies": { "ms": { @@ -15295,9 +15244,9 @@ } }, "punycode": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==" + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==" }, "qs": { "version": "6.14.0", @@ -15703,9 +15652,9 @@ } }, "sift": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/sift/-/sift-16.0.1.tgz", - "integrity": "sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ==" + "version": "17.1.3", + "resolved": "https://registry.npmjs.org/sift/-/sift-17.1.3.tgz", + "integrity": "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==" }, "signal-exit": { "version": "4.1.0", @@ -15713,20 +15662,6 @@ "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", "dev": true }, - "smart-buffer": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", - "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==" - }, - "socks": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", - "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", - "requires": { - "ip-address": "^9.0.5", - "smart-buffer": "^4.2.0" - } - }, "sonic-boom": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", @@ -15776,7 +15711,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", - "optional": true, "requires": { "memory-pager": "^1.0.2" } @@ -16052,11 +15986,11 @@ "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, "tr46": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", - "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", "requires": { - "punycode": "^2.1.1" + "punycode": "^2.3.1" } }, "tree-kill": { @@ -16096,9 +16030,9 @@ } }, "tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" }, "tsup": { "version": "8.5.0", @@ -16359,11 +16293,11 @@ "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" }, "whatwg-url": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", - "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", "requires": { - "tr46": "^3.0.0", + "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, diff --git a/package.json b/package.json index f95f07a..03e34ca 100644 --- a/package.json +++ b/package.json @@ -16,14 +16,14 @@ "@aws-sdk/client-s3": "^3.723.0", "@pretendonetwork/boss-crypto": "^1.0.0", "@pretendonetwork/grpc": "^1.0.6", - "@typegoose/auto-increment": "^3.6.1", + "@typegoose/auto-increment": "^4.13.0", "commander": "^14.0.0", "dicer": "^0.3.1", "dotenv": "^16.4.7", "express": "^5.1.0", "fs-extra": "^11.2.0", "moment": "^2.30.1", - "mongoose": "~7.6.1", + "mongoose": "^8.18.1", "nice-grpc": "^2.1.10", "pino": "^9.9.1", "pino-http": "^10.5.0", diff --git a/src/models/task.ts b/src/models/task.ts index 3a43890..d928141 100644 --- a/src/models/task.ts +++ b/src/models/task.ts @@ -12,6 +12,7 @@ const TaskSchema = new mongoose.Schema({ creator_pid: Number, status: { type: String, + required: true, enum: ['open'] // TODO - What else is there? }, title_id: String, From 28f77dcf77c2bae6fb8761135840ce4abc3af9bd Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Sep 2025 12:45:24 +0200 Subject: [PATCH 09/16] chore: replace dicer with formidable --- package-lock.json | 133 +++++++++++++++++++++++++++++++++++--------- package.json | 3 +- src/services/spr.ts | 100 +++++++++------------------------ 3 files changed, 134 insertions(+), 102 deletions(-) diff --git a/package-lock.json b/package-lock.json index dfbbb63..230fa8a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,9 +14,9 @@ "@pretendonetwork/grpc": "^1.0.6", "@typegoose/auto-increment": "^4.13.0", "commander": "^14.0.0", - "dicer": "^0.3.1", "dotenv": "^16.4.7", "express": "^5.1.0", + "formidable": "^3.5.4", "fs-extra": "^11.2.0", "moment": "^2.30.1", "mongoose": "^8.18.1", @@ -33,6 +33,7 @@ "@smithy/types": "^4.0.0", "@types/dicer": "^0.2.4", "@types/express": "^4.17.21", + "@types/formidable": "^3.4.5", "@types/fs-extra": "^11.0.4", "@types/node": "^22.10.5", "axios": "^1.7.9", @@ -1917,6 +1918,18 @@ "@tybys/wasm-util": "^0.10.0" } }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2000,6 +2013,15 @@ "node": ">=8.0" } }, + "node_modules/@paralleldrive/cuid2": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.2.2.tgz", + "integrity": "sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "^1.1.5" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -3198,6 +3220,16 @@ "@types/send": "*" } }, + "node_modules/@types/formidable": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz", + "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/fs-extra": { "version": "11.0.4", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", @@ -4187,6 +4219,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "license": "MIT" + }, "node_modules/async-function": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz", @@ -4722,15 +4760,14 @@ "node": ">= 0.8" } }, - "node_modules/dicer": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.1.tgz", - "integrity": "sha512-ObioMtXnmjYs3aRtpIJt9rgQSPCIhKVkFPip+E9GUDyWl8N435znUxK/JfNwGZJ2wnn5JKQ7Ly3vOK5Q5dylGA==", + "node_modules/dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "license": "ISC", "dependencies": { - "streamsearch": "^1.1.0" - }, - "engines": { - "node": ">=10.0.0" + "asap": "^2.0.0", + "wrappy": "1" } }, "node_modules/doctrine": { @@ -5997,6 +6034,23 @@ "node": ">= 6" } }, + "node_modules/formidable": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz", + "integrity": "sha512-YikH+7CUTOtP44ZTnUhR7Ic2UASBPOqmaRkRKxRbywPTe5VxF7RRCck4af9wutiZ/QKM5nME9Bie2fFaPz5Gug==", + "license": "MIT", + "dependencies": { + "@paralleldrive/cuid2": "^2.2.2", + "dezalgo": "^1.0.4", + "once": "^1.4.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -8749,14 +8803,6 @@ "node": ">= 0.4" } }, - "node_modules/streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -11123,6 +11169,11 @@ "@tybys/wasm-util": "^0.10.0" } }, + "@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==" + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -11185,6 +11236,14 @@ "integrity": "sha512-T8TbSnGsxo6TDBJx/Sgv/BlVJL3tshxZP7Aq5R1mSnM5OcHY2dQaxLMu2+E8u3gN0MLOzdjurqN4ZRVuzQycOQ==", "dev": true }, + "@paralleldrive/cuid2": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.2.2.tgz", + "integrity": "sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==", + "requires": { + "@noble/hashes": "^1.1.5" + } + }, "@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -12051,6 +12110,15 @@ "@types/send": "*" } }, + "@types/formidable": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz", + "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/fs-extra": { "version": "11.0.4", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", @@ -12671,6 +12739,11 @@ "is-array-buffer": "^3.0.4" } }, + "asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" + }, "async-function": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-function/-/async-function-1.0.0.tgz", @@ -13025,12 +13098,13 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, - "dicer": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.1.tgz", - "integrity": "sha512-ObioMtXnmjYs3aRtpIJt9rgQSPCIhKVkFPip+E9GUDyWl8N435znUxK/JfNwGZJ2wnn5JKQ7Ly3vOK5Q5dylGA==", + "dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", "requires": { - "streamsearch": "^1.1.0" + "asap": "^2.0.0", + "wrappy": "1" } }, "doctrine": { @@ -13935,6 +14009,16 @@ "mime-types": "^2.1.12" } }, + "formidable": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz", + "integrity": "sha512-YikH+7CUTOtP44ZTnUhR7Ic2UASBPOqmaRkRKxRbywPTe5VxF7RRCck4af9wutiZ/QKM5nME9Bie2fFaPz5Gug==", + "requires": { + "@paralleldrive/cuid2": "^2.2.2", + "dezalgo": "^1.0.4", + "once": "^1.4.0" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -15747,11 +15831,6 @@ "internal-slot": "^1.1.0" } }, - "streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==" - }, "string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", diff --git a/package.json b/package.json index 03e34ca..82775c6 100644 --- a/package.json +++ b/package.json @@ -18,9 +18,9 @@ "@pretendonetwork/grpc": "^1.0.6", "@typegoose/auto-increment": "^4.13.0", "commander": "^14.0.0", - "dicer": "^0.3.1", "dotenv": "^16.4.7", "express": "^5.1.0", + "formidable": "^3.5.4", "fs-extra": "^11.2.0", "moment": "^2.30.1", "mongoose": "^8.18.1", @@ -37,6 +37,7 @@ "@smithy/types": "^4.0.0", "@types/dicer": "^0.2.4", "@types/express": "^4.17.21", + "@types/formidable": "^3.4.5", "@types/fs-extra": "^11.0.4", "@types/node": "^22.10.5", "axios": "^1.7.9", diff --git a/src/services/spr.ts b/src/services/spr.ts index 57dd1e1..370ed17 100644 --- a/src/services/spr.ts +++ b/src/services/spr.ts @@ -1,87 +1,38 @@ import crypto from 'node:crypto'; -import { Stream } from 'node:stream'; +import { readFile } from 'node:fs/promises'; +import { formidable } from 'formidable'; import express from 'express'; -import Dicer from 'dicer'; import { getDuplicateCECData, getRandomCECData } from '@/database'; import { getFriends } from '@/util'; import { CECData } from '@/models/cec-data'; import { CECSlot } from '@/models/cec-slot'; import { SendMode } from '@/types/common/spr-slot'; -import RequestException from '@/request-exception'; import { config } from '@/config-manager'; import { restrictHostnames } from '@/middleware/host-limit'; import { logger } from '@/logger'; import { getCDNFileAsBuffer, uploadCDNFile } from '@/cdn'; +import RequestException from '@/request-exception'; +import type { File } from 'formidable'; +import type { Request } from 'express'; import type { SPRSlot } from '@/types/common/spr-slot'; const spr = express.Router(); -function multipartParser(request: express.Request, response: express.Response, next: express.NextFunction): void { - const RE_BOUNDARY = /^multipart\/.+?(?:; boundary=(?:(?:"(.+)")|(?:([^\s]+))))$/i; - const RE_FILE_NAME = /name="(.*)"/; - - const contentType = request.header('content-type'); - - if (!contentType) { - return next(); - } - - const boundary = RE_BOUNDARY.exec(contentType); - - if (!boundary) { - return next(); - } - - const dicer = new Dicer({ boundary: boundary[1] || boundary[2] }); - const files: Record = {}; - - dicer.on('part', (part: Dicer.PartStream) => { - let fileBuffer = Buffer.alloc(0); - let fileName = ''; - - part.on('header', (header) => { - const contentDisposition = header['content-disposition' as keyof object]; - const regexResult = RE_FILE_NAME.exec(contentDisposition); - - if (regexResult) { - fileName = regexResult[1]; - } - }); - - part.on('data', (data: Buffer | string) => { - if (typeof data === 'string') { - data = Buffer.from(data); - } - - fileBuffer = Buffer.concat([fileBuffer, data]); - }); - - part.on('end', () => { - files[fileName] = fileBuffer; - }); - - part.on('error', (error: Error) => { - return next(new RequestException(error.message, 400)); - }); +async function parseMultipart(request: Request): Promise> { + const form = formidable({ + multiples: false + }); + const parsedForm = await form.parse(request).catch((err: Error) => { + throw new RequestException(err.message, 400); }); - dicer.on('finish', function () { - request.files = files; - return next(); - }); - - Stream.pipeline(request, dicer, (error: Error | null) => { - if (error) { - return next(new RequestException(error.message, 400)); - } - }); + const entries = Object.entries(parsedForm[1]); + const entriesWithSinglefile = entries.map(v => [v[0], (v[1] ?? [])[0]]); + return Object.fromEntries(entriesWithSinglefile); } -spr.post('/relay/0', multipartParser, async (request, response) => { - if (!request.files) { - response.sendStatus(400); - return; - } +spr.post('/relay/0', async (request, response) => { + const files = await parseMultipart(request); if (!request.pid || !request.nexAccount) { response.sendStatus(401); @@ -90,15 +41,15 @@ spr.post('/relay/0', multipartParser, async (request, response) => { // * Check that the account is a 3DS and isn't banned if (!request.nexAccount.friendCode || request.nexAccount.accessLevel < 0) { - logger.info(`{request.pid}: User is not a 3DS or is banned`); + logger.info(`${request.pid}: User is not a 3DS or is banned`); response.sendStatus(403); return; } - const sprMetadataBuffer: Buffer | undefined = request.files['spr-meta']; + const sprMetadataFile: File | undefined = files['spr-meta']; - if (!sprMetadataBuffer) { - logger.warn(`{request.pid}: Missing spr-meta file`); + if (!sprMetadataFile) { + logger.warn(`${request.pid}: Missing spr-meta file`); response.sendStatus(400); return; } @@ -106,7 +57,7 @@ spr.post('/relay/0', multipartParser, async (request, response) => { const sprSlots: SPRSlot[] = []; // * Check spr-meta metadata headers - const sprMetadata = sprMetadataBuffer.toString(); + const sprMetadata = await readFile(sprMetadataFile.filepath, 'utf-8'); const metadataHeaders = sprMetadata.split('\r\n'); // * Split header lines if (metadataHeaders.length < 1) { @@ -177,15 +128,15 @@ spr.post('/relay/0', multipartParser, async (request, response) => { let data: Buffer = Buffer.alloc(0); if (size > 0 && sendMode !== SendMode.RecvOnly) { const slot = i.toString().padStart(2, '0'); - const slotData: Buffer | undefined = request.files['spr-slot' + slot]; + const slotDataFile: File | undefined = files['spr-slot' + slot]; - if (!slotData) { + if (!slotDataFile) { logger.warn(`${request.pid}: Missing slot data file`); response.sendStatus(400); return; } - if (slotData.length !== size) { + if (slotDataFile.size !== size) { logger.warn(`${request.pid}: Invalid slot data size`); response.sendStatus(400); return; @@ -200,12 +151,13 @@ spr.post('/relay/0', multipartParser, async (request, response) => { // * This is then followed by a CecMessageHeader (see https://github.com/NarcolepticK/CECDocs/blob/master/Structs/CecMessageHeader.md) // * Check that we at least have enough size for the StreetPass header - if (slotData.length < 0x12) { + if (slotDataFile.size < 0x12) { logger.warn(`${request.pid}: Slot is too short`); response.sendStatus(400); return; } + const slotData = await readFile(slotDataFile.filepath); if (slotData.readUInt32LE() !== 0x6161) { logger.warn(`${request.pid}: Slot header missmatch`); response.sendStatus(400); From 2e2ccaf1c2108d8296a495453dbc0c84374548ef Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Sep 2025 12:46:19 +0200 Subject: [PATCH 10/16] chore: remove dicer types --- package-lock.json | 19 ------------------- package.json | 1 - 2 files changed, 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index 230fa8a..a1318df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,6 @@ "devDependencies": { "@pretendonetwork/eslint-config": "^0.1.1", "@smithy/types": "^4.0.0", - "@types/dicer": "^0.2.4", "@types/express": "^4.17.21", "@types/formidable": "^3.4.5", "@types/fs-extra": "^11.0.4", @@ -3180,15 +3179,6 @@ "@types/node": "*" } }, - "node_modules/@types/dicer": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@types/dicer/-/dicer-0.2.4.tgz", - "integrity": "sha512-fOmnb+GtVJKwGf73zvgLsVJdj+L+InnbLMTGo1/tjCjrzDIbqh5ijziL8AkPkOEfAa3ODLy1uYr/sv5QEOYnWQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -12071,15 +12061,6 @@ "@types/node": "*" } }, - "@types/dicer": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@types/dicer/-/dicer-0.2.4.tgz", - "integrity": "sha512-fOmnb+GtVJKwGf73zvgLsVJdj+L+InnbLMTGo1/tjCjrzDIbqh5ijziL8AkPkOEfAa3ODLy1uYr/sv5QEOYnWQ==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", diff --git a/package.json b/package.json index 82775c6..d912863 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "devDependencies": { "@pretendonetwork/eslint-config": "^0.1.1", "@smithy/types": "^4.0.0", - "@types/dicer": "^0.2.4", "@types/express": "^4.17.21", "@types/formidable": "^3.4.5", "@types/fs-extra": "^11.0.4", From 3099a9cf3a1438d20aecd9708290296771532d97 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 18 Sep 2025 21:11:27 +0200 Subject: [PATCH 11/16] fix: clean up files after request --- src/services/spr.ts | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/services/spr.ts b/src/services/spr.ts index 370ed17..3a0e55f 100644 --- a/src/services/spr.ts +++ b/src/services/spr.ts @@ -1,5 +1,5 @@ import crypto from 'node:crypto'; -import { readFile } from 'node:fs/promises'; +import { readFile, unlink } from 'node:fs/promises'; import { formidable } from 'formidable'; import express from 'express'; import { getDuplicateCECData, getRandomCECData } from '@/database'; @@ -13,12 +13,12 @@ import { logger } from '@/logger'; import { getCDNFileAsBuffer, uploadCDNFile } from '@/cdn'; import RequestException from '@/request-exception'; import type { File } from 'formidable'; -import type { Request } from 'express'; +import type { Request, Response } from 'express'; import type { SPRSlot } from '@/types/common/spr-slot'; const spr = express.Router(); -async function parseMultipart(request: Request): Promise> { +async function parseMultipart(request: Request, response: Response): Promise> { const form = formidable({ multiples: false }); @@ -27,12 +27,25 @@ async function parseMultipart(request: Request): Promise> { }); const entries = Object.entries(parsedForm[1]); - const entriesWithSinglefile = entries.map(v => [v[0], (v[1] ?? [])[0]]); - return Object.fromEntries(entriesWithSinglefile); + const entriesWithSinglefile = entries.map(v => [v[0], (v[1] ?? [])[0]] as const); + const fileOutput = Object.fromEntries(entriesWithSinglefile); + + const cleanup = async (): Promise => { + await Promise.allSettled( + Object.values(fileOutput).map(file => + unlink(file.filepath).catch(() => { /* Ignore cleanup errors */ }) + ) + ); + }; + + response.once('finish', cleanup); + response.once('close', cleanup); + + return fileOutput; } spr.post('/relay/0', async (request, response) => { - const files = await parseMultipart(request); + const files = await parseMultipart(request, response); if (!request.pid || !request.nexAccount) { response.sendStatus(401); From 099ebec85e15ccba7830948db71948a7c1020046 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 18 Sep 2025 21:45:32 +0200 Subject: [PATCH 12/16] fix: fixed bulk deleting empty list of keys --- src/cdn.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cdn.ts b/src/cdn.ts index 023241c..5efdecf 100644 --- a/src/cdn.ts +++ b/src/cdn.ts @@ -112,6 +112,9 @@ export async function deleteCDNFile(namespace: CDNNamespace, key: string): Promi } export async function bulkDeleteCdnFiles(namespace: CDNNamespace, keys: string[]): Promise { + if (keys.length === 0) { + return; + } if (keys.length > 1000) { throw new Error('Cannot bulk delete more than 1000 CDN files in one batch'); } From c9d0bd4b8274eab6a0d2dd3e6e8603b72ff6fcf7 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 18 Sep 2025 21:45:53 +0200 Subject: [PATCH 13/16] fix: fixed infinite looping of deleting + deleting in wrong namespace --- src/scheduled.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scheduled.ts b/src/scheduled.ts index 5e9fd4e..2fa7c37 100644 --- a/src/scheduled.ts +++ b/src/scheduled.ts @@ -16,11 +16,11 @@ async function runCleanSprData(): Promise { const deletedData = await deleteOldCECData(timestampInPast, processingLimit); logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`); - await bulkDeleteCdnFiles('taskFile', deletedData.map(v => v.file_key)); + await bulkDeleteCdnFiles('spr', deletedData.map(v => v.file_key)); logger.info(`CDN removal processed!`); totalRemoved += deletedData.length; - hasDataToDelete = deletedData.length < processingLimit; + hasDataToDelete = deletedData.length === processingLimit; } logger.success(`Completed cleanup of ${totalRemoved}`); From 840ccc259845ca1dd0c3f4e4bd3d294c12d11328 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Fri, 19 Sep 2025 20:45:01 +0200 Subject: [PATCH 14/16] fix: fixed spr metadata being used as a file while it's a field --- src/middleware/multipart.ts | 43 +++++++++++++++++++++++++++++++++++++ src/services/spr.ts | 39 +++++---------------------------- 2 files changed, 48 insertions(+), 34 deletions(-) create mode 100644 src/middleware/multipart.ts diff --git a/src/middleware/multipart.ts b/src/middleware/multipart.ts new file mode 100644 index 0000000..9b4e6d8 --- /dev/null +++ b/src/middleware/multipart.ts @@ -0,0 +1,43 @@ +import { unlink } from 'fs/promises'; +import formidable from 'formidable'; +import RequestException from '@/request-exception'; +import type { File } from 'formidable'; +import type { Request, Response } from 'express'; + +export type MultipartOutput = { + fields: Record; + files: Record; +}; + +export async function parseMultipart(request: Request, response: Response): Promise { + const form = formidable({ + multiples: false + }); + const [fields, files] = await form.parse(request).catch((err: Error) => { + throw new RequestException(err.message, 400); + }); + + const fileEntries = Object.entries(files); + const fileEntriesUnified = fileEntries.map(v => [v[0], (v[1] ?? [])[0]] as const); + const fileOutput = Object.fromEntries(fileEntriesUnified); + + const fieldEntries = Object.entries(fields); + const fieldEntriesUnified = fieldEntries.map(v => [v[0], (v[1] ?? [])[0]] as const); + const fieldOutput = Object.fromEntries(fieldEntriesUnified); + + const cleanup = async (): Promise => { + await Promise.allSettled( + Object.values(fileOutput).map(file => + unlink(file.filepath).catch(() => { /* Ignore cleanup errors */ }) + ) + ); + }; + + response.once('finish', cleanup); + response.once('close', cleanup); + + return { + fields: fieldOutput, + files: fileOutput + }; +} diff --git a/src/services/spr.ts b/src/services/spr.ts index 3a0e55f..ad84cc8 100644 --- a/src/services/spr.ts +++ b/src/services/spr.ts @@ -1,6 +1,5 @@ import crypto from 'node:crypto'; -import { readFile, unlink } from 'node:fs/promises'; -import { formidable } from 'formidable'; +import { readFile } from 'node:fs/promises'; import express from 'express'; import { getDuplicateCECData, getRandomCECData } from '@/database'; import { getFriends } from '@/util'; @@ -11,41 +10,14 @@ import { config } from '@/config-manager'; import { restrictHostnames } from '@/middleware/host-limit'; import { logger } from '@/logger'; import { getCDNFileAsBuffer, uploadCDNFile } from '@/cdn'; -import RequestException from '@/request-exception'; +import { parseMultipart } from '@/middleware/multipart'; import type { File } from 'formidable'; -import type { Request, Response } from 'express'; import type { SPRSlot } from '@/types/common/spr-slot'; const spr = express.Router(); -async function parseMultipart(request: Request, response: Response): Promise> { - const form = formidable({ - multiples: false - }); - const parsedForm = await form.parse(request).catch((err: Error) => { - throw new RequestException(err.message, 400); - }); - - const entries = Object.entries(parsedForm[1]); - const entriesWithSinglefile = entries.map(v => [v[0], (v[1] ?? [])[0]] as const); - const fileOutput = Object.fromEntries(entriesWithSinglefile); - - const cleanup = async (): Promise => { - await Promise.allSettled( - Object.values(fileOutput).map(file => - unlink(file.filepath).catch(() => { /* Ignore cleanup errors */ }) - ) - ); - }; - - response.once('finish', cleanup); - response.once('close', cleanup); - - return fileOutput; -} - spr.post('/relay/0', async (request, response) => { - const files = await parseMultipart(request, response); + const { files, fields } = await parseMultipart(request, response); if (!request.pid || !request.nexAccount) { response.sendStatus(401); @@ -59,9 +31,9 @@ spr.post('/relay/0', async (request, response) => { return; } - const sprMetadataFile: File | undefined = files['spr-meta']; + const sprMetadata: string | undefined = fields['spr-meta']; - if (!sprMetadataFile) { + if (!sprMetadata) { logger.warn(`${request.pid}: Missing spr-meta file`); response.sendStatus(400); return; @@ -70,7 +42,6 @@ spr.post('/relay/0', async (request, response) => { const sprSlots: SPRSlot[] = []; // * Check spr-meta metadata headers - const sprMetadata = await readFile(sprMetadataFile.filepath, 'utf-8'); const metadataHeaders = sprMetadata.split('\r\n'); // * Split header lines if (metadataHeaders.length < 1) { From 2bcff40f6c0e84a55aaf2732f755e66684a8ba57 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Fri, 19 Sep 2025 20:55:34 +0200 Subject: [PATCH 15/16] fix: add closed value to task enum --- src/models/task.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/task.ts b/src/models/task.ts index d928141..47d4c54 100644 --- a/src/models/task.ts +++ b/src/models/task.ts @@ -13,7 +13,7 @@ const TaskSchema = new mongoose.Schema({ status: { type: String, required: true, - enum: ['open'] // TODO - What else is there? + enum: ['open', 'closed'] }, title_id: String, description: String, From 4275d2ed9d956af5e16ea767886e8a7391d842c3 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Fri, 19 Sep 2025 20:58:53 +0200 Subject: [PATCH 16/16] fix: fix typo in enum values --- src/models/task.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/task.ts b/src/models/task.ts index 47d4c54..cb374b1 100644 --- a/src/models/task.ts +++ b/src/models/task.ts @@ -13,7 +13,7 @@ const TaskSchema = new mongoose.Schema({ status: { type: String, required: true, - enum: ['open', 'closed'] + enum: ['open', 'close'] }, title_id: String, description: String,