Leahnaya/BOSS
1
0
Fork 0
mirror of https://github.com/PretendoNetwork/BOSS.git synced 2026-07-12 23:02:09 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: implement bulk deletion of CDN items

Browse Source
This commit is contained in:
mrjvs 2025-09-17 11:40:21 +02:00
parent 4599c82a67
commit 193bbcee07
2 changed files with 24 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/cdn.ts
View File

@ -2,7 +2,7 @@ import path from 'node:path';
import { Stream } from 'node:stream';
import { buffer as bufferConsumer } from 'node:stream/consumers';
import fs from 'fs-extra';
import { DeleteObjectCommand, GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3';
import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3';
import { config, disabledFeatures } from '@/config-manager';
import { fileStatOrNull } from './util';
import { logger } from './logger';
@ -111,6 +111,26 @@ export async function deleteCDNFile(namespace: CDNNamespace, key: string): Promi
}));
}
export async function bulkDeleteCdnFiles(namespace: CDNNamespace, keys: string[]): Promise<void> {
if (keys.length > 1000) {
throw new Error('Cannot bulk delete more than 1000 CDN files in one batch');
}
if (!s3) {
await Promise.allSettled(keys.map(v => deleteCDNFile(namespace, v)));
return;
}
const fullKeys = keys.map(v => buildKey(namespace, v));
await s3.send(new DeleteObjectsCommand({
Delete: {
Objects: fullKeys.map(v => ({ Key: v })),
Quiet: true
},
Bucket: config.cdn.s3.bucket
}));
}
export function streamFileToResponse(response: Response, stream: Readable, headers: Record<string, string> = {}): void {
response.setHeaders(new Headers(headers));
Stream.pipeline(stream, response, (err) => {

6
src/scheduled.ts
View File

@ -2,6 +2,7 @@ import { CronJob } from 'cron';
import { logger } from './logger';
import { deleteOldCECData } from './database';
import { config } from './config-manager';
import { bulkDeleteCdnFiles } from './cdn';
async function runCleanSprData(): Promise<void> {
const maxAgeMs = 14 * 24 * 60 * 60 * 1000; // 14 days
@ -15,9 +16,8 @@ async function runCleanSprData(): Promise<void> {
const deletedData = await deleteOldCECData(timestampInPast, processingLimit);
logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`);
// TODO remove references to CEC datas that have been removed
// TODO CDN removal
await bulkDeleteCdnFiles('taskFile', deletedData.map(v => v.file_key));
logger.info(`CDN removal processed!`);
totalRemoved += deletedData.length;
hasDataToDelete = deletedData.length < processingLimit;