diff --git a/src/cdn.ts b/src/cdn.ts index 164dfd2..023241c 100644 --- a/src/cdn.ts +++ b/src/cdn.ts @@ -2,7 +2,7 @@ import path from 'node:path'; import { Stream } from 'node:stream'; import { buffer as bufferConsumer } from 'node:stream/consumers'; import fs from 'fs-extra'; -import { DeleteObjectCommand, GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3'; +import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3'; import { config, disabledFeatures } from '@/config-manager'; import { fileStatOrNull } from './util'; import { logger } from './logger'; @@ -111,6 +111,26 @@ export async function deleteCDNFile(namespace: CDNNamespace, key: string): Promi })); } +export async function bulkDeleteCdnFiles(namespace: CDNNamespace, keys: string[]): Promise { + if (keys.length > 1000) { + throw new Error('Cannot bulk delete more than 1000 CDN files in one batch'); + } + + if (!s3) { + await Promise.allSettled(keys.map(v => deleteCDNFile(namespace, v))); + return; + } + + const fullKeys = keys.map(v => buildKey(namespace, v)); + await s3.send(new DeleteObjectsCommand({ + Delete: { + Objects: fullKeys.map(v => ({ Key: v })), + Quiet: true + }, + Bucket: config.cdn.s3.bucket + })); +} + export function streamFileToResponse(response: Response, stream: Readable, headers: Record = {}): void { response.setHeaders(new Headers(headers)); Stream.pipeline(stream, response, (err) => { diff --git a/src/scheduled.ts b/src/scheduled.ts index 6f5fc53..5e9fd4e 100644 --- a/src/scheduled.ts +++ b/src/scheduled.ts @@ -2,6 +2,7 @@ import { CronJob } from 'cron'; import { logger } from './logger'; import { deleteOldCECData } from './database'; import { config } from './config-manager'; +import { bulkDeleteCdnFiles } from './cdn'; async function runCleanSprData(): Promise { const maxAgeMs = 14 * 24 * 60 * 60 * 1000; // 14 days @@ -15,9 +16,8 @@ async function runCleanSprData(): Promise { const deletedData = await deleteOldCECData(timestampInPast, processingLimit); logger.info(`Deleted one batch of ${deletedData.length} CEC data objects, preparing CDN removal`); - // TODO remove references to CEC datas that have been removed - - // TODO CDN removal + await bulkDeleteCdnFiles('taskFile', deletedData.map(v => v.file_key)); + logger.info(`CDN removal processed!`); totalRemoved += deletedData.length; hasDataToDelete = deletedData.length < processingLimit;