diff --git a/app/data/ArchiveCompressor.mjs b/app/data/ArchiveCompressor.mjs index 1aea019..f68d6d9 100644 --- a/app/data/ArchiveCompressor.mjs +++ b/app/data/ArchiveCompressor.mjs @@ -12,7 +12,8 @@ import { S3Client, } from '@aws-sdk/client-s3'; import prefixedConsole from '../common/prefixedConsole.mjs'; -import { createArchiveManifest } from './ArchiveManifest.mjs'; +import { createArchiveManifest, parseArchiveManifest } from './ArchiveManifest.mjs'; +import ArchiveVerifier, { AppleDoubleArchiveError } from './ArchiveVerifier.mjs'; import TarZstdWriter from './TarZstdWriter.mjs'; const downloadLimit = 5; @@ -60,9 +61,14 @@ export default class ArchiveCompressor maxDays = Infinity; rebuildBefore = null; - constructor(s3Client, archiveWriter = new TarZstdWriter) { + constructor( + s3Client, + archiveWriter = new TarZstdWriter, + archiveVerifier = new ArchiveVerifier, + ) { this._client = s3Client; this.archiveWriter = archiveWriter; + this.archiveVerifier = archiveVerifier; } async process() { @@ -77,7 +83,9 @@ export default class ArchiveCompressor try { let candidates = await this.getCandidates(); - this.console.log(`Found ${candidates.length} dates to archive`); + this.console.log( + `Found ${candidates.length} dates to ${this.rebuildBefore === null ? 'archive' : 'inspect'}`, + ); for (let candidate of candidates) { if (compressed >= this.maxDays) { @@ -144,7 +152,7 @@ export default class ArchiveCompressor async previewDate(candidate) { let objects = await this.getReadyObjects(candidate); - if (!objects) { + if (!objects || candidate.exists && !await this.needsRebuild(candidate, objects)) { return false; } @@ -159,7 +167,7 @@ export default class ArchiveCompressor async archiveDate(candidate) { let objects = await this.getReadyObjects(candidate); - if (!objects) { + if (!objects || candidate.exists && !await this.needsRebuild(candidate, objects)) { return false; } @@ -215,6 +223,63 @@ export default class ArchiveCompressor return objects; } + async needsRebuild(candidate, objects) { + let manifest = await this.getManifest(candidate); + let archiveStream = await this.getArchive(candidate.archiveKey); + let sourceObjects = objects.map(object => { + if (typeof object.etag !== 'string') { + throw new Error(`S3 returned incomplete metadata for ${object.key}`); + } + + return { + path: object.path, + bytes: object.bytes, + etag: object.etag, + }; + }); + + try { + await this.archiveVerifier.verify(archiveStream, manifest, sourceObjects); + } catch (error) { + if (error instanceof AppleDoubleArchiveError) { + return true; + } + + throw error; + } + + this.console.log(`Skipping ${candidate.date}; existing archive is valid`); + + return false; + } + + async getManifest(candidate) { + let response = await this.s3Client.send(new GetObjectCommand({ + Bucket: process.env.AWS_S3_ARCHIVE_BUCKET, + Key: candidate.manifestKey, + })); + if (!response.Body) { + throw new Error(`S3 returned no body for ${candidate.manifestKey}`); + } + + return parseArchiveManifest( + await response.Body.transformToString(), + candidate.archiveKey, + ); + } + + async getArchive(key) { + let response = await this.s3Client.send(new GetObjectCommand({ + Bucket: process.env.AWS_S3_ARCHIVE_BUCKET, + Key: key, + })); + if (!response.Body) { + throw new Error(`S3 returned no body for ${key}`); + } + + return response.Body; + } + async downloadObjects(prefix, objects, sourceDirectory) { let files = []; let nextObject = 0; @@ -341,8 +406,10 @@ export default class ArchiveCompressor } return { + path: this.relativePath(prefix, object.Key), key: object.Key, bytes: object.Size, + etag: object.ETag, lastModified: object.LastModified, }; }); diff --git a/app/data/ArchiveCompressor.test.mjs b/app/data/ArchiveCompressor.test.mjs index 105ee51..afaeb3d 100644 --- a/app/data/ArchiveCompressor.test.mjs +++ b/app/data/ArchiveCompressor.test.mjs @@ -7,6 +7,28 @@ import { } from '@aws-sdk/client-s3'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import ArchiveCompressor from './ArchiveCompressor.mjs'; +import { AppleDoubleArchiveError } from './ArchiveVerifier.mjs'; + +const archiveKey = '2025/03/2025-03-05.tar.zst'; +const manifestKey = `${archiveKey}.manifest.json`; + +function completedManifest() { + return { + version: 1, + date: '2025-03-05', + createdAt: '2026-08-20T00:00:00.000Z', + archive: { + path: archiveKey, + bytes: 7, + hash: `sha256:${'a'.repeat(64)}`, + }, + files: [{ + path: 'alpha.json', + bytes: 6, + hash: `sha256:${'b'.repeat(64)}`, + }], + }; +} class FakeS3Client { @@ -28,6 +50,17 @@ class FakeS3Client if (command instanceof GetObjectCommand) { this.downloads.push(command.input.Key); + if (command.input.Key === manifestKey) { + return { + Body: { + transformToString: async () => JSON.stringify(completedManifest()), + }, + }; + } + if (command.input.Key === archiveKey) { + return { Body: Readable.from('archive') }; + } + return { Body: Readable.from(this.files.get(command.input.Key)) }; } @@ -66,6 +99,7 @@ class FakeS3Client return { Contents: [...this.files] .filter(([Key]) => Key.startsWith(prefix)) .map(([Key, body]) => ({ + ETag: '"9f9f90dbe3e5ee1218c86b8839db1995"', Key, LastModified: new Date('2025-03-05T23:45:00.000Z'), Size: body.length, @@ -187,15 +221,36 @@ describe('ArchiveCompressor', () => { expect(archiveWriter.write).not.toHaveBeenCalled(); }); - it('rebuilds a completed archive older than the repair cutoff', async () => { + it('leaves a valid completed archive older than the repair cutoff alone', async () => { let s3Client = new FakeS3Client; s3Client.completed = true; + let verifier = { verify: vi.fn() }; + let archiveWriter = { write: vi.fn() }; + let compressor = new ArchiveCompressor(s3Client, archiveWriter, verifier); + compressor.rebuildBefore = Date.parse('2026-08-22T16:49:12.000Z'); + compressor._console = { error: vi.fn(), log: vi.fn() }; + + await compressor.process(); + + expect(verifier.verify).toHaveBeenCalledOnce(); + expect(s3Client.uploads).toEqual([]); + expect(archiveWriter.write).not.toHaveBeenCalled(); + }); + + it('rebuilds a completed archive containing macOS metadata', async () => { + let s3Client = new FakeS3Client; + s3Client.completed = true; + let verifier = { + verify: vi.fn(async () => { + throw new AppleDoubleArchiveError('._alpha.json'); + }), + }; let archiveWriter = { async write(sourceDirectory, archivePath) { await fs.writeFile(archivePath, 'replacement archive'); }, }; - let compressor = new ArchiveCompressor(s3Client, archiveWriter); + let compressor = new ArchiveCompressor(s3Client, archiveWriter, verifier); compressor.rebuildBefore = Date.parse('2026-08-22T16:49:12.000Z'); compressor._console = { error: vi.fn(), log: vi.fn() }; diff --git a/app/data/ArchiveVerifier.mjs b/app/data/ArchiveVerifier.mjs index cfb4b22..1f687a6 100644 --- a/app/data/ArchiveVerifier.mjs +++ b/app/data/ArchiveVerifier.mjs @@ -1,11 +1,20 @@ import crypto from 'node:crypto'; import { spawn } from 'node:child_process'; +import path from 'node:path'; import { Transform } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import * as tar from 'tar-stream'; class ArchiveContentError extends Error {} +export class AppleDoubleArchiveError extends ArchiveContentError +{ + constructor(filePath) { + super(`Tar archive contains an unexpected file: ${filePath}`); + this.path = filePath; + } +} + function processCompletion(child) { let stderr = ''; child.stderr.setEncoding('utf8'); @@ -137,6 +146,10 @@ export default class ArchiveVerifier let manifestFile = manifestByPath.get(header.name); let sourceObject = sourceByPath.get(header.name); if (!manifestFile || !sourceObject) { + if (path.posix.basename(header.name).startsWith('._')) { + throw new AppleDoubleArchiveError(header.name); + } + throw new ArchiveContentError(`Tar archive contains an unexpected file: ${header.name}`); } verified.add(header.name); diff --git a/app/data/ArchiveVerifier.test.mjs b/app/data/ArchiveVerifier.test.mjs index 490de7b..1687791 100644 --- a/app/data/ArchiveVerifier.test.mjs +++ b/app/data/ArchiveVerifier.test.mjs @@ -1,7 +1,7 @@ import { Readable } from 'node:stream'; import { describe, expect, it } from 'vitest'; import * as tar from 'tar-stream'; -import ArchiveVerifier from './ArchiveVerifier.mjs'; +import ArchiveVerifier, { AppleDoubleArchiveError } from './ArchiveVerifier.mjs'; async function createTar(files) { let pack = tar.pack(); @@ -114,6 +114,23 @@ describe('ArchiveVerifier', () => { )).rejects.toThrow('Tar archive contains an unexpected file: extra.json'); }); + it('identifies unexpected macOS AppleDouble files', async () => { + let archive = await createTar([ + ['._alpha.json', 'metadata'], + ['alpha.json', 'alpha\n'], + ]); + let verifier = new ArchiveVerifier(identityDecompressor); + + let error = await verifier.verify( + Readable.from(archive), + manifestFor(archive), + [sourceObject()], + ).catch(reason => reason); + + expect(error).toBeInstanceOf(AppleDoubleArchiveError); + expect(error.path).toBe('._alpha.json'); + }); + it('rejects a file whose MD5 does not match its S3 ETag', async () => { let archive = await createTar([['alpha.json', 'alpha\n']]); let verifier = new ArchiveVerifier(identityDecompressor);