Only rebuild archives with macOS metadata

app/data/ArchiveCompressor.mjs

@@ -12,7 +12,8 @@ import {
   S3Client,
 } from '@aws-sdk/client-s3';
 import prefixedConsole from '../common/prefixedConsole.mjs';
-import { createArchiveManifest } from './ArchiveManifest.mjs';
+import { createArchiveManifest, parseArchiveManifest } from './ArchiveManifest.mjs';
+import ArchiveVerifier, { AppleDoubleArchiveError } from './ArchiveVerifier.mjs';
 import TarZstdWriter from './TarZstdWriter.mjs';
 
 const downloadLimit = 5;
@@ -60,9 +61,14 @@ export default class ArchiveCompressor
   maxDays = Infinity;
   rebuildBefore = null;
 
-  constructor(s3Client, archiveWriter = new TarZstdWriter) {
+  constructor(
+    s3Client,
+    archiveWriter = new TarZstdWriter,
+    archiveVerifier = new ArchiveVerifier,
+  ) {
     this._client = s3Client;
     this.archiveWriter = archiveWriter;
+    this.archiveVerifier = archiveVerifier;
   }
 
   async process() {
@@ -77,7 +83,9 @@ export default class ArchiveCompressor
 
     try {
       let candidates = await this.getCandidates();
-      this.console.log(`Found ${candidates.length} dates to archive`);
+      this.console.log(
+        `Found ${candidates.length} dates to ${this.rebuildBefore === null ? 'archive' : 'inspect'}`,
+      );
 
       for (let candidate of candidates) {
         if (compressed >= this.maxDays) {
@@ -144,7 +152,7 @@ export default class ArchiveCompressor
 
   async previewDate(candidate) {
     let objects = await this.getReadyObjects(candidate);
-    if (!objects) {
+    if (!objects || candidate.exists && !await this.needsRebuild(candidate, objects)) {
       return false;
     }
 
@@ -159,7 +167,7 @@ export default class ArchiveCompressor
 
   async archiveDate(candidate) {
     let objects = await this.getReadyObjects(candidate);
-    if (!objects) {
+    if (!objects || candidate.exists && !await this.needsRebuild(candidate, objects)) {
       return false;
     }
 
@@ -215,6 +223,63 @@ export default class ArchiveCompressor
     return objects;
   }
 
+  async needsRebuild(candidate, objects) {
+    let manifest = await this.getManifest(candidate);
+    let archiveStream = await this.getArchive(candidate.archiveKey);
+    let sourceObjects = objects.map(object => {
+      if (typeof object.etag !== 'string') {
+        throw new Error(`S3 returned incomplete metadata for ${object.key}`);
+      }
+
+      return {
+        path: object.path,
+        bytes: object.bytes,
+        etag: object.etag,
+      };
+    });
+
+    try {
+      await this.archiveVerifier.verify(archiveStream, manifest, sourceObjects);
+    } catch (error) {
+      if (error instanceof AppleDoubleArchiveError) {
+        return true;
+      }
+
+      throw error;
+    }
+
+    this.console.log(`Skipping ${candidate.date}; existing archive is valid`);
+
+    return false;
+  }
+
+  async getManifest(candidate) {
+    let response = await this.s3Client.send(new GetObjectCommand({
+      Bucket: process.env.AWS_S3_ARCHIVE_BUCKET,
+      Key: candidate.manifestKey,
+    }));
+    if (!response.Body) {
+      throw new Error(`S3 returned no body for ${candidate.manifestKey}`);
+    }
+
+    return parseArchiveManifest(
+      await response.Body.transformToString(),
+      candidate.archiveKey,
+    );
+  }
+
+  async getArchive(key) {
+    let response = await this.s3Client.send(new GetObjectCommand({
+      Bucket: process.env.AWS_S3_ARCHIVE_BUCKET,
+      Key: key,
+    }));
+    if (!response.Body) {
+      throw new Error(`S3 returned no body for ${key}`);
+    }
+
+    return response.Body;
+  }
+
   async downloadObjects(prefix, objects, sourceDirectory) {
     let files = [];
     let nextObject = 0;
@@ -341,8 +406,10 @@ export default class ArchiveCompressor
         }
 
         return {
+          path: this.relativePath(prefix, object.Key),
           key: object.Key,
           bytes: object.Size,
+          etag: object.ETag,
           lastModified: object.LastModified,
         };
       });

app/data/ArchiveCompressor.test.mjs

@@ -7,6 +7,28 @@ import {
 } from '@aws-sdk/client-s3';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import ArchiveCompressor from './ArchiveCompressor.mjs';
+import { AppleDoubleArchiveError } from './ArchiveVerifier.mjs';
+
+const archiveKey = '2025/03/2025-03-05.tar.zst';
+const manifestKey = `${archiveKey}.manifest.json`;
+
+function completedManifest() {
+  return {
+    version: 1,
+    date: '2025-03-05',
+    createdAt: '2026-08-20T00:00:00.000Z',
+    archive: {
+      path: archiveKey,
+      bytes: 7,
+      hash: `sha256:${'a'.repeat(64)}`,
+    },
+    files: [{
+      path: 'alpha.json',
+      bytes: 6,
+      hash: `sha256:${'b'.repeat(64)}`,
+    }],
+  };
+}
 
 class FakeS3Client
 {
@@ -28,6 +50,17 @@ class FakeS3Client
     if (command instanceof GetObjectCommand) {
       this.downloads.push(command.input.Key);
 
+      if (command.input.Key === manifestKey) {
+        return {
+          Body: {
+            transformToString: async () => JSON.stringify(completedManifest()),
+          },
+        };
+      }
+      if (command.input.Key === archiveKey) {
+        return { Body: Readable.from('archive') };
+      }
+
       return { Body: Readable.from(this.files.get(command.input.Key)) };
     }
 
@@ -66,6 +99,7 @@ class FakeS3Client
       return { Contents: [...this.files]
         .filter(([Key]) => Key.startsWith(prefix))
         .map(([Key, body]) => ({
+          ETag: '"9f9f90dbe3e5ee1218c86b8839db1995"',
           Key,
           LastModified: new Date('2025-03-05T23:45:00.000Z'),
           Size: body.length,
@@ -187,15 +221,36 @@ describe('ArchiveCompressor', () => {
     expect(archiveWriter.write).not.toHaveBeenCalled();
   });
 
-  it('rebuilds a completed archive older than the repair cutoff', async () => {
+  it('leaves a valid completed archive older than the repair cutoff alone', async () => {
     let s3Client = new FakeS3Client;
     s3Client.completed = true;
+    let verifier = { verify: vi.fn() };
+    let archiveWriter = { write: vi.fn() };
+    let compressor = new ArchiveCompressor(s3Client, archiveWriter, verifier);
+    compressor.rebuildBefore = Date.parse('2026-08-22T16:49:12.000Z');
+    compressor._console = { error: vi.fn(), log: vi.fn() };
+
+    await compressor.process();
+
+    expect(verifier.verify).toHaveBeenCalledOnce();
+    expect(s3Client.uploads).toEqual([]);
+    expect(archiveWriter.write).not.toHaveBeenCalled();
+  });
+
+  it('rebuilds a completed archive containing macOS metadata', async () => {
+    let s3Client = new FakeS3Client;
+    s3Client.completed = true;
+    let verifier = {
+      verify: vi.fn(async () => {
+        throw new AppleDoubleArchiveError('._alpha.json');
+      }),
+    };
     let archiveWriter = {
       async write(sourceDirectory, archivePath) {
         await fs.writeFile(archivePath, 'replacement archive');
       },
     };
-    let compressor = new ArchiveCompressor(s3Client, archiveWriter);
+    let compressor = new ArchiveCompressor(s3Client, archiveWriter, verifier);
     compressor.rebuildBefore = Date.parse('2026-08-22T16:49:12.000Z');
     compressor._console = { error: vi.fn(), log: vi.fn() };
 

app/data/ArchiveVerifier.mjs

@@ -1,11 +1,20 @@
 import crypto from 'node:crypto';
 import { spawn } from 'node:child_process';
+import path from 'node:path';
 import { Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import * as tar from 'tar-stream';
 
 class ArchiveContentError extends Error {}
 
+export class AppleDoubleArchiveError extends ArchiveContentError
+{
+  constructor(filePath) {
+    super(`Tar archive contains an unexpected file: ${filePath}`);
+    this.path = filePath;
+  }
+}
+
 function processCompletion(child) {
   let stderr = '';
   child.stderr.setEncoding('utf8');
@@ -137,6 +146,10 @@ export default class ArchiveVerifier
     let manifestFile = manifestByPath.get(header.name);
     let sourceObject = sourceByPath.get(header.name);
     if (!manifestFile || !sourceObject) {
+      if (path.posix.basename(header.name).startsWith('._')) {
+        throw new AppleDoubleArchiveError(header.name);
+      }
+
       throw new ArchiveContentError(`Tar archive contains an unexpected file: ${header.name}`);
     }
     verified.add(header.name);

app/data/ArchiveVerifier.test.mjs

@@ -1,7 +1,7 @@
 import { Readable } from 'node:stream';
 import { describe, expect, it } from 'vitest';
 import * as tar from 'tar-stream';
-import ArchiveVerifier from './ArchiveVerifier.mjs';
+import ArchiveVerifier, { AppleDoubleArchiveError } from './ArchiveVerifier.mjs';
 
 async function createTar(files) {
   let pack = tar.pack();
@@ -114,6 +114,23 @@ describe('ArchiveVerifier', () => {
     )).rejects.toThrow('Tar archive contains an unexpected file: extra.json');
   });
 
+  it('identifies unexpected macOS AppleDouble files', async () => {
+    let archive = await createTar([
+      ['._alpha.json', 'metadata'],
+      ['alpha.json', 'alpha\n'],
+    ]);
+    let verifier = new ArchiveVerifier(identityDecompressor);
+
+    let error = await verifier.verify(
+      Readable.from(archive),
+      manifestFor(archive),
+      [sourceObject()],
+    ).catch(reason => reason);
+
+    expect(error).toBeInstanceOf(AppleDoubleArchiveError);
+    expect(error.path).toBe('._alpha.json');
+  });
+
   it('rejects a file whose MD5 does not match its S3 ETag', async () => {
     let archive = await createTar([['alpha.json', 'alpha\n']]);
     let verifier = new ArchiveVerifier(identityDecompressor);