Hold deferred archive snapshots in a map keyed by archive path

Each region's FestivalUpdater saves the combined festivals.json, so a run saves it several
times. A newer copy now replaces the held one instead of being added alongside it, so each
snapshot is held and uploaded once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matt Isenhower
2026-11-11 09:39:08 -08:00
parent e7388fce6a
commit 008db9b459
5 changed files with 16 additions and 10 deletions

View File

@@ -22,8 +22,10 @@ export default class RunLimits {
this.deferArchive = deferArchive;
}
// Archive snapshots waiting to be written, when deferArchive is set
heldSnapshots = [];
// Archive snapshots waiting to be written, when deferArchive is set, by archive key. Several
// updaters can save the same file in one run (each region's FestivalUpdater writes the
// combined festivals.json); a newer copy replaces the one held, so each is written once.
heldSnapshots = new Map();
// Set once work was left for a later run because the deadline passed
incomplete = false;

View File

@@ -31,9 +31,9 @@ export async function archiveSnapshot({ key, text, name }, console = prefixedCon
* @returns {Promise<{ archived: number, failed: number }>} How many were written and not
*/
export async function flushArchive(limits) {
// Several updaters can save the same file in one run (each region's FestivalUpdater writes
// the combined festivals.json); only the newest copy of each snapshot is written.
let snapshots = [...new Map(limits.heldSnapshots.splice(0).map(snapshot => [snapshot.key, snapshot])).values()];
let snapshots = [...limits.heldSnapshots.values()];
limits.heldSnapshots.clear();
let limit = pLimit(4);
let results = await Promise.all(snapshots.map(snapshot => limit(() => archiveSnapshot(snapshot))));
let archived = results.filter(Boolean).length;

View File

@@ -28,7 +28,9 @@ describe('archive', () => {
it('writes only the newest copy of a snapshot saved more than once', async () => {
let limits = new RunLimits({ deferArchive: true });
let festivals = text => ({ key: '2026-11-08.12-00-00.festivals.json', text, name: 'festivals' });
limits.heldSnapshots.push(festivals('{"US":1}'), snapshot('gear'), festivals('{"US":1,"EU":1}'));
for (let held of [festivals('{"US":1}'), snapshot('gear'), festivals('{"US":1,"EU":1}')]) {
limits.heldSnapshots.set(held.key, held);
}
expect(await flushArchive(limits)).toEqual({ archived: 2, failed: 0 });
expect(testBuckets().archive.text('2026-11-08.12-00-00.festivals.json')).toBe('{"US":1,"EU":1}');
@@ -36,7 +38,9 @@ describe('archive', () => {
it('flushes held snapshots once, counting the ones that failed', async () => {
let limits = new RunLimits({ deferArchive: true });
limits.heldSnapshots.push(snapshot('gear'), snapshot('coop'), snapshot('schedules'));
for (let held of [snapshot('gear'), snapshot('coop'), snapshot('schedules')]) {
limits.heldSnapshots.set(held.key, held);
}
let write = storage().archive.writeText.bind(storage().archive);
vi.spyOn(storage().archive, 'writeText').mockImplementation((key, text) =>
@@ -48,7 +52,7 @@ describe('archive', () => {
'2026-11-08.12-00-00.gear.json',
'2026-11-08.12-00-00.schedules.json',
]);
expect(limits.heldSnapshots).toEqual([]);
expect(limits.heldSnapshots.size).toBe(0);
expect(await flushArchive(limits)).toEqual({ archived: 0, failed: 0 });
});
});

View File

@@ -200,7 +200,7 @@ export default class DataUpdater
let snapshot = { key: this.getArchivePath(this.filename), text: s, name: this.filename };
if (this.limits.deferArchive) {
this.limits.heldSnapshots.push(snapshot);
this.limits.heldSnapshots.set(snapshot.key, snapshot);
} else {
await archiveSnapshot(snapshot, { warn: (...args) => this.console.warn(...args) });
}

View File

@@ -116,7 +116,7 @@ describe('DataUpdater', () => {
expect(testBuckets().public.json('data/gear.json')).toEqual({ data: { a: 1 } });
expect(testBuckets().archive.objects.size).toBe(0);
expect(limits.heldSnapshots).toEqual([
expect([...limits.heldSnapshots.values()]).toEqual([
{ key: '2026-11-08.12-00-00.gear.json', text: JSON.stringify({ data: { a: 1 } }), name: 'gear' },
]);
});