diff --git a/app/data/RunLimits.mjs b/app/data/RunLimits.mjs index 94025b7..60e23ec 100644 --- a/app/data/RunLimits.mjs +++ b/app/data/RunLimits.mjs @@ -22,8 +22,10 @@ export default class RunLimits { this.deferArchive = deferArchive; } - // Archive snapshots waiting to be written, when deferArchive is set - heldSnapshots = []; + // Archive snapshots waiting to be written, when deferArchive is set, by archive key. Several + // updaters can save the same file in one run (each region's FestivalUpdater writes the + // combined festivals.json); a newer copy replaces the one held, so each is written once. + heldSnapshots = new Map(); // Set once work was left for a later run because the deadline passed incomplete = false; diff --git a/app/data/archive.mjs b/app/data/archive.mjs index 459f750..811691b 100644 --- a/app/data/archive.mjs +++ b/app/data/archive.mjs @@ -31,9 +31,9 @@ export async function archiveSnapshot({ key, text, name }, console = prefixedCon * @returns {Promise<{ archived: number, failed: number }>} How many were written and not */ export async function flushArchive(limits) { - // Several updaters can save the same file in one run (each region's FestivalUpdater writes - // the combined festivals.json); only the newest copy of each snapshot is written. - let snapshots = [...new Map(limits.heldSnapshots.splice(0).map(snapshot => [snapshot.key, snapshot])).values()]; + let snapshots = [...limits.heldSnapshots.values()]; + limits.heldSnapshots.clear(); + let limit = pLimit(4); let results = await Promise.all(snapshots.map(snapshot => limit(() => archiveSnapshot(snapshot)))); let archived = results.filter(Boolean).length; diff --git a/app/data/archive.test.mjs b/app/data/archive.test.mjs index 4a8b68d..36617b1 100644 --- a/app/data/archive.test.mjs +++ b/app/data/archive.test.mjs @@ -28,7 +28,9 @@ describe('archive', () => { it('writes only the newest copy of a snapshot saved more than once', async () => { let limits = new RunLimits({ deferArchive: true }); let festivals = text => ({ key: '2026-11-08.12-00-00.festivals.json', text, name: 'festivals' }); - limits.heldSnapshots.push(festivals('{"US":1}'), snapshot('gear'), festivals('{"US":1,"EU":1}')); + for (let held of [festivals('{"US":1}'), snapshot('gear'), festivals('{"US":1,"EU":1}')]) { + limits.heldSnapshots.set(held.key, held); + } expect(await flushArchive(limits)).toEqual({ archived: 2, failed: 0 }); expect(testBuckets().archive.text('2026-11-08.12-00-00.festivals.json')).toBe('{"US":1,"EU":1}'); @@ -36,7 +38,9 @@ describe('archive', () => { it('flushes held snapshots once, counting the ones that failed', async () => { let limits = new RunLimits({ deferArchive: true }); - limits.heldSnapshots.push(snapshot('gear'), snapshot('coop'), snapshot('schedules')); + for (let held of [snapshot('gear'), snapshot('coop'), snapshot('schedules')]) { + limits.heldSnapshots.set(held.key, held); + } let write = storage().archive.writeText.bind(storage().archive); vi.spyOn(storage().archive, 'writeText').mockImplementation((key, text) => @@ -48,7 +52,7 @@ describe('archive', () => { '2026-11-08.12-00-00.gear.json', '2026-11-08.12-00-00.schedules.json', ]); - expect(limits.heldSnapshots).toEqual([]); + expect(limits.heldSnapshots.size).toBe(0); expect(await flushArchive(limits)).toEqual({ archived: 0, failed: 0 }); }); }); diff --git a/app/data/updaters/DataUpdater.mjs b/app/data/updaters/DataUpdater.mjs index a49c6de..2825d0a 100644 --- a/app/data/updaters/DataUpdater.mjs +++ b/app/data/updaters/DataUpdater.mjs @@ -200,7 +200,7 @@ export default class DataUpdater let snapshot = { key: this.getArchivePath(this.filename), text: s, name: this.filename }; if (this.limits.deferArchive) { - this.limits.heldSnapshots.push(snapshot); + this.limits.heldSnapshots.set(snapshot.key, snapshot); } else { await archiveSnapshot(snapshot, { warn: (...args) => this.console.warn(...args) }); } diff --git a/app/data/updaters/DataUpdater.test.mjs b/app/data/updaters/DataUpdater.test.mjs index c368e82..d9ff480 100644 --- a/app/data/updaters/DataUpdater.test.mjs +++ b/app/data/updaters/DataUpdater.test.mjs @@ -116,7 +116,7 @@ describe('DataUpdater', () => { expect(testBuckets().public.json('data/gear.json')).toEqual({ data: { a: 1 } }); expect(testBuckets().archive.objects.size).toBe(0); - expect(limits.heldSnapshots).toEqual([ + expect([...limits.heldSnapshots.values()]).toEqual([ { key: '2026-11-08.12-00-00.gear.json', text: JSON.stringify({ data: { a: 1 } }), name: 'gear' }, ]); });