import { spawn } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { parseSync } from "oxc-parser"; import { setup as ensureTestDb } from "../ensure-test-db.ts"; import { ROOT } from "../living-docs/ast.ts"; import { applyMutant, MODULE_SCOPE_ID, type Mutant, type MutationPlan, } from "./mutants.ts"; import { type BaselineRun, type MutantRun, RUN_DESCRIPTOR_ENV, } from "./run-descriptor.ts"; import type { TestCandidate } from "./test-selection.ts"; export interface Target { /** Repo relative. */ file: string; source: string; plan: MutationPlan; candidates: TestCandidate[]; } export type MutantStatus = | "caught" | "timeout" | "survived" | "noCoverage" | "ignored" | "error"; export interface MutantResult { mutant: Mutant; status: MutantStatus; /** The first failing test, when a test failure (not a timeout) caught the mutant. */ killedBy: { file: string; test: string | null } | null; /** How many test files the mutant was run against. */ testFileCount: number; /** How many test files execute the mutated function, more than `testFileCount` when capped. */ coveringTestFileCount: number; durationMs: number; } export interface TargetResult { target: Target; /** Candidate test files that load the target, repo relative. */ testFiles: string[]; results: MutantResult[]; } export interface AuditResult { targets: TargetResult[]; /** Candidate test files failing against the unmutated source, left out of every run. */ failingTestFiles: string[]; baselineMs: number; mutantsMs: number; } interface Baseline { /** Test files (repo relative) by target index and scope id. */ coverage: Map>>; durations: Map; failingTestFiles: string[]; wallMs: number; } const EMPTY_BASELINE: Baseline = { coverage: new Map(), durations: new Map(), failingTestFiles: [], wallMs: 0, }; interface VitestJson { testResults: Array<{ name: string; status: string; startTime: number; endTime: number; assertionResults: Array<{ fullName: string; status: string }>; }>; } const VITEST_BIN = path.join(ROOT, "node_modules/vitest/vitest.mjs"); const CONFIG_PATH = path.join(import.meta.dirname, "vitest.config.ts"); const MIN_TIMEOUT_MS = 20_000; const TIMEOUT_FACTOR = 4; /** * Runs the baseline (the unmutated, instrumented targets against every candidate test file) and * then one Vitest process per mutant against the test files that execute the mutated function. */ export async function runAudit({ targets, concurrency, maxTestFiles, onBaselineDone, onMutantDone, }: { targets: Target[]; concurrency: number; /** Run a mutant against at most this many of the nearest test files, 0 for all of them. */ maxTestFiles: number; onBaselineDone: (testFileCount: number, wallMs: number) => void; onMutantDone: (result: MutantResult) => void; }): Promise { assertMutantsParse(targets); ensureTestDb(); const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "mutation-")); const children = new Set(); const killChildren = () => { for (const pid of children) killProcessGroup(pid); }; const onInterrupt = () => { killChildren(); process.exit(130); }; process.on("SIGINT", onInterrupt); try { const candidateFiles = [ ...new Set( targets.flatMap((target) => target.candidates.map((c) => c.file)), ), ]; const baseline = candidateFiles.length > 0 ? await runBaseline({ targets, testFiles: candidateFiles, workDir, children, }) : EMPTY_BASELINE; onBaselineDone(candidateFiles.length, baseline.wallMs); const mutantsStartedAt = performance.now(); const targetResults: TargetResult[] = targets.map((target, index) => ({ target, testFiles: [ ...(baseline.coverage.get(index)?.get(MODULE_SCOPE_ID) ?? []), ].sort((a, b) => a.localeCompare(b)), results: [], })); const jobs = targets.flatMap((target, targetIndex) => target.plan.mutants.map((mutant) => ({ target, targetIndex, mutant })), ); let nextJob = 0; const work = async () => { for (let job = jobs[nextJob++]; job; job = jobs[nextJob++]) { const result = await runMutant({ ...job, baseline, maxTestFiles, workDir, children, }); targetResults[job.targetIndex].results.push(result); onMutantDone(result); } }; await Promise.all(Array.from({ length: concurrency }, work)); for (const targetResult of targetResults) { targetResult.results.sort((a, b) => a.mutant.id - b.mutant.id); } return { targets: targetResults, failingTestFiles: baseline.failingTestFiles, baselineMs: baseline.wallMs, mutantsMs: performance.now() - mutantsStartedAt, }; } finally { process.off("SIGINT", onInterrupt); killChildren(); fs.rmSync(workDir, { recursive: true, force: true }); } } async function runBaseline({ targets, testFiles, workDir, children, }: { targets: Target[]; testFiles: string[]; workDir: string; children: Set; }): Promise { const startedAt = performance.now(); const coverageFile = path.join(workDir, "coverage.tsv"); const outputFile = path.join(workDir, "baseline.json"); fs.writeFileSync(coverageFile, ""); const descriptor: BaselineRun = { mode: "baseline", moduleCacheDir: moduleCacheDir(workDir), targets: targets.map((target) => path.join(ROOT, target.file)), coverageFile, }; const { output } = await runVitest({ descriptor, descriptorPath: path.join(workDir, "baseline-run.json"), args: ["--reporter=json", `--outputFile=${outputFile}`, ...testFiles], timeoutMs: 30 * 60_000, children, captureOutput: true, }); const json = readVitestJson(outputFile); if (!json) { throw new Error(`The baseline run produced no results:\n${output}`); } const durations = new Map(); const failingTestFiles: string[] = []; for (const testResult of json.testResults) { const file = path.relative(ROOT, testResult.name); durations.set(file, testResult.endTime - testResult.startTime); if (testResult.status !== "passed") failingTestFiles.push(file); } const failing = new Set(failingTestFiles); const coverage: Baseline["coverage"] = new Map(); for (const line of fs.readFileSync(coverageFile, "utf8").split("\n")) { if (!line) continue; const [testFile, targetIndex, scopeId] = line.split("\t"); const file = path.relative(ROOT, testFile); if (failing.has(file)) continue; const scopes = coverage.get(Number(targetIndex)) ?? new Map(); const files = scopes.get(Number(scopeId)) ?? new Set(); files.add(file); scopes.set(Number(scopeId), files); coverage.set(Number(targetIndex), scopes); } return { coverage, durations, failingTestFiles: failingTestFiles.sort((a, b) => a.localeCompare(b)), wallMs: performance.now() - startedAt, }; } async function runMutant({ target, targetIndex, mutant, baseline, maxTestFiles, workDir, children, }: { target: Target; targetIndex: number; mutant: Mutant; baseline: Baseline; maxTestFiles: number; workDir: string; children: Set; }): Promise { const base = { mutant, killedBy: null, testFileCount: 0, coveringTestFileCount: 0, durationMs: 0, }; if (mutant.ignoredReason !== null) return { ...base, status: "ignored" }; const distances = new Map( target.candidates.map((candidate) => [candidate.file, candidate.distance]), ); const coveringTestFiles = [ ...(baseline.coverage.get(targetIndex)?.get(mutant.scopeId) ?? []), ].sort( (a, b) => distances.get(a)! - distances.get(b)! || baseline.durations.get(a)! - baseline.durations.get(b)! || a.localeCompare(b), ); if (coveringTestFiles.length === 0) return { ...base, status: "noCoverage" }; const testFiles = maxTestFiles > 0 ? coveringTestFiles.slice(0, maxTestFiles) : coveringTestFiles; const startedAt = performance.now(); const runName = `${targetIndex}-${mutant.id}`; const outputFile = path.join(workDir, `${runName}.json`); const descriptor: MutantRun = { mode: "mutant", moduleCacheDir: moduleCacheDir(workDir), target: path.join(ROOT, target.file), start: mutant.start, end: mutant.end, replacement: mutant.replacement, testFiles: testFiles.map((file) => path.join(ROOT, file)), }; const baselineMs = testFiles.reduce( (sum, file) => sum + (baseline.durations.get(file) ?? 0), 0, ); const { exitCode, timedOut } = await runVitest({ descriptor, descriptorPath: path.join(workDir, `${runName}-run.json`), args: [ "--bail=1", "--maxWorkers=1", "--reporter=json", `--outputFile=${outputFile}`, ...testFiles, ], timeoutMs: MIN_TIMEOUT_MS + TIMEOUT_FACTOR * baselineMs, children, captureOutput: false, }); const json = readVitestJson(outputFile); fs.rmSync(outputFile, { force: true }); const result = { ...base, testFileCount: testFiles.length, coveringTestFileCount: coveringTestFiles.length, durationMs: performance.now() - startedAt, }; if (timedOut) return { ...result, status: "timeout" }; if (exitCode === 0) return { ...result, status: "survived" }; // without results Vitest itself failed, which says nothing about the mutant if (!json) return { ...result, status: "error" }; return { ...result, status: "caught", killedBy: firstFailure(json) }; } function firstFailure(json: VitestJson) { const failedFile = json.testResults.find( (testResult) => testResult.status === "failed", ); if (!failedFile) return null; return { file: path.relative(ROOT, failedFile.name), test: failedFile.assertionResults.find( (assertion) => assertion.status === "failed", )?.fullName ?? null, }; } function runVitest({ descriptor, descriptorPath, args, timeoutMs, children, captureOutput, }: { descriptor: BaselineRun | MutantRun; descriptorPath: string; args: string[]; timeoutMs: number; children: Set; captureOutput: boolean; }) { fs.writeFileSync(descriptorPath, JSON.stringify(descriptor)); return new Promise<{ exitCode: number | null; timedOut: boolean; output: string; }>((resolve, reject) => { const child = spawn( process.execPath, [VITEST_BIN, "run", "--config", CONFIG_PATH, ...args], { cwd: ROOT, // its own process group, so a timeout can take the test workers down with it detached: true, stdio: ["ignore", "ignore", captureOutput ? "pipe" : "ignore"], env: { ...process.env, VITE_SITE_DOMAIN: "http://localhost:5173", [RUN_DESCRIPTOR_ENV]: descriptorPath, }, }, ); const pid = child.pid; if (pid === undefined) { reject(new Error("Could not start Vitest")); return; } children.add(pid); let output = ""; child.stderr?.on("data", (chunk) => { output += chunk; }); let timedOut = false; const timer = setTimeout(() => { timedOut = true; killProcessGroup(pid); }, timeoutMs); child.on("error", reject); child.on("close", (exitCode) => { clearTimeout(timer); children.delete(pid); fs.rmSync(descriptorPath, { force: true }); resolve({ exitCode, timedOut, output }); }); }); } function moduleCacheDir(workDir: string) { return path.join(workDir, "module-cache"); } function killProcessGroup(pid: number) { try { process.kill(-pid, "SIGKILL"); } catch { // already gone } } function readVitestJson(file: string): VitestJson | null { try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return null; } } /** A mutant that does not parse would fail every test and count as caught. */ function assertMutantsParse(targets: Target[]) { for (const target of targets) { for (const mutant of target.plan.mutants) { const { errors } = parseSync( target.file, applyMutant(target.source, mutant), ); if (errors.length > 0) { throw new Error( `${target.file}:${mutant.line} mutant "${mutant.change}" does not parse: ${errors[0].message}`, ); } } } }