mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-18 14:08:21 +00:00
Select mutants from changed lines and ignore logging-only mutants; rank survivors by risk topic from whole-word identifier matches. Publish one quality report comment, edited in place, with the verdict, change risk (coverage gaps and CRAP refactor hints) and mutation survivors grouped by line. Bundle the report data into one quality-report artifact; shard artifacts are internal. Map api/ source aliases into coverage evidence and enforce a per-shard execution budget. Move Redis cluster runtime files out of the checkout.
306 lines
12 KiB
TypeScript
306 lines
12 KiB
TypeScript
import { test } from 'node:test';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { strict as assert } from 'node:assert';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
|
|
|
import type { ShardResult, MutationReport } from './merge.mts';
|
|
import type { ShardPlan } from './shards.mts';
|
|
|
|
import { mergeReports, mutationSummary } from './merge.mts';
|
|
import {
|
|
assertMutants,
|
|
describeMutatedScope,
|
|
isReconstructedShardComplete,
|
|
mutantKey,
|
|
mutatedScope,
|
|
} from './shards.mts';
|
|
import { mutationTarget } from './risk.mts';
|
|
|
|
test('four real Stryker runs preserve the unsharded mutant set, including nested and static code', () => {
|
|
const directory = realpathSync(mkdtempSync(join(tmpdir(), 'quality-shards-')));
|
|
const source = `function factory() {
|
|
function positive(value) { return value > 0; }
|
|
return { positive, enabled: true };
|
|
}
|
|
function negative(value) { return value < 0; }
|
|
function zero(value) { return value === 0; }
|
|
function untouched() { return false; }
|
|
module.exports = { ...factory(), negative, zero, untouched };\n`;
|
|
writeFileSync(join(directory, 'subject[one].js'), source);
|
|
writeFileSync(join(directory, 'package-lock.json'), '{}');
|
|
writeFileSync(
|
|
join(directory, 'test.cjs'),
|
|
`const assert = require('node:assert/strict');
|
|
const subject = require('./subject[one].js');
|
|
assert.equal(subject.positive(1), true);
|
|
assert.equal(subject.positive(0), false);
|
|
assert.equal(subject.negative(-1), true);
|
|
assert.equal(subject.zero(0), true);
|
|
assert.equal(subject.enabled, true);\n`,
|
|
);
|
|
const script = `import { Stryker } from ${JSON.stringify(import.meta.resolve('@stryker-mutator/core'))};
|
|
import { mutationTarget } from ${JSON.stringify(new URL('./risk.mts', import.meta.url).href)};
|
|
import { planShards, mutantKey, assertMutants, mutationPatterns } from ${JSON.stringify(new URL('./shards.mts', import.meta.url).href)};
|
|
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
const source = readFileSync('subject[one].js', 'utf8');
|
|
const target = mutationTarget('subject[one].js', source, [{start: 1, count: 6}]);
|
|
const plan = await planShards([target], 'base', 'head', 'fixture');
|
|
writeFileSync('plan.json', JSON.stringify(plan));
|
|
for (const index of [0,1,2,3,4]) {
|
|
const shard = plan.shards[index - 1];
|
|
const targets = shard?.targets ?? [target];
|
|
const output = 'report-' + index;
|
|
mkdirSync(output);
|
|
writeFileSync(output + '/functions.json', JSON.stringify(targets));
|
|
if (shard?.expected.length === 0) continue;
|
|
const results = await new Stryker({
|
|
plugins: [${JSON.stringify(new URL('./progress.mts', import.meta.url).href)}],
|
|
testRunner: 'command', commandRunner: { command: ${JSON.stringify(JSON.stringify(process.execPath) + ' test.cjs')} },
|
|
ignorePatterns: ['**', '!subject[[]one].js', '!test.cjs'],
|
|
mutate: mutationPatterns(targets), reporters: ['function-progress', 'json'],
|
|
jsonReporter: {fileName: output + '/mutation.json'},
|
|
coverageAnalysis: 'off', concurrency: 1, timeoutMS: 1000, thresholds: {break: 0},
|
|
}).runMutationTest();
|
|
if (shard) assertMutants(shard.expected, results.map(m => mutantKey('subject[one].js', m)));
|
|
}`;
|
|
writeFileSync(join(directory, 'run.mjs'), script);
|
|
try {
|
|
const run = spawnSync(process.execPath, ['run.mjs'], {
|
|
cwd: directory,
|
|
encoding: 'utf8',
|
|
timeout: 120000,
|
|
});
|
|
assert.equal(run.status, 0, run.stdout + run.stderr);
|
|
const plan: ShardPlan = JSON.parse(readFileSync(join(directory, 'plan.json'), 'utf8'));
|
|
const baseline: MutationReport = JSON.parse(
|
|
readFileSync(join(directory, 'report-0/mutation.json'), 'utf8'),
|
|
);
|
|
const results: ShardResult[] = plan.shards.map((shard) => ({
|
|
plan,
|
|
completion: {
|
|
digest: plan.digest,
|
|
index: shard.index,
|
|
complete: true,
|
|
total: shard.expected.length,
|
|
},
|
|
report: shard.expected.length
|
|
? JSON.parse(readFileSync(join(directory, `report-${shard.index}/mutation.json`), 'utf8'))
|
|
: undefined,
|
|
}));
|
|
const merged = mergeReports(results);
|
|
const keys = (report: MutationReport) =>
|
|
Object.entries(report.files).flatMap(([file, value]) =>
|
|
value.mutants.map((mutant) =>
|
|
mutantKey(file, { ...mutant, replacement: mutant.replacement! }),
|
|
),
|
|
);
|
|
assertMutants(keys(baseline), keys(merged));
|
|
assert.ok(keys(merged).length > 5);
|
|
assert.ok(plan.shards.filter((shard) => shard.expected.length > 0).length >= 3);
|
|
assert.ok(
|
|
Object.values(merged.files)
|
|
.flatMap((file) => file.mutants)
|
|
.every((mutant) => mutant.location.end.line <= 6),
|
|
);
|
|
for (const shard of plan.shards.filter((item) => item.expected.length)) {
|
|
const progress = JSON.parse(
|
|
readFileSync(join(directory, `report-${shard.index}/progress.json`), 'utf8'),
|
|
);
|
|
assert.equal(progress.complete, true);
|
|
assert.equal(progress.total, shard.expected.length);
|
|
}
|
|
assert.throws(() => mergeReports(results.slice(1)), /four shards/);
|
|
const changed = structuredClone(results);
|
|
changed[0].plan = { ...changed[0].plan, head: 'wrong-head' };
|
|
assert.throws(() => mergeReports(changed), /mismatched shard/);
|
|
const mixedModes = structuredClone(results);
|
|
mixedModes[0].plan = { ...mixedModes[0].plan, staticMutants: false };
|
|
assert.throws(() => mergeReports(mixedModes), /mismatched shard/);
|
|
const missing = structuredClone(results);
|
|
missing[0].completion.complete = false;
|
|
assert.throws(() => mergeReports(missing), /incomplete/);
|
|
assert.throws(
|
|
() => mergeReports([results[0], results[0], results[2], results[3]]),
|
|
/duplicate/,
|
|
);
|
|
const incomplete = structuredClone(results);
|
|
Object.values(incomplete[0].report!.files)[0].mutants.pop();
|
|
assert.throws(() => mergeReports(incomplete), /Mutation set differs/);
|
|
} finally {
|
|
rmSync(directory, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('combined scores use mutant counts and reject duplicates', () => {
|
|
const result = (id: string, status: 'Killed' | 'Survived') => ({
|
|
id,
|
|
status,
|
|
mutatorName: 'BooleanLiteral',
|
|
replacement: 'false',
|
|
location: { start: { line: 1, column: 1 }, end: { line: 1, column: 5 } },
|
|
});
|
|
const report: MutationReport = {
|
|
schemaVersion: '2',
|
|
thresholds: { high: 90, low: 70 },
|
|
files: {
|
|
'a.js': { source: 'true', language: 'javascript', mutants: [result('1', 'Killed')] },
|
|
'b.js': {
|
|
source: 'true',
|
|
language: 'javascript',
|
|
mutants: Array.from({ length: 9 }, (_, i) => result(String(i + 2), 'Survived')),
|
|
},
|
|
},
|
|
};
|
|
assert.match(mutationSummary(report, 'fixture'), /10.00%/);
|
|
report.files['a.js'].mutants.push({
|
|
...result('ignored', 'Killed'),
|
|
status: 'Ignored',
|
|
static: true,
|
|
});
|
|
assert.match(mutationSummary(report, 'fixture', false), /10.00%/);
|
|
assert.match(mutationSummary(report, 'fixture', false), /Static=off/);
|
|
assert.match(mutationSummary(report, 'fixture', false), /Ignored \| 1/);
|
|
assert.throws(() => assertMutants(['a', 'b'], ['a', 'a']), /duplicate/);
|
|
assert.throws(() => assertMutants(['a'], ['b']), /unexpected/);
|
|
});
|
|
|
|
test('merging namespaces test references and refuses unfinished or conflicting source reports', () => {
|
|
const reports: MutationReport[] = [1, 2, 3, 4].map((line) => ({
|
|
schemaVersion: '1.0',
|
|
thresholds: { high: 90, low: 70 },
|
|
files: {
|
|
'shared.js': {
|
|
source: 'shared source',
|
|
language: 'javascript',
|
|
mutants: [
|
|
{
|
|
id: '0',
|
|
status: 'Killed',
|
|
replacement: 'false',
|
|
mutatorName: 'BooleanLiteral',
|
|
location: { start: { line, column: 1 }, end: { line, column: 5 } },
|
|
coveredBy: ['0'],
|
|
killedBy: ['0'],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
testFiles: { 'test.js': { source: 'shared test', tests: [{ id: '0', name: 'test' }] } },
|
|
}));
|
|
const plan: ShardPlan = {
|
|
base: 'base',
|
|
head: 'head',
|
|
workspace: 'fixture',
|
|
digest: 'digest',
|
|
staticMutants: true,
|
|
shards: reports.map((report, index) => ({
|
|
index: index + 1,
|
|
targets: [],
|
|
expected: [
|
|
mutantKey('shared.js', { ...report.files['shared.js'].mutants[0], replacement: 'false' }),
|
|
],
|
|
})),
|
|
};
|
|
const results = reports.map(
|
|
(report, index): ShardResult => ({
|
|
plan,
|
|
report,
|
|
completion: { digest: plan.digest, index: index + 1, complete: true, total: 1 },
|
|
}),
|
|
);
|
|
const merged = mergeReports(results);
|
|
assert.deepEqual(
|
|
merged.testFiles!['test.js'].tests.map(({ id }) => id),
|
|
['1:0', '2:0', '3:0', '4:0'],
|
|
);
|
|
assert.deepEqual(
|
|
merged.files['shared.js'].mutants.map(({ killedBy }) => killedBy),
|
|
[['1:0'], ['2:0'], ['3:0'], ['4:0']],
|
|
);
|
|
assert.equal(reports[0].files['shared.js'].mutants[0].id, '0');
|
|
for (const status of ['Pending', 'Unknown', undefined, null, 0, {}]) {
|
|
const unfinished = structuredClone(results);
|
|
Reflect.set(unfinished[0].report!.files['shared.js'].mutants[0], 'status', status);
|
|
assert.throws(() => mergeReports(unfinished), /Incomplete/);
|
|
}
|
|
const absent = structuredClone(results);
|
|
Reflect.deleteProperty(absent[0].report!.files['shared.js'].mutants[0], 'status');
|
|
assert.throws(() => mergeReports(absent), /Incomplete/);
|
|
// RuntimeError/CompileError ("Invalid") are terminal, same as Killed or Survived: the mutated
|
|
// program could not be run to a result, but the shard still finished testing it.
|
|
for (const status of [
|
|
'Killed',
|
|
'Survived',
|
|
'Timeout',
|
|
'NoCoverage',
|
|
'CompileError',
|
|
'RuntimeError',
|
|
'Ignored',
|
|
]) {
|
|
const complete = structuredClone(results);
|
|
Reflect.set(complete[0].report!.files['shared.js'].mutants[0], 'status', status);
|
|
assert.equal(mergeReports(complete).files['shared.js'].mutants[0].status, status);
|
|
}
|
|
const changed = structuredClone(results);
|
|
changed[1].report!.files['shared.js'].source = 'other source';
|
|
assert.throws(() => mergeReports(changed), /Mismatched source/);
|
|
const emptyPlan: ShardPlan = {
|
|
...plan,
|
|
shards: plan.shards.map((s) => ({ ...s, expected: [] })),
|
|
};
|
|
const empty = mergeReports(
|
|
emptyPlan.shards.map((s) => ({
|
|
plan: emptyPlan,
|
|
completion: { digest: plan.digest, index: s.index, complete: true, total: 0 },
|
|
})),
|
|
);
|
|
assert.match(mutationSummary(empty, 'fixture'), /N\/A/);
|
|
});
|
|
|
|
test('isReconstructedShardComplete is true only once every planned key finished before the kill', () => {
|
|
// run.mts's budget-timeout race: the worker's own per-mutant callback had already recorded every
|
|
// planned mutant in progress.jsonl before the deadline killed it; only the final completion.json
|
|
// write lost the race. That must read as complete, not a false partial merge.mts would reject.
|
|
assert.equal(isReconstructedShardComplete(['a', 'b'], new Set(['a', 'b'])), true);
|
|
// Order and extra finished keys beyond the plan (never produced in practice) don't matter.
|
|
assert.equal(isReconstructedShardComplete(['a', 'b'], new Set(['b', 'a', 'c'])), true);
|
|
assert.equal(isReconstructedShardComplete([], new Set()), true);
|
|
// A genuinely unfinished planned mutant keeps the shard partial.
|
|
assert.equal(isReconstructedShardComplete(['a', 'b'], new Set(['a'])), false);
|
|
assert.equal(isReconstructedShardComplete(['a'], new Set()), false);
|
|
});
|
|
|
|
test('the shard summary counts only functions its ranges touch and names module-level files', () => {
|
|
const source = [
|
|
'export const LIMIT = 1;',
|
|
'export function first(a: number) {',
|
|
' return a + LIMIT;',
|
|
'}',
|
|
'export function second(a: number) {',
|
|
' return a - LIMIT;',
|
|
'}',
|
|
'',
|
|
].join('\n');
|
|
const both = mutationTarget('a.ts', source, [
|
|
{ start: 3, count: 1 },
|
|
{ start: 6, count: 1 },
|
|
]);
|
|
assert.equal(both.functions.length, 2);
|
|
assert.equal(both.mutate.length, 2);
|
|
// A shard keeps the file's whole function list but only some of its ranges.
|
|
const shard = { ...both, mutate: [both.mutate[1]] };
|
|
const moduleOnly = mutationTarget('b.ts', source, [{ start: 1, count: 1 }]);
|
|
assert.equal(moduleOnly.functions.length, 0);
|
|
assert.deepEqual(mutatedScope([shard, moduleOnly]), {
|
|
functions: 1,
|
|
functionFiles: 1,
|
|
files: 2,
|
|
});
|
|
assert.equal(
|
|
describeMutatedScope([shard, moduleOnly]),
|
|
'Changed functions: 1 in 1 file. Mutated files: 2, of which 1 has changes only outside functions.',
|
|
);
|
|
assert.equal(describeMutatedScope([both]), 'Changed functions: 2 in 1 file.');
|
|
});
|