mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-18 14:08:21 +00:00
Select mutants from changed lines and ignore logging-only mutants; rank survivors by risk topic from whole-word identifier matches. Publish one quality report comment, edited in place, with the verdict, change risk (coverage gaps and CRAP refactor hints) and mutation survivors grouped by line. Bundle the report data into one quality-report artifact; shard artifacts are internal. Map api/ source aliases into coverage evidence and enforce a per-shard execution budget. Move Redis cluster runtime files out of the checkout.
674 lines
29 KiB
TypeScript
674 lines
29 KiB
TypeScript
import yaml from 'js-yaml';
|
|
import { test } from 'node:test';
|
|
import { tmpdir } from 'node:os';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { createRequire } from 'node:module';
|
|
import { dirname, join } from 'node:path';
|
|
import { strict as assert } from 'node:assert';
|
|
import { spawnSync } from 'node:child_process';
|
|
import {
|
|
cpSync,
|
|
existsSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
rmSync,
|
|
symlinkSync,
|
|
writeFileSync,
|
|
} from 'node:fs';
|
|
|
|
import type { MutationReport } from './merge.mts';
|
|
|
|
import { git, reportJobs } from './changes.mts';
|
|
import { assertMutants, mutantKey } from './shards.mts';
|
|
import { mutationSummary } from './merge.mts';
|
|
import {
|
|
REPORT_JOB,
|
|
REPORT_ARTIFACT,
|
|
FINDINGS_ARTIFACT,
|
|
REPORT_UPLOAD_STEP,
|
|
FINDINGS_UPLOAD_STEP,
|
|
} from './comment.mts';
|
|
|
|
interface Step {
|
|
if?: string;
|
|
id?: string;
|
|
name?: string;
|
|
uses?: string;
|
|
run?: string;
|
|
with?: Record<string, string | number | boolean>;
|
|
env?: Record<string, string>;
|
|
'working-directory'?: string;
|
|
'timeout-minutes'?: number;
|
|
'continue-on-error'?: boolean;
|
|
}
|
|
interface Job {
|
|
name?: string;
|
|
env?: Record<string, string>;
|
|
services?: Record<string, { image: string }>;
|
|
'timeout-minutes'?: string | number;
|
|
if?: string;
|
|
permissions?: Record<string, string>;
|
|
needs?: string[];
|
|
concurrency?: { group: string; 'cancel-in-progress': boolean; queue?: string };
|
|
outputs?: Record<string, string>;
|
|
strategy?: { matrix: { include?: string; workspace?: string } };
|
|
defaults?: { run: { 'working-directory': string } };
|
|
steps: Step[];
|
|
}
|
|
interface Workflow {
|
|
'run-name'?: string;
|
|
on: {
|
|
pull_request?: { paths: string[] };
|
|
issue_comment?: { types: string[] };
|
|
merge_group?: { types: string[] };
|
|
workflow_dispatch: {
|
|
inputs: {
|
|
base: { default: string };
|
|
scope: { default: string };
|
|
static: { default: string };
|
|
};
|
|
};
|
|
};
|
|
permissions: Record<string, string>;
|
|
env: Record<string, string>;
|
|
jobs: Record<string, Job>;
|
|
}
|
|
|
|
const workflow = yaml.load(
|
|
readFileSync(new URL('../../.github/workflows/change-quality.yml', import.meta.url), 'utf8'),
|
|
) as Workflow;
|
|
const require = createRequire(import.meta.url);
|
|
const root = fileURLToPath(new URL('../../', import.meta.url));
|
|
const checkouts = (job: Job) => job.steps.filter(({ uses }) => uses === 'actions/checkout@v5');
|
|
const report = workflow.jobs.reports;
|
|
const runner = report.steps.find(({ name }) => name === 'Generate advisory report')!;
|
|
|
|
// Only the workflow revision may supply tooling; the merge ref must never supply measured source.
|
|
test('workflow separates tooling, source, installs and artifacts without changing comparison refs', () => {
|
|
assert.deepEqual(workflow.permissions, { contents: 'read' });
|
|
assert.equal(workflow.on.workflow_dispatch.inputs.base.default, 'origin/dev');
|
|
assert.deepEqual(workflow.env, {
|
|
QUALITY_BASE: "${{ github.event.pull_request.base.sha || inputs.base || 'origin/dev' }}",
|
|
QUALITY_HEAD: '${{ github.event.pull_request.head.sha || github.sha }}',
|
|
});
|
|
for (const name of ['verify', 'select', 'mutation-summary']) {
|
|
assert.equal(checkouts(workflow.jobs[name]).length, 1);
|
|
assert.equal(checkouts(workflow.jobs[name])[0].with?.ref, '${{ github.sha }}');
|
|
}
|
|
assert.deepEqual(
|
|
checkouts(report).map(({ with: options }) => options),
|
|
[
|
|
{
|
|
ref: '${{ env.QUALITY_HEAD }}',
|
|
path: 'source',
|
|
'fetch-depth': 0,
|
|
'persist-credentials': false,
|
|
},
|
|
{ ref: '${{ github.sha }}', path: 'tooling', 'persist-credentials': false },
|
|
],
|
|
);
|
|
for (const job of Object.values(workflow.jobs)) {
|
|
for (const step of checkouts(job)) assert.equal(step.with?.['persist-credentials'], false);
|
|
}
|
|
assert.equal(report.defaults?.run['working-directory'], 'source');
|
|
const installs = report.steps.filter(({ run }) => run?.startsWith('npm ci'));
|
|
assert.deepEqual(
|
|
installs.map(({ run, 'working-directory': cwd }) => [run, cwd]),
|
|
[
|
|
['npm ci', undefined],
|
|
['npm ci --ignore-scripts --workspaces=false', 'tooling'],
|
|
],
|
|
);
|
|
assert.equal(runner['working-directory'], undefined);
|
|
assert.match(runner.run!, /node \.\.\/tooling\/scripts\/quality\/run\.mts/);
|
|
assert.equal(report['timeout-minutes'], "${{ matrix.task.check == 'mutation' && 240 || 45 }}");
|
|
assert.equal(workflow.on.workflow_dispatch.inputs.scope.default, 'all');
|
|
assert.equal(workflow.on.workflow_dispatch.inputs.static.default, 'off');
|
|
assert.match(report.env!.QUALITY_STATIC, /inputs\.static \|\| 'off'/);
|
|
assert.match(runner.run!, /QUALITY_STATIC:-off/);
|
|
assert.equal(
|
|
workflow.jobs.select.steps.find(({ id }) => id === 'select')?.env?.QUALITY_SCOPE,
|
|
"${{ needs.request.outputs.scope || inputs.scope || 'all' }}",
|
|
);
|
|
assert.deepEqual(report.strategy?.matrix, {
|
|
include: '${{ fromJSON(needs.select.outputs.reports) }}',
|
|
});
|
|
assert.equal(workflow.jobs.select.outputs?.reports, '${{ steps.select.outputs.reports }}');
|
|
assert.equal(
|
|
workflow.jobs.select.outputs?.mutationWorkspaces,
|
|
'${{ steps.select.outputs.mutationWorkspaces }}',
|
|
);
|
|
assert.equal(
|
|
workflow.jobs.select.outputs?.hasMutations,
|
|
'${{ steps.select.outputs.hasMutations }}',
|
|
);
|
|
assert.equal(
|
|
workflow.jobs['mutation-summary'].if,
|
|
"always() && needs.select.outputs.hasMutations == 'true' && " +
|
|
"(needs.reports.result == 'success' || needs.reports.result == 'failure')",
|
|
);
|
|
assert.deepEqual(workflow.jobs['mutation-summary'].strategy?.matrix, {
|
|
workspace: '${{ fromJSON(needs.select.outputs.mutationWorkspaces) }}',
|
|
});
|
|
assert.equal(
|
|
report.steps.find(({ name }) => name === 'Build shared packages')?.env?.QUALITY_WORKSPACE,
|
|
'${{ matrix.workspace }}',
|
|
);
|
|
assert.equal(
|
|
report.steps.find(({ uses }) => uses === 'actions/upload-artifact@v6')?.with?.path,
|
|
"source/coverage/changes/${{ matrix.workspace }}/${{ matrix.task.check }}${{ matrix.task.check == 'mutation' && format('-{0}', matrix.task.shard) || '' }}/",
|
|
);
|
|
});
|
|
|
|
test('PR updates retain PostgreSQL contracts without starting source quality reports', () => {
|
|
const select = workflow.jobs.select;
|
|
assert.equal(
|
|
select.steps.find(({ id }) => id === 'select')?.if,
|
|
"github.event_name != 'pull_request'",
|
|
);
|
|
const postgresSelection = select.steps.find(({ id }) => id === 'postgres')!;
|
|
assert.equal(postgresSelection.if, "github.event_name != 'issue_comment'");
|
|
assert.match(postgresSelection.run!, /git diff --name-only/);
|
|
assert.ok(workflow.on.pull_request?.paths.includes('packages/data-schemas/src/postgres/**'));
|
|
assert.ok(workflow.on.pull_request?.paths.includes('packages/api/src/app/persistence*'));
|
|
const postgres = workflow.jobs.postgres;
|
|
assert.equal(
|
|
postgres.if,
|
|
"always() && needs.select.result == 'success' && needs.select.outputs.postgresChanged == 'true'",
|
|
);
|
|
assert.deepEqual(postgres.needs, ['select']);
|
|
assert.equal(checkouts(postgres)[0].with?.ref, '${{ env.QUALITY_HEAD }}');
|
|
assert.equal(postgres.services?.postgres.image, 'postgres:16-alpine');
|
|
assert.equal(report.services?.postgres.image, 'postgres:16-alpine');
|
|
assert.ok(postgres.steps.some(({ run }) => run?.includes('npm run test:postgres')));
|
|
assert.ok(postgres.steps.some(({ run }) => run?.includes('delete-user.recovery.spec.js')));
|
|
assert.equal(report.env?.PG_TEST_URL, 'postgresql://postgres@127.0.0.1:5432/postgres');
|
|
const prewarm = report.steps.findIndex(
|
|
({ uses }) => uses === './tooling/.github/actions/mongodb-binary',
|
|
);
|
|
assert.ok(prewarm > 0 && prewarm < report.steps.indexOf(runner));
|
|
assert.equal(report.steps[prewarm].with?.['working-directory'], 'source');
|
|
});
|
|
|
|
test('API risk coverage provisions pinned host Redis before executing source tests', () => {
|
|
const provision = report.steps.findIndex(
|
|
({ uses }) => uses === './tooling/.github/actions/redis-binaries',
|
|
);
|
|
const node = report.steps.findIndex(({ uses }) => uses === 'actions/setup-node@v5');
|
|
const tooling = report.steps.findIndex(({ name }) => name === 'Check out quality tooling');
|
|
assert.ok(provision > node && provision > tooling, 'trusted provisioning needs Node and tooling');
|
|
assert.ok(provision < report.steps.indexOf(runner), 'test fixtures need host Redis on PATH');
|
|
assert.equal(
|
|
report.steps[provision].if,
|
|
"matrix.workspace == 'api' && matrix.task.check == 'risk'",
|
|
);
|
|
assert.equal(report.steps[provision]['timeout-minutes'], 8);
|
|
assert.ok(
|
|
workflow.jobs.verify.steps.some(
|
|
({ run }) => run === 'node --test .github/actions/redis-binaries/provision.test.mjs',
|
|
),
|
|
);
|
|
assert.ok(workflow.on.pull_request?.paths.includes('.github/actions/redis-binaries/**'));
|
|
});
|
|
|
|
test('comment commands from humans and bots reach trusted admission before report jobs', () => {
|
|
const request = workflow.jobs.request;
|
|
assert.equal(
|
|
request.if,
|
|
"github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, '/quality')",
|
|
);
|
|
assert.equal(checkouts(request)[0].with?.ref, '${{ github.sha }}');
|
|
assert.ok(request.steps.some(({ run }) => run === 'node scripts/quality/comment.mts request'));
|
|
for (const name of ['verify', 'select']) {
|
|
assert.match(workflow.jobs[name].if!, /needs\.request\.outputs\.accepted == 'true'/);
|
|
}
|
|
});
|
|
|
|
test('comment publication is trusted and serialized while PR execution stays read-only', (t) => {
|
|
assert.ok(workflow.on.pull_request?.paths.includes('scripts/quality/**'));
|
|
assert.equal(
|
|
workflow.jobs.select.if,
|
|
"always() && github.event_name != 'merge_group' && (github.event_name != 'issue_comment' || needs.request.outputs.accepted == 'true')",
|
|
);
|
|
assert.deepEqual(workflow.on.issue_comment?.types, ['created']);
|
|
assert.equal(workflow.jobs.request.permissions?.issues, 'write');
|
|
assert.match(workflow['run-name']!, /Quality PR #\{0\} request \{1\}/);
|
|
for (const name of ['verify', 'select', 'reports', 'mutation-summary', 'report']) {
|
|
assert.deepEqual(workflow.jobs[name].permissions ?? workflow.permissions, { contents: 'read' });
|
|
}
|
|
for (const name of ['reports', 'mutation-summary']) {
|
|
const upload = workflow.jobs[name].steps.find(
|
|
({ uses }) => uses === 'actions/upload-artifact@v6',
|
|
);
|
|
assert.equal(upload?.with?.overwrite, true, 'reruns replace failed artifacts');
|
|
// A failed run still uploads its record (`failure.json` for failing tests) for the bundle.
|
|
assert.equal(upload?.if, 'always()', `${name} uploads after a failed step`);
|
|
// Intermediate data: never linked from the comment and kept only long enough to bundle.
|
|
assert.match(String(upload?.with?.name), /^quality-internal-/u);
|
|
assert.equal(upload?.with?.['retention-days'], 1);
|
|
}
|
|
const report = workflow.jobs.report;
|
|
assert.equal(report.name, REPORT_JOB, 'comment.mts finds the report job by this name');
|
|
assert.deepEqual(report.needs, ['request', 'select', 'reports', 'mutation-summary']);
|
|
assert.ok(report.steps.every(({ run }) => !run?.includes('npm')));
|
|
assert.equal(checkouts(report)[0].with?.['sparse-checkout'], 'scripts/quality');
|
|
const downloads = report.steps.filter(({ uses }) =>
|
|
uses?.startsWith('actions/download-artifact'),
|
|
);
|
|
const internal = [
|
|
'quality-internal-risk-data-schemas',
|
|
'quality-internal-risk-api',
|
|
'quality-internal-risk-server',
|
|
'quality-internal-mutation-data-schemas',
|
|
].map((name) => `${name}-\${{ env.QUALITY_HEAD }}`);
|
|
// bundle.mts reads `<directory>/<artifact name>/`; a single pattern match extracts into `path`.
|
|
assert.deepEqual(
|
|
downloads.map(({ with: options }) => [options?.pattern, options?.path]),
|
|
internal.map((name) => [name, `coverage/internal/${name}`]),
|
|
);
|
|
assert.ok(downloads.every((step) => step['continue-on-error'] === undefined));
|
|
const uploads = report.steps.filter(({ uses }) => uses === 'actions/upload-artifact@v6');
|
|
assert.deepEqual(
|
|
uploads.map(({ name, with: options }) => [name, options?.name, options?.['retention-days']]),
|
|
[
|
|
[REPORT_UPLOAD_STEP, REPORT_ARTIFACT, 14],
|
|
[FINDINGS_UPLOAD_STEP, FINDINGS_ARTIFACT, 1],
|
|
],
|
|
'comment.mts trusts these exact step names before reading or linking the artifacts',
|
|
);
|
|
assert.ok(uploads.every(({ with: options }) => options?.overwrite === true));
|
|
assert.equal(uploads[1].with?.path, 'coverage/quality-report/findings.json');
|
|
for (const name of ['request', 'progress']) {
|
|
const job = workflow.jobs[name];
|
|
assert.equal(checkouts(job)[0].with?.ref, '${{ github.sha }}');
|
|
assert.ok(job.steps.every(({ run }) => !run?.includes('npm')));
|
|
assert.equal(job.permissions?.['pull-requests'], 'write');
|
|
assert.equal(job.permissions?.statuses, 'write');
|
|
assert.equal(job.permissions?.actions, 'write');
|
|
assert.equal(job.concurrency?.group, 'quality-decision-${{ github.event.issue.number }}');
|
|
assert.equal(job.concurrency?.['cancel-in-progress'], false);
|
|
assert.equal(job.concurrency?.queue, 'max');
|
|
}
|
|
const publisher = yaml.load(
|
|
readFileSync(new URL('../../.github/workflows/quality-decision.yml', import.meta.url), 'utf8'),
|
|
) as Workflow;
|
|
assert.equal(checkouts(publisher.jobs.publish)[0].with?.ref, '${{ github.workflow_sha }}');
|
|
assert.equal(
|
|
publisher.jobs.publish.concurrency?.group,
|
|
'quality-decision-${{ needs.resolve.outputs.pr }}',
|
|
);
|
|
assert.equal(publisher.jobs.publish.concurrency?.['cancel-in-progress'], false);
|
|
assert.equal(publisher.jobs.publish.concurrency?.queue, 'max');
|
|
assert.equal(publisher.permissions.actions, 'write');
|
|
assert.deepEqual(workflow.on.merge_group, { types: ['checks_requested'] });
|
|
assert.equal(publisher.on.merge_group, undefined);
|
|
const bridge = workflow.jobs['queue-request'];
|
|
assert.equal(bridge.if, "github.event_name == 'merge_group'");
|
|
assert.deepEqual(bridge.permissions ?? workflow.permissions, { contents: 'read' });
|
|
assert.equal(checkouts(bridge).length, 0);
|
|
assert.ok(bridge.steps.every(({ run, uses }) => !uses && run?.startsWith('echo ')));
|
|
for (const name of ['verify', 'select'])
|
|
assert.match(workflow.jobs[name].if!, /github.event_name != 'merge_group'/);
|
|
const queue = publisher.jobs.queue;
|
|
assert.deepEqual(queue.needs, ['resolve']);
|
|
assert.equal(queue.if, "needs.resolve.outputs.queue != ''");
|
|
assert.deepEqual(queue.permissions, {
|
|
contents: 'read',
|
|
'pull-requests': 'read',
|
|
actions: 'read',
|
|
statuses: 'write',
|
|
});
|
|
assert.equal(checkouts(queue)[0].with?.ref, '${{ github.workflow_sha }}');
|
|
assert.equal(checkouts(queue)[0].with?.['persist-credentials'], false);
|
|
assert.equal(queue.concurrency?.group, 'quality-queue-${{ github.event.workflow_run.head_sha }}');
|
|
assert.ok(queue.steps.some(({ run }) => run === 'node scripts/quality/queue.mts'));
|
|
assert.ok(
|
|
queue.steps.every(
|
|
({ run, uses }) => !run?.includes('npm') && !uses?.startsWith('actions/download-artifact'),
|
|
),
|
|
);
|
|
assert.equal(publisher.jobs.inline, undefined);
|
|
assert.ok(publisher.jobs.publish.steps.every(({ run }) => !run?.includes('npm')));
|
|
// The privileged job downloads only the small findings file from the triggering run.
|
|
const fetched = publisher.jobs.publish.steps.filter(({ uses }) =>
|
|
uses?.startsWith('actions/download-artifact'),
|
|
);
|
|
assert.deepEqual(
|
|
fetched.map(({ with: options }) => [options?.name, options?.['run-id'], options?.path]),
|
|
[[FINDINGS_ARTIFACT, '${{ github.event.workflow_run.id }}', 'findings']],
|
|
);
|
|
assert.equal(fetched[0]['continue-on-error'], true);
|
|
const directory = mkdtempSync(join(tmpdir(), 'quality-event-'));
|
|
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
|
for (const [event, expected] of [
|
|
[{ pull_request: { number: 12 } }, 'pr=12\n'],
|
|
[{ workflow_run: { event: 'merge_group', status: 'completed', id: 55 } }, 'queue=55\n'],
|
|
[{ workflow_run: { event: 'merge_group', status: 'in_progress', id: 55 } }, ''],
|
|
[
|
|
{ workflow_run: { event: 'issue_comment', display_title: 'Quality PR #13 request 99' } },
|
|
'pr=13\n',
|
|
],
|
|
[{ workflow_run: { event: 'pull_request', display_title: 'Quality PR #13 request 99' } }, ''],
|
|
[
|
|
{
|
|
workflow_run: {
|
|
event: 'issue_comment',
|
|
display_title: 'Quality PR #13 request $(touch injected)',
|
|
},
|
|
},
|
|
'',
|
|
],
|
|
] as const) {
|
|
const path = join(directory, 'event.json');
|
|
const output = join(directory, 'output');
|
|
writeFileSync(path, JSON.stringify(event));
|
|
writeFileSync(output, '');
|
|
const result = spawnSync('bash', ['-e', '-c', publisher.jobs.resolve.steps[0].run!], {
|
|
cwd: directory,
|
|
env: { ...process.env, GITHUB_EVENT_PATH: path, GITHUB_OUTPUT: output },
|
|
encoding: 'utf8',
|
|
});
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.equal(readFileSync(output, 'utf8'), expected);
|
|
assert.equal(existsSync(join(directory, 'injected')), false);
|
|
}
|
|
});
|
|
|
|
for (const olderHead of [true, false]) {
|
|
test(`real reports use exact head source and dependencies with ${olderHead ? 'absent' : 'edited'} head tooling`, (t) => {
|
|
const directory = mkdtempSync(join(tmpdir(), 'quality-workflow-'));
|
|
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
|
const upstream = join(directory, 'upstream');
|
|
const write = (cwd: string, file: string, content: string) => {
|
|
mkdirSync(dirname(join(cwd, file)), { recursive: true });
|
|
writeFileSync(join(cwd, file), content);
|
|
};
|
|
const commit = () => {
|
|
git(['add', '.'], upstream);
|
|
git(['commit', '--no-gpg-sign', '-qm', 'fixture'], upstream);
|
|
return git(['rev-parse', 'HEAD'], upstream).trim();
|
|
};
|
|
mkdirSync(upstream);
|
|
git(['init', '-q', '-b', 'dev'], upstream);
|
|
git(['config', 'user.name', 'Quality Test'], upstream);
|
|
git(['config', 'user.email', 'quality@example.invalid'], upstream);
|
|
write(upstream, 'package.json', '{"name":"quality-fixture","private":true}');
|
|
write(upstream, 'package-lock.json', '{}');
|
|
write(upstream, '.gitignore', 'node_modules/\ncoverage/\n.stryker-tmp/\n');
|
|
const owners = [
|
|
{ name: 'data-schemas', path: 'packages/data-schemas', config: 'jest.config.mjs' },
|
|
{ name: 'api', path: 'packages/api', config: 'jest.config.mjs' },
|
|
{ name: 'server', path: 'api', config: 'jest.config.js' },
|
|
];
|
|
const mode = olderHead ? 'on' : 'off';
|
|
const sourceCode =
|
|
'module.exports = (value) => value ? 1 : 0;\nmodule.exports.untouched = () => 3;\nmodule.exports.staticValue = (() => 1)();\nfunction hybrid(value) { if (value > 0) return 1; return 0; } module.exports.hybrid = hybrid; module.exports.loaded = hybrid(1);\n';
|
|
for (const owner of owners) {
|
|
write(upstream, `${owner.path}/src/feature.js`, sourceCode);
|
|
write(
|
|
upstream,
|
|
`${owner.path}/${owner.config}`,
|
|
`${owner.config.endsWith('.mjs') ? 'export default' : 'module.exports ='} { testEnvironment: "node" };`,
|
|
);
|
|
write(
|
|
upstream,
|
|
`${owner.path}/src/feature.test.js`,
|
|
"const feature = require('./feature'); test('uses head dependency', () => { expect(feature(true)).toBe(require('head-only')); expect(feature(false)).toBe(0); expect(feature.untouched()).toBe(3); expect(feature.staticValue).toBe(2); expect(feature.hybrid(1)).toBe(2); expect(feature.hybrid(0)).toBe(0); expect(feature.loaded).toBe(2); });",
|
|
);
|
|
}
|
|
const original = commit();
|
|
mkdirSync(join(upstream, 'scripts'), { recursive: true });
|
|
cpSync(join(root, 'scripts/quality'), join(upstream, 'scripts/quality'), { recursive: true });
|
|
cpSync(join(root, 'scripts/crap.mts'), join(upstream, 'scripts/crap.mts'));
|
|
const base = commit();
|
|
git(['checkout', '-qb', 'pr', olderHead ? original : base], upstream);
|
|
for (const owner of owners)
|
|
write(
|
|
upstream,
|
|
`${owner.path}/src/feature.js`,
|
|
sourceCode
|
|
.replace('? 1', '? 2')
|
|
.replaceAll('() => 1', '() => 2')
|
|
.replace('return 1', 'return 2'),
|
|
);
|
|
if (!olderHead) {
|
|
const selector = readFileSync(join(upstream, 'scripts/quality/changes.mts'), 'utf8');
|
|
write(
|
|
upstream,
|
|
'scripts/quality/changes.mts',
|
|
`${selector}\nconsole.log('PR tooling edit');\n`,
|
|
);
|
|
}
|
|
const head = commit();
|
|
git(['checkout', '-qb', 'merge', base], upstream);
|
|
git(['merge', '--no-gpg-sign', '--no-ff', '-qm', 'merge fixture', head], upstream);
|
|
const merge = git(['rev-parse', 'HEAD'], upstream).trim();
|
|
assert.notEqual(merge, head);
|
|
|
|
for (const [path, ref] of [
|
|
['source', head],
|
|
['tooling', merge],
|
|
]) {
|
|
git(['clone', '-q', upstream, join(directory, path)], directory);
|
|
git(['checkout', '-q', '--detach', ref], join(directory, path));
|
|
}
|
|
const source = join(directory, 'source');
|
|
const tooling = join(directory, 'tooling');
|
|
assert.equal(existsSync(join(source, 'scripts/quality/run.mts')), !olderHead);
|
|
symlinkSync(join(root, 'node_modules'), join(tooling, 'node_modules'), 'junction');
|
|
// A distinct head-owned Jest records both risk CLI and mutation module resolution.
|
|
const marker = join(directory, 'head-jest.log');
|
|
const record = `require('node:fs').appendFileSync(${JSON.stringify(marker)}, 'head Jest\\n');`;
|
|
write(
|
|
source,
|
|
'node_modules/jest/package.json',
|
|
'{"name":"jest","main":"index.js","exports":{".":"./index.js","./bin/jest":"./bin/jest.js"}}',
|
|
);
|
|
write(
|
|
source,
|
|
'node_modules/jest/index.js',
|
|
`${record} module.exports = require(${JSON.stringify(require.resolve('jest'))});`,
|
|
);
|
|
write(
|
|
source,
|
|
'node_modules/jest/bin/jest.js',
|
|
`${record} require(${JSON.stringify(require.resolve('jest/bin/jest'))});`,
|
|
);
|
|
write(source, 'node_modules/head-only/index.js', 'module.exports = 2;');
|
|
for (const dependency of [
|
|
'jest-cli',
|
|
'jest-config',
|
|
'jest-environment-node',
|
|
'istanbul-lib-coverage',
|
|
]) {
|
|
symlinkSync(
|
|
dirname(createRequire(require.resolve('jest')).resolve(`${dependency}/package.json`)),
|
|
join(source, 'node_modules', dependency),
|
|
'junction',
|
|
);
|
|
}
|
|
// The source-alias pass compiles packages/api/src with the measured checkout's own Babel,
|
|
// which the real repository installs for api/'s Jest transform.
|
|
mkdirSync(join(source, 'node_modules/@babel'), { recursive: true });
|
|
for (const dependency of [
|
|
'babel-jest',
|
|
'@babel/core',
|
|
'@babel/preset-env',
|
|
'@babel/preset-typescript',
|
|
'@babel/plugin-transform-dynamic-import',
|
|
'@babel/plugin-transform-modules-commonjs',
|
|
]) {
|
|
symlinkSync(
|
|
dirname(require.resolve(`${dependency}/package.json`)),
|
|
join(source, 'node_modules', dependency),
|
|
'junction',
|
|
);
|
|
}
|
|
git(['update-ref', 'refs/remotes/origin/dev', base], source);
|
|
// The real job inherits a workflow-level QUALITY_BASE/QUALITY_HEAD (and, on other triggers,
|
|
// QUALITY_SCOPE/QUALITY_CHECK/QUALITY_WORKSPACE/QUALITY_STATIC/QUALITY_SHARD/QUALITY_PR/
|
|
// QUALITY_REPOSITORY) from its own CI run. Drop every inherited QUALITY_* value so this
|
|
// fixture's explicit values below are the only ones each step call ever sees.
|
|
const inherited = Object.fromEntries(
|
|
Object.entries(process.env).filter(([key]) => !key.startsWith('QUALITY_')),
|
|
);
|
|
const env = {
|
|
...inherited,
|
|
QUALITY_HEAD: head,
|
|
QUALITY_WORKSPACE: 'data-schemas',
|
|
QUALITY_STATIC: mode,
|
|
// The merge step's --repository/--pr: a fixed fixture value, standing in for the real
|
|
// workflow's github.repository / pull_request.number context.
|
|
QUALITY_REPOSITORY: 'quality-fixture/quality-fixture',
|
|
QUALITY_PR: '1',
|
|
GITHUB_OUTPUT: '',
|
|
GITHUB_STEP_SUMMARY: '',
|
|
};
|
|
const run = (command: string, cwd: string, extra: Record<string, string>) => {
|
|
const result = spawnSync('bash', ['-e', '-c', command], {
|
|
cwd,
|
|
env: { ...env, ...extra },
|
|
encoding: 'utf8',
|
|
timeout: 45000,
|
|
maxBuffer: 8 * 1024 * 1024,
|
|
});
|
|
assert.equal(result.status, 0, `${result.error ?? ''}\n${result.stdout}\n${result.stderr}`);
|
|
return result.stdout;
|
|
};
|
|
const select = workflow.jobs.select.steps.find(({ id }) => id === 'select');
|
|
assert.ok(select?.run);
|
|
for (const comparison of ['origin/dev', base]) {
|
|
const output = run(select.run, tooling, {
|
|
QUALITY_BASE: comparison,
|
|
QUALITY_SCOPE: 'all',
|
|
});
|
|
assert.match(output, new RegExp(`"head": "${head}"`));
|
|
assert.match(output, new RegExp(`"base": "${olderHead ? original : base}"`));
|
|
assert.match(output, /"data-schemas"/);
|
|
if (!olderHead) assert.match(output, /PR tooling edit/);
|
|
}
|
|
// Dispatch measures github.sha itself, with either the default ref or an explicit base SHA.
|
|
for (const comparison of ['origin/dev', base]) {
|
|
const output = run(select.run, tooling, {
|
|
QUALITY_HEAD: merge,
|
|
QUALITY_BASE: comparison,
|
|
QUALITY_SCOPE: 'all',
|
|
});
|
|
assert.match(output, new RegExp(`"head": "${merge}"`));
|
|
assert.match(output, new RegExp(`"base": "${base}"`));
|
|
}
|
|
for (const {
|
|
workspace,
|
|
task: { check, shard },
|
|
} of reportJobs(owners.map(({ name }) => name))) {
|
|
const file = `${owners.find(({ name }) => name === workspace)!.path}/src/feature.js`;
|
|
rmSync(marker, { force: true });
|
|
run(runner.run!, source, {
|
|
QUALITY_BASE: check === 'risk' ? 'origin/dev' : base,
|
|
QUALITY_WORKSPACE: workspace,
|
|
QUALITY_CHECK: check,
|
|
QUALITY_SHARD: String(shard),
|
|
});
|
|
const output = join(
|
|
source,
|
|
`coverage/changes/${workspace}`,
|
|
shard ? `mutation-${shard}` : check,
|
|
);
|
|
const selection = JSON.parse(readFileSync(join(output, 'selection.json'), 'utf8'));
|
|
assert.equal(selection.head, head);
|
|
assert.equal(selection.base, olderHead ? original : base);
|
|
assert.deepEqual(
|
|
selection.files.map((item: { file: string }) => item.file),
|
|
[file],
|
|
);
|
|
if (check === 'risk') {
|
|
assert.match(readFileSync(marker, 'utf8'), /head Jest/);
|
|
const risk = JSON.parse(readFileSync(join(output, 'risk.json'), 'utf8'));
|
|
assert.equal(risk.functions.length, 3, 'unchanged sibling must not be scored');
|
|
assert.equal(risk.functions[0].file, file);
|
|
assert.equal(risk.functions[0].statementCoverage, 1);
|
|
assert.equal(risk.head, head);
|
|
} else {
|
|
const completion = JSON.parse(readFileSync(join(output, 'completion.json'), 'utf8'));
|
|
assert.equal(completion.complete, true);
|
|
assert.equal(completion.index, shard);
|
|
if (completion.total) {
|
|
assert.match(readFileSync(marker, 'utf8'), /head Jest/);
|
|
const progress = JSON.parse(readFileSync(join(output, 'progress.json'), 'utf8'));
|
|
assert.equal(progress.complete, true);
|
|
assert.equal(progress.staticMutants, mode === 'on');
|
|
assert.equal(progress.completed, completion.total);
|
|
}
|
|
cpSync(
|
|
output,
|
|
join(
|
|
tooling,
|
|
`coverage/shards/quality-internal-mutation-data-schemas-${head}-shard-${shard}`,
|
|
),
|
|
{ recursive: true },
|
|
);
|
|
}
|
|
assert.equal(git(['rev-parse', 'HEAD'], source).trim(), head);
|
|
assert.equal(git(['diff', 'HEAD', '--', file], source), '');
|
|
}
|
|
const combine = workflow.jobs['mutation-summary'].steps.find(
|
|
({ name }) => name === 'Verify and combine mutation results',
|
|
)!;
|
|
// The real job sets QUALITY_BASE at job level, so give the combine step the same base SHA
|
|
// each shard's report was generated against.
|
|
run(combine.run!, tooling, { QUALITY_BASE: base });
|
|
const combined = JSON.parse(
|
|
readFileSync(join(tooling, 'coverage/changes/data-schemas/mutation/mutation.json'), 'utf8'),
|
|
);
|
|
const mutants: MutationReport['files'][string]['mutants'] =
|
|
combined.files['packages/data-schemas/src/feature.js'].mutants;
|
|
console.log(
|
|
JSON.stringify({
|
|
fixture: 'exact-head sharded Stryker',
|
|
static: mode,
|
|
planned: mutants.length,
|
|
ignored: mutants.filter(({ status }) => status === 'Ignored').length,
|
|
tested: mutants.filter(({ status }) =>
|
|
['Killed', 'Timeout', 'Survived', 'NoCoverage'].includes(status),
|
|
).length,
|
|
}),
|
|
);
|
|
assert.ok(mutants.length > 0);
|
|
assert.equal(
|
|
mutants.some((mutant: { status: string }) => mutant.status === 'Ignored'),
|
|
mode === 'off',
|
|
);
|
|
assert.ok(
|
|
mutants.some(
|
|
(mutant) =>
|
|
mutant.static && mutant.location.start.line === 4 && mutant.status !== 'Ignored',
|
|
),
|
|
`hybrid mutants remain tested: ${JSON.stringify(mutants)}`,
|
|
);
|
|
run(
|
|
'node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --workspace data-schemas --check mutation --static "$QUALITY_STATIC"',
|
|
source,
|
|
{ QUALITY_BASE: base },
|
|
);
|
|
const unsharded = JSON.parse(
|
|
readFileSync(join(source, 'coverage/changes/data-schemas/mutation/mutation.json'), 'utf8'),
|
|
);
|
|
const keys = (report: MutationReport) =>
|
|
Object.entries(report.files).flatMap(([file, value]) =>
|
|
value.mutants.map((mutant) =>
|
|
mutantKey(file, { ...mutant, replacement: mutant.replacement! }),
|
|
),
|
|
);
|
|
assertMutants(keys(unsharded), keys(combined));
|
|
assert.equal(
|
|
mutationSummary(unsharded, 'data-schemas', mode === 'on'),
|
|
mutationSummary(combined, 'data-schemas', mode === 'on'),
|
|
);
|
|
|
|
assert.match(
|
|
readFileSync(join(tooling, 'coverage/changes/data-schemas/mutation/summary.md'), 'utf8'),
|
|
/All four shards completed/,
|
|
);
|
|
});
|
|
}
|