LibreChat/.github/workflows/change-quality.yml
Atef Bellaaj 715bf1f9e9
🧪 feat: Changed-Line Mutation Selection and Single Quality Report Comment (#18111)
Select mutants from changed lines and ignore logging-only mutants; rank survivors by risk topic from whole-word identifier matches.
Publish one quality report comment, edited in place, with the verdict, change risk (coverage gaps and CRAP refactor hints) and mutation survivors grouped by line.
Bundle the report data into one quality-report artifact; shard artifacts are internal.
Map api/ source aliases into coverage evidence and enforce a per-shard execution budget.
Move Redis cluster runtime files out of the checkout.
2026-11-16 08:00:48 -05:00

520 lines
21 KiB
YAML

name: Change Quality
run-name: >-
${{ github.event_name == 'issue_comment' && format('Quality PR #{0} request {1}', github.event.issue.number, github.event.comment.id) || format('Change Quality {0}', github.ref_name) }}
on:
merge_group:
types: [checks_requested]
pull_request:
paths:
- 'scripts/quality/**'
- 'scripts/crap*'
- '.github/workflows/change-quality.yml'
- '.github/workflows/quality-decision.yml'
- 'package.json'
- 'package-lock.json'
- '.github/actions/mongodb-binary/**'
- '.github/actions/redis-binaries/**'
- 'packages/data-schemas/src/postgres/**'
- 'packages/data-schemas/src/types/favorite*'
- 'packages/data-schemas/src/methods/favorite*'
- 'packages/data-schemas/src/methods/index*'
- 'packages/data-schemas/jest.postgres*'
- 'packages/api/src/app/persistence*'
- 'packages/api/src/app/postgres*'
- 'packages/data-provider/src/config*'
- 'api/models/index*'
- 'api/server/index*'
- 'api/server/experimental*'
- 'config/postgres*'
- 'config/delete-user*'
issue_comment:
types: [created]
workflow_dispatch:
inputs:
base:
description: Base ref to compare with
default: origin/dev
type: string
scope:
description: Report scopes (all or comma-separated data-schemas,api,server)
default: all
type: string
check:
description: Checks to run
default: both
type: choice
options: [risk, mutation, both]
static:
description: Include static mutants
default: 'off'
type: choice
options: ['on', 'off']
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'issue_comment' && github.run_id || github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
QUALITY_BASE: ${{ github.event.pull_request.base.sha || inputs.base || 'origin/dev' }}
QUALITY_HEAD: ${{ github.event.pull_request.head.sha || github.sha }}
jobs:
queue-request:
name: Request queue quality decision
if: github.event_name == 'merge_group'
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- run: echo 'Quality Decision publishes this group from trusted default-branch code.'
request:
name: Accept quality request
concurrency:
group: quality-decision-${{ github.event.issue.number }}
cancel-in-progress: false
queue: max
if: >-
github.event_name == 'issue_comment' && github.event.issue.pull_request &&
startsWith(github.event.comment.body, '/quality')
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
issues: write
pull-requests: write
actions: write
statuses: write
outputs:
accepted: ${{ steps.request.outputs.accepted }}
head: ${{ steps.request.outputs.head }}
base: ${{ steps.request.outputs.base }}
scope: ${{ steps.request.outputs.scope }}
check: ${{ steps.request.outputs.check }}
static: ${{ steps.request.outputs.static }}
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false
sparse-checkout: scripts/quality
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- id: request
env:
GH_TOKEN: ${{ github.token }}
run: node scripts/quality/comment.mts request
verify:
needs: [request]
if: always() && github.event_name != 'merge_group' && (github.event_name != 'issue_comment' || needs.request.outputs.accepted == 'true')
name: Verify quality tooling
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
cache: npm
- run: npm ci
- name: Check reporter types
run: >-
npx --no-install tsc --noEmit --target ES2023 --module NodeNext
--moduleResolution NodeNext --allowImportingTsExtensions --skipLibCheck
--strict --types node scripts/quality/*.mts
- name: Test change selection and metrics
run: node --test scripts/crap.test.mts scripts/quality/*.test.mts
- name: Test Redis binary provisioning
run: node --test .github/actions/redis-binaries/provision.test.mjs
select:
needs: [request]
if: always() && github.event_name != 'merge_group' && (github.event_name != 'issue_comment' || needs.request.outputs.accepted == 'true')
env:
QUALITY_BASE: ${{ needs.request.outputs.base || github.event.pull_request.base.sha || inputs.base || 'origin/dev' }}
QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }}
name: Select changed source
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
reports: ${{ steps.select.outputs.reports }}
mutationWorkspaces: ${{ steps.select.outputs.mutationWorkspaces }}
hasMutations: ${{ steps.select.outputs.hasMutations }}
hasChanges: ${{ steps.select.outputs.hasChanges }}
postgresChanged: ${{ steps.postgres.outputs.changed }}
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- name: Fetch requested commits
if: github.event_name == 'issue_comment'
run: |
git fetch --no-tags origin "$QUALITY_BASE" "$QUALITY_HEAD"
- name: Select workspaces
id: select
if: github.event_name != 'pull_request'
env:
QUALITY_SCOPE: ${{ needs.request.outputs.scope || inputs.scope || 'all' }}
QUALITY_CHECK: ${{ needs.request.outputs.check || inputs.check || 'both' }}
run: node scripts/quality/changes.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" --scope "$QUALITY_SCOPE" --check "${QUALITY_CHECK:-both}"
- name: Select PostgreSQL schema and lifecycle changes
id: postgres
if: github.event_name != 'issue_comment'
run: |
if git diff --name-only "$QUALITY_BASE...$QUALITY_HEAD" | grep -Eq '^(packages/data-schemas/src/(postgres/|types/favorite|methods/(favorite|index))|packages/api/src/app/(persistence|postgres)|packages/data-provider/src/config|api/(models/index|server/(index|experimental))|config/(postgres|delete-user)|packages/data-schemas/jest.postgres|package-lock.json|.github/workflows/change-quality.yml)'; then
echo 'changed=true' >> "$GITHUB_OUTPUT"
fi
postgres:
name: PostgreSQL 16 contracts and readiness
needs: [select]
if: always() && needs.select.result == 'success' && needs.select.outputs.postgresChanged == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_HOST_AUTH_METHOD: trust
ports: ['5432:5432']
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 5
env:
PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres
NODE_OPTIONS: --max-old-space-size=6144
steps:
- uses: actions/checkout@v5
with:
ref: ${{ env.QUALITY_HEAD }}
persist-credentials: false
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
cache: npm
- run: npm ci
- run: npm run build:data-provider && npm run build:data-schemas && npm run build:api
- run: npm run test:postgres --workspace packages/data-schemas
- run: npx --no-install jest src/app/persistence.spec.ts src/app/postgres.spec.ts src/favorites/handlers.spec.ts src/user/erasure.spec.ts --coverage=false --runInBand
working-directory: packages/api
- run: npx --no-install jest --config config/jest.config.js --runTestsByPath config/__tests__/delete-user.recovery.spec.js --coverage=false --runInBand
reports:
name: ${{ matrix.task.check == 'risk' && 'Change risk' || format('Mutation testing {0}/4', matrix.task.shard) }} (${{ matrix.workspace }})
needs: [request, select, verify]
if: always() && needs.select.result == 'success' && needs.verify.result == 'success' && needs.select.outputs.hasChanges == 'true'
runs-on: ubuntu-latest
# Whole edited database functions can produce thousands of mutation cases.
timeout-minutes: ${{ matrix.task.check == 'mutation' && 240 || 45 }}
defaults:
run:
working-directory: source
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.select.outputs.reports) }}
services:
redis:
image: redis:7-alpine
ports: ['6379:6379']
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 5
postgres:
image: postgres:16-alpine
env:
POSTGRES_HOST_AUTH_METHOD: trust
ports: ['5432:5432']
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 5
env:
QUALITY_BASE: ${{ needs.request.outputs.base || github.event.pull_request.base.sha || inputs.base || 'origin/dev' }}
QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }}
QUALITY_STATIC: ${{ needs.request.outputs.static || inputs.static || 'off' }}
# Wall-clock budget for one mutation shard's Stryker run; run.mts validates and defaults it.
QUALITY_MUTATION_BUDGET_MINUTES: ${{ vars.QUALITY_MUTATION_BUDGET_MINUTES }}
# Wall-clock budget for the api/+risk source-alias pass; run.mts validates and defaults it.
QUALITY_ALIAS_BUDGET_MINUTES: ${{ vars.QUALITY_ALIAS_BUDGET_MINUTES }}
PG_TEST_URL: postgresql://postgres@127.0.0.1:5432/postgres
REDIS_URI: redis://127.0.0.1:6379
NODE_OPTIONS: --max-old-space-size=6144
steps:
- uses: actions/checkout@v5
with:
ref: ${{ env.QUALITY_HEAD }}
path: source
fetch-depth: 0
persist-credentials: false
- name: Check out quality tooling
uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
path: tooling
persist-credentials: false
- name: Start Redis cluster for API coverage
if: matrix.workspace == 'api' && matrix.task.check == 'risk'
working-directory: tooling/redis-config
timeout-minutes: 5
run: |
docker run --detach --network host --name quality-redis-cluster \
--volume "$PWD:/cluster:ro" --tmpfs /var/lib/quality-redis \
--env REDIS_CLUSTER_RUNTIME_DIR=/var/lib/quality-redis \
--workdir /cluster redis:7 \
bash -c 'bash start-cluster.sh && tail -f /dev/null'
for attempt in $(seq 1 30); do
if docker exec quality-redis-cluster redis-cli -p 7001 cluster info | grep -q 'cluster_state:ok'; then
exit 0
fi
sleep 1
done
docker logs quality-redis-cluster
docker exec quality-redis-cluster sh -c 'cat /var/lib/quality-redis/logs/redis-*.log'
exit 1
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
cache: npm
cache-dependency-path: |
source/package-lock.json
tooling/package-lock.json
- name: Provision Redis tools for API coverage
if: matrix.workspace == 'api' && matrix.task.check == 'risk'
timeout-minutes: 8
uses: ./tooling/.github/actions/redis-binaries
- run: npm ci
- name: Install quality tooling
working-directory: tooling
run: npm ci --ignore-scripts --workspaces=false
- name: Build shared packages
run: |
npm run build:data-provider
if [ "$QUALITY_WORKSPACE" != data-schemas ]; then
npm run build:data-schemas
npm run build:api
fi
env:
QUALITY_WORKSPACE: ${{ matrix.workspace }}
- name: Provision MongoDB memory-server binary
uses: ./tooling/.github/actions/mongodb-binary
with:
working-directory: source
- name: Generate advisory report
env:
QUALITY_WORKSPACE: ${{ matrix.workspace }}
QUALITY_CHECK: ${{ matrix.task.check }}
QUALITY_SHARD: ${{ matrix.task.shard }}
# The api/+risk run also executes api/'s specs against packages/api/src (coverage.mts's
# cross-workspace evidence) through the @librechat/api barrel, which related-test
# selection reaches broadly; instrument and run more suites than the job default budgets.
NODE_OPTIONS: ${{ matrix.workspace == 'api' && matrix.task.check == 'risk' && '--max-old-space-size=8192' || '--max-old-space-size=6144' }}
run: |
args=()
if [ "$QUALITY_CHECK" = mutation ]; then
args+=(--shard "$QUALITY_SHARD")
fi
node ../tooling/scripts/quality/run.mts --base "$QUALITY_BASE" --head "$QUALITY_HEAD" \
--workspace "$QUALITY_WORKSPACE" --check "$QUALITY_CHECK" --static "${QUALITY_STATIC:-off}" "${args[@]}"
# Internal: read by the mutation and quality report jobs, never linked from the comment.
# Quality decisions require the combined report, so upload failures remain blocking.
- name: Upload reports
if: always()
timeout-minutes: 3
uses: actions/upload-artifact@v6
with:
name: quality-internal-${{ matrix.task.check }}-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}${{ matrix.task.check == 'mutation' && format('-shard-{0}', matrix.task.shard) || '' }}
path: source/coverage/changes/${{ matrix.workspace }}/${{ matrix.task.check }}${{ matrix.task.check == 'mutation' && format('-{0}', matrix.task.shard) || '' }}/
if-no-files-found: warn
retention-days: 1
overwrite: true
- name: Stop Redis cluster
if: always() && matrix.workspace == 'api' && matrix.task.check == 'risk'
run: docker rm -f quality-redis-cluster || true
mutation-summary:
name: Mutation report (${{ matrix.workspace }})
needs: [request, select, reports]
env:
QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }}
QUALITY_BASE: ${{ needs.request.outputs.base || github.event.pull_request.base.sha || '' }}
QUALITY_REPOSITORY: ${{ github.repository }}
# 0 for a workflow_dispatch run with no pull request to report against.
QUALITY_PR: ${{ github.event.pull_request.number || github.event.issue.number || 0 }}
# Combine only shards that ran. When verification fails or the run is cancelled, the shard
# jobs are skipped or cancelled, so there is nothing to combine and the summary is skipped.
if: >-
always() && needs.select.outputs.hasMutations == 'true' &&
(needs.reports.result == 'success' || needs.reports.result == 'failure')
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
workspace: ${{ fromJSON(needs.select.outputs.mutationWorkspaces) }}
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
cache: npm
- run: npm ci
- uses: actions/download-artifact@v6
with:
pattern: quality-internal-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}-shard-*
path: coverage/shards
- name: Verify and combine mutation results
env:
QUALITY_WORKSPACE: ${{ matrix.workspace }}
run: >-
node scripts/quality/merge.mts --directory coverage/shards
--workspace "$QUALITY_WORKSPACE"
--head "$QUALITY_HEAD"
--prefix "quality-internal-mutation-$QUALITY_WORKSPACE-$QUALITY_HEAD-"
--repository "$QUALITY_REPOSITORY"
--pr "$QUALITY_PR"
--base "$QUALITY_BASE"
# Internal: the quality report job reads this; a budget-incomplete run (a distinct,
# non-zero exit) still reaches this step via always().
- uses: actions/upload-artifact@v6
if: always()
timeout-minutes: 3
with:
name: quality-internal-mutation-${{ matrix.workspace }}-${{ env.QUALITY_HEAD }}
path: coverage/changes/${{ matrix.workspace }}/mutation/
if-no-files-found: warn
retention-days: 1
overwrite: true
report:
name: Quality report
needs: [request, select, reports, mutation-summary]
# Not one of the gate's report jobs: a failure here fails the workflow conclusion instead.
if: >-
always() && needs.select.result == 'success' && needs.select.outputs.hasChanges == 'true' &&
(needs.reports.result == 'success' || needs.reports.result == 'failure') &&
needs.mutation-summary.result != 'cancelled'
env:
QUALITY_HEAD: ${{ needs.request.outputs.head || github.event.pull_request.head.sha || github.sha }}
QUALITY_BASE: ${{ needs.request.outputs.base || github.event.pull_request.base.sha || '' }}
QUALITY_REPOSITORY: ${{ github.repository }}
# 0 for a workflow_dispatch run with no pull request to report against.
QUALITY_PR: ${{ github.event.pull_request.number || github.event.issue.number || 0 }}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
# Reads report data only: no dependency install, no test run, read-only token.
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false
sparse-checkout: scripts/quality
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
# One exact-name `pattern` per artifact: a pattern that matches nothing succeeds with no
# download (an unselected or failed workspace, which the bundle marks unavailable), and a
# single match is extracted straight into `path`, so each lands in its own directory.
- name: Download change-risk report (data-schemas)
uses: actions/download-artifact@v6
with:
pattern: quality-internal-risk-data-schemas-${{ env.QUALITY_HEAD }}
path: coverage/internal/quality-internal-risk-data-schemas-${{ env.QUALITY_HEAD }}
- name: Download change-risk report (api)
uses: actions/download-artifact@v6
with:
pattern: quality-internal-risk-api-${{ env.QUALITY_HEAD }}
path: coverage/internal/quality-internal-risk-api-${{ env.QUALITY_HEAD }}
- name: Download change-risk report (server)
uses: actions/download-artifact@v6
with:
pattern: quality-internal-risk-server-${{ env.QUALITY_HEAD }}
path: coverage/internal/quality-internal-risk-server-${{ env.QUALITY_HEAD }}
- name: Download mutation report (data-schemas)
uses: actions/download-artifact@v6
with:
pattern: quality-internal-mutation-data-schemas-${{ env.QUALITY_HEAD }}
path: coverage/internal/quality-internal-mutation-data-schemas-${{ env.QUALITY_HEAD }}
- name: Bundle the quality report
env:
QUALITY_REPORTS: ${{ needs.select.outputs.reports }}
QUALITY_REFACTOR_HINTS: ${{ vars.QUALITY_REFACTOR_HINTS }}
run: >-
node scripts/quality/bundle.mts --directory coverage/internal
--output coverage/quality-report
--repository "$QUALITY_REPOSITORY"
--pr "$QUALITY_PR"
--base "$QUALITY_BASE"
--head "$QUALITY_HEAD"
# A same-named artifact uploaded by a PR-code job must not stand in for these: overwrite
# replaces it, and the comment writer requires these exact steps to have succeeded.
- name: Upload quality report
timeout-minutes: 3
uses: actions/upload-artifact@v6
with:
name: quality-report
path: coverage/quality-report/
if-no-files-found: error
retention-days: 14
overwrite: true
# Separate and small: the privileged comment writer downloads only this file.
- name: Upload report findings
timeout-minutes: 3
uses: actions/upload-artifact@v6
with:
name: quality-internal-findings
path: coverage/quality-report/findings.json
if-no-files-found: error
retention-days: 1
overwrite: true
progress:
name: Update quality progress
concurrency:
group: quality-decision-${{ github.event.issue.number }}
cancel-in-progress: false
queue: max
needs: [request, select, verify]
if: always() && needs.request.outputs.accepted == 'true'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
actions: write
pull-requests: write
statuses: write
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false
sparse-checkout: scripts/quality
- uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- env:
GH_TOKEN: ${{ github.token }}
run: node scripts/quality/comment.mts progress