LibreChat/.github/workflows/playwright-mock.yml
Marco Beretta e36598f019
⏩ ci: Record Playwright Shard Timings Automatically and Run Fifteen Shards (#18115)
* ci: Record Playwright Shard Timings From Green Dev Runs
* ci: Run the Playwright Mock Suite on Fifteen Memory Shards
* ci: Run the Playwright Workflow Script Tests Once per Run
* test: Make the Due Schedule E2E Wait for One Engine Tick Only
* test: Cover the Shard Timings Hand-off With Acceptance Scenarios
2026-11-14 01:25:48 +01:00

822 lines
46 KiB
YAML

name: Playwright E2E Tests
on:
pull_request:
paths:
- '**'
- '!**.md'
- '!.github/workflows/**'
- '.github/workflows/playwright-mock.yml'
# Post-merge safety net, as backend-review.yml and frontend-review.yml have for jest. A
# regression that reaches dev fails this suite on every open pull request, and without a run
# on dev itself nothing names the merge that caused it. A push runs the pull request profile
# with no lane narrowed, so a red run on dev is what the next pull request would see. The
# nightly stays on the default branch, where it covers the full suite on both stores.
push:
branches:
- dev
paths:
- '**'
- '!**.md'
- '!.github/workflows/**'
- '.github/workflows/playwright-mock.yml'
# The merge queue tests the commit about to land on dev. It has no path filter and runs
# the pull request profile with no lane narrowed, so a failure in the queue is one the
# pull request's own checks can reproduce.
merge_group:
types: [checks_requested]
schedule:
- cron: '0 5 * * *'
workflow_dispatch:
inputs:
reason:
description: 'Reason for manual trigger'
required: false
default: 'Manual e2e run'
coverage:
description: 'Coverage profile: full suite on both stores, or the PR shards'
type: choice
options:
- full
- pr
default: full
permissions:
contents: read
concurrency:
# Pull requests, the nightly and manual runs keep a per-ref group in which a newer run cancels
# the older one. Pushes to dev take a group of their own that never cancels, so the run in
# progress always finishes; GitHub holds one pending run per group and a newer one replaces
# it, so merges that land meanwhile collapse into a single run of the newest commit. A
# cancelling group would rarely finish on a busy day (nine merges landed within 89 seconds on
# 2026-11-09) and a run per commit, which the jest workflows use, costs about 200
# runner-minutes on 20 runners each time. A red run therefore names a range of merges, from
# the commit of the last finished run to its own, and the dev tip always gets a run.
group: playwright-mock-${{ github.event_name == 'push' && 'post-merge-' || '' }}${{ github.ref }}
cancel-in-progress: ${{ github.event_name != 'push' }}
env:
NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}'
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
jobs:
# Stage 2 of codegraph gating (stage 1 = backend jest in backend-review.yml). The graph decides
# two matrix lanes whose relevance is a reachability question: the redis-transport shards (the
# stream-boundary specs under a real Redis round-trip) and the MCP list_changed lanes. Memory
# shards always run. Monotone and fail-open: a lane is dropped ONLY on an explicit `false` from
# the service; unavailable/unconfigured/non-synchronize events (dev pushes among them) keep the
# full matrix. Lock attribution rides along so a backend dependency bump (reaches nothing in
# the graph) still earns both lanes. Kill switch: repo variable CODEGRAPH_GATING=off.
codegraph_select:
name: Codegraph select
runs-on: ubuntu-latest
timeout-minutes: 5
if: >-
github.event_name == 'pull_request' &&
github.event.action == 'synchronize' &&
vars.CODEGRAPH_GATING != 'off' &&
github.event.pull_request != null &&
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)
outputs:
decided: ${{ steps.sel.outputs.decided }}
e2e_include: ${{ steps.sel.outputs.e2e_include }}
mcp_run: ${{ steps.sel.outputs.mcp_run }}
e2e_skip: ${{ steps.sel.outputs.e2e_skip }}
steps:
- name: Select matrix lanes, fail open on any doubt
id: sel
env:
URL: ${{ secrets.CODEGRAPH_URL }}
TOKEN: ${{ secrets.CODEGRAPH_TOKEN }}
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR: ${{ github.event.pull_request.number }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
CHANGED: ${{ github.event.pull_request.changed_files }}
E2E_SKIP_ARMED: ${{ vars.CODEGRAPH_E2E_SKIP }}
FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/15","stream_store":"memory","redis_image":"","suite":"full","shard":"1/15","artifact":"memory-1-of-15","email_change":"true"},{"name":"memory, shard 2/15","stream_store":"memory","redis_image":"","suite":"full","shard":"2/15","artifact":"memory-2-of-15"},{"name":"memory, shard 3/15","stream_store":"memory","redis_image":"","suite":"full","shard":"3/15","artifact":"memory-3-of-15"},{"name":"memory, shard 4/15","stream_store":"memory","redis_image":"","suite":"full","shard":"4/15","artifact":"memory-4-of-15"},{"name":"memory, shard 5/15","stream_store":"memory","redis_image":"","suite":"full","shard":"5/15","artifact":"memory-5-of-15"},{"name":"memory, shard 6/15","stream_store":"memory","redis_image":"","suite":"full","shard":"6/15","artifact":"memory-6-of-15"},{"name":"memory, shard 7/15","stream_store":"memory","redis_image":"","suite":"full","shard":"7/15","artifact":"memory-7-of-15"},{"name":"memory, shard 8/15","stream_store":"memory","redis_image":"","suite":"full","shard":"8/15","artifact":"memory-8-of-15"},{"name":"memory, shard 9/15","stream_store":"memory","redis_image":"","suite":"full","shard":"9/15","artifact":"memory-9-of-15"},{"name":"memory, shard 10/15","stream_store":"memory","redis_image":"","suite":"full","shard":"10/15","artifact":"memory-10-of-15"},{"name":"memory, shard 11/15","stream_store":"memory","redis_image":"","suite":"full","shard":"11/15","artifact":"memory-11-of-15"},{"name":"memory, shard 12/15","stream_store":"memory","redis_image":"","suite":"full","shard":"12/15","artifact":"memory-12-of-15"},{"name":"memory, shard 13/15","stream_store":"memory","redis_image":"","suite":"full","shard":"13/15","artifact":"memory-13-of-15"},{"name":"memory, shard 14/15","stream_store":"memory","redis_image":"","suite":"full","shard":"14/15","artifact":"memory-14-of-15"},{"name":"memory, shard 15/15","stream_store":"memory","redis_image":"","suite":"full","shard":"15/15","artifact":"memory-15-of-15"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}'
run: |
set +e
note() { echo "$1" >> "$GITHUB_STEP_SUMMARY"; }
note "### Codegraph select — GATING (Playwright matrix lanes)"
if [ -z "$URL" ] || [ -z "$TOKEN" ]; then note "_no codegraph config; running FULL_"; exit 0; fi
# A failed or truncated page must not become a shorter file list: the pipeline would hide
# gh's exit status behind jq, and a partial list can turn a required lane off. Check the
# fetch status AND the count against the PR's own changed_files (Codex P1, #15136).
if ! gh api "repos/$REPO/pulls/$PR/files" --paginate \
--jq '.[] | {path: .filename, status, patch}' > files.ndjson; then
note "_could not fetch changed files; running FULL_"; exit 0
fi
jq -s . files.ndjson > files.json
N=$(jq 'length' files.json)
if [ "$N" -eq 0 ] || { [ -n "$CHANGED" ] && [ "$N" -ne "$CHANGED" ]; }; then
note "_changed-file list incomplete ($N of ${CHANGED:-?}); running FULL_"; exit 0
fi
jq -c --arg b "$BASE_SHA" --arg h "$HEAD_SHA" \
'{files: ., mode: "safe", lockBaseSha: $b, lockHeadSha: $h}' files.json > body.json
# curl's status is checked explicitly: a transfer that times out or truncates after a
# parseable body must fail open, not be honoured (Codex P1, #15136). --fail-with-body
# also turns HTTP errors into a failure while keeping the error text for the summary.
RESP=$(curl -sS --fail-with-body -m 45 -H "Authorization: Bearer $TOKEN" \
-H 'content-type: application/json' --data-binary @body.json "$URL/v1/select"); RC=$?
if [ "$RC" -ne 0 ] || [ -z "$RESP" ] || ! echo "$RESP" | jq -e '.matrix["playwright-mock"]' >/dev/null 2>&1; then
note "_codegraph unavailable (curl exit $RC: ${RESP:0:120}); running FULL_"
exit 0
fi
# A lane is skipped only on the JSON boolean false — tested inside jq, because `jq -r`
# prints the string "false" and the boolean identically (Codex P1, #15136). Anything
# else (true, null, a string, missing) runs.
if echo "$RESP" | jq -e '.e2e.fail_open == true' >/dev/null 2>&1; then
note "_fail-open decision (root/workflow/lockfile change or stale graph): everything runs_"
fi
REDIS=$(echo "$RESP" | jq -c '.matrix["playwright-mock"].redis_transport')
MCP=$(echo "$RESP" | jq -c '.matrix["playwright-mock"].mcp_tool_list_changed')
REDIS_SKIP=0; MCP_SKIP=0
echo "$RESP" | jq -e '.matrix["playwright-mock"].redis_transport == false' >/dev/null 2>&1 && REDIS_SKIP=1
echo "$RESP" | jq -e '.matrix["playwright-mock"].mcp_tool_list_changed == false' >/dev/null 2>&1 && MCP_SKIP=1
note "| lane | decision |"
note "|---|---|"
if [ "$REDIS_SKIP" = 1 ]; then
INCLUDE=$(echo "$FULL_INCLUDE" | jq -c '.include |= map(select(.suite != "transport"))')
note "| redis transport | skip (no reach into the stream boundary) |"
else
INCLUDE="$FULL_INCLUDE"
note "| redis transport | run |"
fi
if ! echo "$INCLUDE" | jq -e '.include | length >= 15' >/dev/null 2>&1; then
note "_matrix assembly failed; running FULL_"
exit 0
fi
echo "e2e_include=$INCLUDE" >> "$GITHUB_OUTPUT"
# Graduated per-spec skips are DARK until the operator arms repo variable
# CODEGRAPH_E2E_SKIP=on (the election switch — flipped only when the pre-registered
# resume condition holds). Even then, act only on a well-typed list from a non-fail-open
# decision: every entry must be a pool spec path, or nothing is skipped. The server
# already intersects with this PR's skippable tier and applies the streak bars
# (2x where history-coupled); see codegraph-poc service/graduate.ts.
SKIP=""
if [ "$E2E_SKIP_ARMED" = "on" ]; then
if echo "$RESP" | jq -e '(.e2e.fail_open != true) and (.e2e.graduated | type == "array" and all(.[]?; type == "string" and test("^e2e/specs/mock/[A-Za-z0-9._/-]+\\.spec\\.ts$") and (contains("..") | not)))' >/dev/null 2>&1; then
SKIP=$(echo "$RESP" | jq -r '[.e2e.graduated[] | sub("^e2e/"; "")] | join(" ")')
else
note "_graduated list absent or malformed; no specs skipped_"
fi
fi
echo "e2e_skip=$SKIP" >> "$GITHUB_OUTPUT"
if [ -n "$SKIP" ]; then
note "| graduated spec skips | $(echo "$SKIP" | wc -w | tr -d ' ') (armed) |"
echo "codegraph-e2e-graduated-skips: $SKIP"
fi
echo "codegraph-select: redis_transport=$REDIS mcp_tool_list_changed=$MCP matrix_entries=$(echo "$INCLUDE" | jq '.include | length')"
if [ "$MCP_SKIP" = 1 ]; then
echo "mcp_run=false" >> "$GITHUB_OUTPUT"
note "| MCP list_changed | skip (no reach into MCP) |"
else
echo "mcp_run=true" >> "$GITHUB_OUTPUT"
note "| MCP list_changed | run |"
fi
echo "decided=true" >> "$GITHUB_OUTPUT"
note ""
note "memory shards always run · kill switch: repo variable \`CODEGRAPH_GATING=off\` · full matrix on PR open, on every dev push and nightly"
exit 0
# Every shard has to plan from the same timings, or the partition stops being disjoint and a
# spec runs twice or not at all. The newest recorded copy is therefore resolved once, here,
# and handed to the shards as a job output: a shard that restored the cache itself could see
# a newer entry than its siblings, and a re-run shard a newer one than the attempt it joins.
# playwright-timings.yml writes the cache from green runs on dev, and a pull request reads
# dev's entries as its base branch. With no entry or a malformed one the output is empty and
# every shard plans from the committed e2e/timings.mock.json, so nothing in this job fails on
# the data; like every job here it still answers to the gate.
shard_timings:
name: Shard timings
if: >-
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'push' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' &&
github.event.pull_request != null &&
(contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) ||
github.event.pull_request.head.repo.full_name == github.repository))
runs-on: ubuntu-latest
timeout-minutes: 3
outputs:
timings: ${{ steps.read.outputs.timings }}
steps:
- name: Restore the newest recorded timings
id: restore
continue-on-error: true
uses: actions/cache/restore@v5
with:
path: e2e/.generated/shard-timings.json
key: e2e-shard-timings-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: e2e-shard-timings-
- name: Hand one copy to every shard
id: read
run: |
FILE=e2e/.generated/shard-timings.json
if [ ! -s "$FILE" ]; then
echo "no recorded timings; shards plan from e2e/timings.mock.json"
exit 0
fi
if ! TIMINGS=$(jq -ce '{files: (.files | with_entries(select(.value | type == "number")))} | select(.files | length > 0)' "$FILE"); then
echo "::warning::recorded shard timings are malformed; shards plan from e2e/timings.mock.json"
exit 0
fi
echo "timings=$TIMINGS" >> "$GITHUB_OUTPUT"
echo "recorded timings for $(jq '.files | length' <<< "$TIMINGS") spec files"
e2e_shards:
name: e2e (${{ matrix.name }})
needs: [codegraph_select, shard_timings]
if: >-
!cancelled() &&
(github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'push' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' &&
github.event.pull_request != null &&
(contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) ||
github.event.pull_request.head.repo.full_name == github.repository)))
runs-on: ubuntu-latest
timeout-minutes: 30
# Separate runners isolate each shard's user, MongoDB, Redis and fixture state.
# Keep workers: 1 in Playwright; shared-user cleanup is not worker-safe.
strategy:
fail-fast: false
matrix: >-
${{
(github.event_name == 'pull_request' && needs.codegraph_select.outputs.e2e_include != '' &&
fromJSON(needs.codegraph_select.outputs.e2e_include)) ||
((github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'merge_group' || inputs.coverage == 'pr') &&
fromJSON('{"include":[{"name":"memory, shard 1/15","stream_store":"memory","redis_image":"","suite":"full","shard":"1/15","artifact":"memory-1-of-15","email_change":"true"},{"name":"memory, shard 2/15","stream_store":"memory","redis_image":"","suite":"full","shard":"2/15","artifact":"memory-2-of-15"},{"name":"memory, shard 3/15","stream_store":"memory","redis_image":"","suite":"full","shard":"3/15","artifact":"memory-3-of-15"},{"name":"memory, shard 4/15","stream_store":"memory","redis_image":"","suite":"full","shard":"4/15","artifact":"memory-4-of-15"},{"name":"memory, shard 5/15","stream_store":"memory","redis_image":"","suite":"full","shard":"5/15","artifact":"memory-5-of-15"},{"name":"memory, shard 6/15","stream_store":"memory","redis_image":"","suite":"full","shard":"6/15","artifact":"memory-6-of-15"},{"name":"memory, shard 7/15","stream_store":"memory","redis_image":"","suite":"full","shard":"7/15","artifact":"memory-7-of-15"},{"name":"memory, shard 8/15","stream_store":"memory","redis_image":"","suite":"full","shard":"8/15","artifact":"memory-8-of-15"},{"name":"memory, shard 9/15","stream_store":"memory","redis_image":"","suite":"full","shard":"9/15","artifact":"memory-9-of-15"},{"name":"memory, shard 10/15","stream_store":"memory","redis_image":"","suite":"full","shard":"10/15","artifact":"memory-10-of-15"},{"name":"memory, shard 11/15","stream_store":"memory","redis_image":"","suite":"full","shard":"11/15","artifact":"memory-11-of-15"},{"name":"memory, shard 12/15","stream_store":"memory","redis_image":"","suite":"full","shard":"12/15","artifact":"memory-12-of-15"},{"name":"memory, shard 13/15","stream_store":"memory","redis_image":"","suite":"full","shard":"13/15","artifact":"memory-13-of-15"},{"name":"memory, shard 14/15","stream_store":"memory","redis_image":"","suite":"full","shard":"14/15","artifact":"memory-14-of-15"},{"name":"memory, shard 15/15","stream_store":"memory","redis_image":"","suite":"full","shard":"15/15","artifact":"memory-15-of-15"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}')) ||
fromJSON('{"include":[{"name":"memory, shard 1/15","stream_store":"memory","redis_image":"","suite":"full","shard":"1/15","artifact":"memory-1-of-15","email_change":"true"},{"name":"memory, shard 2/15","stream_store":"memory","redis_image":"","suite":"full","shard":"2/15","artifact":"memory-2-of-15"},{"name":"memory, shard 3/15","stream_store":"memory","redis_image":"","suite":"full","shard":"3/15","artifact":"memory-3-of-15"},{"name":"memory, shard 4/15","stream_store":"memory","redis_image":"","suite":"full","shard":"4/15","artifact":"memory-4-of-15"},{"name":"memory, shard 5/15","stream_store":"memory","redis_image":"","suite":"full","shard":"5/15","artifact":"memory-5-of-15"},{"name":"memory, shard 6/15","stream_store":"memory","redis_image":"","suite":"full","shard":"6/15","artifact":"memory-6-of-15"},{"name":"memory, shard 7/15","stream_store":"memory","redis_image":"","suite":"full","shard":"7/15","artifact":"memory-7-of-15"},{"name":"memory, shard 8/15","stream_store":"memory","redis_image":"","suite":"full","shard":"8/15","artifact":"memory-8-of-15"},{"name":"memory, shard 9/15","stream_store":"memory","redis_image":"","suite":"full","shard":"9/15","artifact":"memory-9-of-15"},{"name":"memory, shard 10/15","stream_store":"memory","redis_image":"","suite":"full","shard":"10/15","artifact":"memory-10-of-15"},{"name":"memory, shard 11/15","stream_store":"memory","redis_image":"","suite":"full","shard":"11/15","artifact":"memory-11-of-15"},{"name":"memory, shard 12/15","stream_store":"memory","redis_image":"","suite":"full","shard":"12/15","artifact":"memory-12-of-15"},{"name":"memory, shard 13/15","stream_store":"memory","redis_image":"","suite":"full","shard":"13/15","artifact":"memory-13-of-15"},{"name":"memory, shard 14/15","stream_store":"memory","redis_image":"","suite":"full","shard":"14/15","artifact":"memory-14-of-15"},{"name":"memory, shard 15/15","stream_store":"memory","redis_image":"","suite":"full","shard":"15/15","artifact":"memory-15-of-15"},{"name":"redis, shard 1/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/10","artifact":"redis-1-of-10"},{"name":"redis, shard 2/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/10","artifact":"redis-2-of-10"},{"name":"redis, shard 3/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"3/10","artifact":"redis-3-of-10"},{"name":"redis, shard 4/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"4/10","artifact":"redis-4-of-10"},{"name":"redis, shard 5/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"5/10","artifact":"redis-5-of-10"},{"name":"redis, shard 6/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"6/10","artifact":"redis-6-of-10"},{"name":"redis, shard 7/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"7/10","artifact":"redis-7-of-10"},{"name":"redis, shard 8/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"8/10","artifact":"redis-8-of-10"},{"name":"redis, shard 9/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"9/10","artifact":"redis-9-of-10"},{"name":"redis, shard 10/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"10/10","artifact":"redis-10-of-10"}]}')
}}
services:
redis:
image: ${{ matrix.redis_image }}
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 3s
--health-retries 10
env:
E2E_CHROMIUM_CHANNEL: chrome
E2E_STREAM_STORE: ${{ matrix.stream_store }}
REDIS_URI: redis://127.0.0.1:6379
steps:
- uses: actions/checkout@v5
- name: Use Node.js 24.22.3
uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- name: Restore node_modules cache
id: cache-node-modules
uses: actions/cache@v5
with:
path: |
node_modules
client/node_modules
packages/client/node_modules
packages/chat/node_modules
packages/data-provider/node_modules
packages/data-schemas/node_modules
packages/api/node_modules
packages/channel-gateway/node_modules
api/node_modules
key: node-modules-e2e-${{ runner.os }}-24.22.3-${{ hashFiles('package-lock.json') }}
- name: Install dependencies
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci
- name: Restore data-provider build cache
id: cache-data-provider
uses: actions/cache@v5
with:
path: packages/data-provider/dist
key: build-data-provider-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
- name: Build data-provider
if: steps.cache-data-provider.outputs.cache-hit != 'true'
run: npm run build:data-provider
- name: Restore data-schemas build cache
id: cache-data-schemas
uses: actions/cache@v5
with:
path: packages/data-schemas/dist
key: build-data-schemas-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
- name: Build data-schemas
if: steps.cache-data-schemas.outputs.cache-hit != 'true'
run: npm run build:data-schemas
- name: Restore api build cache
id: cache-api
uses: actions/cache@v5
with:
path: packages/api/dist
key: build-api-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'packages/api/src/**', 'packages/api/tsconfig*.json', 'packages/api/tsdown.config.mjs', 'packages/api/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json') }}
- name: Build api
if: steps.cache-api.outputs.cache-hit != 'true'
run: npm run build:api
- name: Restore client-package build cache
id: cache-client-package
uses: actions/cache@v5
with:
path: packages/client/dist
key: build-client-package-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
- name: Build client-package
if: steps.cache-client-package.outputs.cache-hit != 'true'
run: npm run build:client-package
# About a second, so it is not cached; the chat-package scenarios load the built entries.
- name: Build chat package
run: npm run build:chat
- name: Restore client app build cache
id: cache-client-app
uses: actions/cache@v5
with:
path: client/dist
key: build-client-app-e2e-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'client/src/**', 'client/public/**', 'client/index.html', 'client/package.json', 'client/vite.config.*', 'client/tsconfig*.json', 'client/tailwind.config.*', 'client/postcss.config.*', 'packages/client/src/**', 'packages/chat/src/**', 'packages/client/tailwind.preset.cjs', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
- name: Build client app
if: steps.cache-client-app.outputs.cache-hit != 'true'
run: npm run build:client
- name: Provision MongoDB memory-server binary
uses: ./.github/actions/mongodb-binary
- name: Verify Chrome is present
run: google-chrome --version
# `video: 'on-first-retry'` needs ffmpeg; without it the first retry dies in
# browserContext.newPage before the test body runs, so a flaky test loses the
# retry that would have recovered it.
#
# This step used to burn its full 90s bound on every job. Playwright's bundled
# extractor hangs on Node 24.16.0 (a yauzl/extract-zip regression fixed in
# Playwright 1.60.0): the 2.3MB download finished in under a second, then
# extraction stalled and the timeout reaped it, leaving a truncated binary and
# no INSTALLATION_COMPLETE marker — so ffmpeg was never actually installed and
# retries never got video. With Playwright bumped past the fix the install
# takes about a second, and a restored cache skips it outright.
#
# The cache is only saved once the binary is verified to run, so a partial
# extraction can never be promoted into a cache every later job restores.
# Kept non-fatal: retry video is a debugging aid, not something CI asserts on.
- name: Resolve Playwright version
id: playwright-version
run: |
version=$(node -p "require('./package-lock.json').packages['node_modules/playwright-core'].version")
echo "version=${version}" >> "$GITHUB_OUTPUT"
- name: Restore Playwright ffmpeg cache
id: cache-ffmpeg
uses: actions/cache/restore@v5
with:
path: ~/.cache/ms-playwright
key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
- name: Install Playwright ffmpeg (best effort)
id: install-ffmpeg
if: steps.cache-ffmpeg.outputs.cache-hit != 'true'
timeout-minutes: 3
continue-on-error: true
run: |
timeout -k 10 60 npx playwright install ffmpeg
.github/scripts/verify-playwright-ffmpeg.sh
- name: Save Playwright ffmpeg cache
if: steps.install-ffmpeg.outcome == 'success'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ~/.cache/ms-playwright
key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
# The runner's Chrome is an apt package, so its real library dependencies are
# already satisfied; all `install-deps` adds here are optional CJK/Thai/Cyrillic
# font packages (~21MB from azure.archive.ubuntu.com). Nothing in CI asserts on
# them — visual baselines are opt-in via E2E_VISUAL_SNAPSHOTS — so a stalled
# Ubuntu mirror must never be able to fail the suite.
- name: Install optional Playwright font dependencies (best effort)
timeout-minutes: 4
continue-on-error: true
run: .github/scripts/install-playwright-fonts.sh
- name: Run full mock-LLM Tier-1 e2e
if: matrix.suite == 'full'
env:
CI: 'true'
E2E_SKIP: ${{ needs.codegraph_select.outputs.e2e_skip }}
# Seconds of work memory shard 1 does after its suite (both registered-email profiles
# and the title ownership regressions), so the plan hands it that much less of the
# suite. Every lane of one store plans with the same value, so they agree.
E2E_SHARD_RESERVE: ${{ matrix.stream_store == 'memory' && '1:130' || '' }}
# The run's one copy of the recorded timings; empty when `shard_timings` found none.
E2E_SHARD_TIMINGS: ${{ needs.shard_timings.outputs.timings }}
# Shards are planned by recorded duration, not test count: Playwright lists the suite,
# and .github/scripts/playwright-shards.mjs assigns whole spec files longest-first to
# the least-loaded shard, using the timings `shard_timings` resolved for this run or,
# without them, e2e/timings.mock.json. Every shard derives the same disjoint partition
# of the listed files, so together they run exactly the suite.
#
# Graduated-spec skipping (dark until repo var CODEGRAPH_E2E_SKIP=on upstream) subtracts
# the earned skips from that listing before planning, so an unknown or stale name simply
# matches nothing; if subtraction would drop everything, nothing is skipped. Skipped specs
# still execute post-merge in every full-suite vote run, which is the net that catches a
# wrong skip.
run: |
TIMINGS=e2e/timings.mock.json
if [ -n "$E2E_SHARD_TIMINGS" ]; then
mkdir -p e2e/.generated
TIMINGS=e2e/.generated/shard-timings.json
printf '%s' "$E2E_SHARD_TIMINGS" > "$TIMINGS"
fi
PLAYWRIGHT_JSON_OUTPUT_FILE=e2e/.generated/playwright-list.json \
npx playwright test --config=e2e/playwright.config.mock.ts --list --reporter=json > /dev/null
node .github/scripts/playwright-shards.mjs plan \
--list e2e/.generated/playwright-list.json \
--timings "$TIMINGS" \
--shard '${{ matrix.shard }}' \
--reserve "$E2E_SHARD_RESERVE" \
--skip "$E2E_SKIP" \
--out e2e/.generated/shard-tests.txt
npx playwright test --config=e2e/playwright.config.mock.ts --test-list=e2e/.generated/shard-tests.txt
- name: Run Redis stream transport e2e
if: matrix.suite == 'transport'
run: npx playwright test --config=e2e/playwright.config.redis.ts --shard=${{ matrix.shard }}
env:
CI: 'true'
- name: Run registered-email change e2e
if: matrix.email_change == 'true'
run: npx playwright test --config=e2e/playwright.config.email.ts
env:
CI: 'true'
- name: Run disabled registered-email change e2e
if: matrix.email_change == 'true'
run: npx playwright test --config=e2e/playwright.config.email.ts
env:
CI: 'true'
E2E_ALLOW_EMAIL_CHANGE: 'false'
# Reports and traces are debugging aids that no job downloads, so an upload that fails must
# not fail a shard whose tests passed. On 2026-11-11 the artifact service answered every
# request with an HTML page: each upload retried for 9 to 10.5 minutes before it failed,
# six shards whose tests had passed went red, and one of them ended within seconds of the
# job's 30 minute timeout. `continue-on-error` keeps a failed upload out of the job result,
# and the bound keeps a stalled one from running the job into that timeout: the slowest of
# 519 successful report uploads took 10 seconds. The runner still annotates the run with
# the upload's own error, or with the step timeout.
- name: Upload Playwright HTML report
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-report-${{ matrix.artifact }}
path: e2e/playwright-report/**
retention-days: 7
if-no-files-found: ignore
- name: Upload traces & screenshots
if: failure()
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-test-results-${{ matrix.artifact }}
path: e2e/specs/.test-results/**
retention-days: 7
if-no-files-found: ignore
- name: Run first-turn title ownership regressions
if: ${{ !cancelled() && matrix.stream_store == 'memory' && startsWith(matrix.shard, '1/') }}
env:
CI: 'true'
E2E_TITLE_CONVO: 'true'
run: >-
npx playwright test --config=e2e/playwright.config.mock.ts
conversation-management.spec.ts --grep 'first-turn|final-title polling'
--reporter=line,e2e/reporters/retried.mjs
--output=e2e/specs/.test-results/title-ownership
- name: Upload title ownership diagnostics
if: ${{ always() && matrix.stream_store == 'memory' && startsWith(matrix.shard, '1/') }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-title-ownership-${{ matrix.artifact }}
path: e2e/specs/.test-results/title-ownership/**
retention-days: 7
if-no-files-found: ignore
mcp_tool_list_changed:
name: MCP list_changed (replica count ${{ matrix.replicas }})
needs: [codegraph_select]
if: >-
!cancelled() &&
needs.codegraph_select.outputs.mcp_run != 'false' &&
(github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'push' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' &&
github.event.pull_request != null &&
(contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) ||
github.event.pull_request.head.repo.full_name == github.repository)))
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
replicas: [1, 2]
env:
CI: 'true'
E2E_CHROMIUM_CHANNEL: chrome
E2E_MCP_LIST_CHANGED: 'true'
E2E_REPLICAS: ${{ matrix.replicas }}
steps:
- uses: actions/checkout@v4
- name: Use Node.js 24.22.3
uses: actions/setup-node@v4
with:
node-version: '24.22.3'
- name: Restore node_modules cache
id: cache-node-modules
uses: actions/cache@v4
with:
path: |
node_modules
client/node_modules
packages/client/node_modules
packages/chat/node_modules
packages/data-provider/node_modules
packages/data-schemas/node_modules
packages/api/node_modules
packages/channel-gateway/node_modules
api/node_modules
key: node-modules-e2e-${{ runner.os }}-24.22.3-${{ hashFiles('package-lock.json') }}
- name: Install dependencies
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci
- name: Build e2e dependencies
run: npm run e2e:prepare
- name: Provision MongoDB memory-server binary
uses: ./.github/actions/mongodb-binary
- name: Verify Chrome is present
run: google-chrome --version
# ffmpeg for retry video — see the note in the e2e_shards job.
- name: Resolve Playwright version
id: playwright-version
run: |
version=$(node -p "require('./package-lock.json').packages['node_modules/playwright-core'].version")
echo "version=${version}" >> "$GITHUB_OUTPUT"
- name: Restore Playwright ffmpeg cache
id: cache-ffmpeg
uses: actions/cache/restore@v5
with:
path: ~/.cache/ms-playwright
key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
- name: Install Playwright ffmpeg (best effort)
id: install-ffmpeg
if: steps.cache-ffmpeg.outputs.cache-hit != 'true'
timeout-minutes: 3
continue-on-error: true
run: |
timeout -k 10 60 npx playwright install ffmpeg
.github/scripts/verify-playwright-ffmpeg.sh
- name: Save Playwright ffmpeg cache
if: steps.install-ffmpeg.outcome == 'success'
continue-on-error: true
uses: actions/cache/save@v5
with:
path: ~/.cache/ms-playwright
key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
# This job deliberately skips the optional font install: the MCP suite
# does not enable visual snapshot assertions, so nothing here needs apt.
# Redis is a hard requirement for this job, so this step stays fatal. The
# action puts the pinned redis-server and redis-cli on PATH from the
# Actions cache, or builds them from the pinned commit on a miss; no
# distro mirror has to resolve for this job to get Redis.
- name: Install Redis runtime dependencies
timeout-minutes: 8
uses: ./.github/actions/redis-binaries
# A daemonized redis-server returns before it opens its port, so the
# first ping waits for it instead of racing it.
- name: Start standalone Redis and Redis Cluster
run: |
redis-server --daemonize yes --port 6379
timeout 30 bash -c 'until redis-cli -p 6379 ping; do sleep 0.5; done'
chmod +x redis-config/start-cluster.sh redis-config/stop-cluster.sh
./redis-config/start-cluster.sh
redis-cli -p 7001 cluster info
- name: Test MCP notifications with in-memory cache
env:
E2E_STREAM_STORE: memory
run: >-
npx playwright test --config=e2e/playwright.config.mock.ts
mcp-tool-list-changed.spec.ts --retries=0
- name: Test MCP notifications with standalone Redis cache
env:
E2E_STREAM_STORE: redis
REDIS_URI: redis://127.0.0.1:6379
run: >-
npx playwright test --config=e2e/playwright.config.mock.ts
mcp-tool-list-changed.spec.ts --retries=0
- name: Test MCP notifications with Redis Cluster cache
env:
E2E_STREAM_STORE: redis-cluster
REDIS_URI: redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003
run: >-
npx playwright test --config=e2e/playwright.config.mock.ts
mcp-tool-list-changed.spec.ts --retries=0
# Debugging aids that no job downloads: non-fatal and bounded, as the note on the
# e2e_shards uploads explains.
- name: Upload Playwright HTML report
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: playwright-mcp-list-changed-${{ matrix.replicas }}-replicas
path: e2e/playwright-report/**
retention-days: 7
if-no-files-found: ignore
- name: Upload traces & screenshots
if: failure()
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: playwright-mcp-list-changed-results-${{ matrix.replicas }}-replicas
path: e2e/specs/.test-results/**
retention-days: 7
if-no-files-found: ignore
- name: Stop Redis processes
if: always()
run: |
./redis-config/stop-cluster.sh || true
redis-cli -p 6379 shutdown || true
mcp_apps:
name: MCP Apps (browser and server)
# Always run the dedicated fixture on eligible PRs, dev pushes, schedule, and manual runs. The
# regular mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane.
if: >-
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'push' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' &&
github.event.pull_request != null &&
(contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) ||
github.event.pull_request.head.repo.full_name == github.repository))
runs-on: ubuntu-latest
timeout-minutes: 45
env:
CI: 'true'
E2E_CHROMIUM_CHANNEL: chrome
steps:
- uses: actions/checkout@v5
- name: Use Node.js 24.22.3
uses: actions/setup-node@v5
with:
node-version: '24.22.3'
- name: Restore node_modules cache
id: cache-node-modules
uses: actions/cache@v5
with:
path: |
node_modules
client/node_modules
packages/client/node_modules
packages/chat/node_modules
packages/data-provider/node_modules
packages/data-schemas/node_modules
packages/api/node_modules
packages/channel-gateway/node_modules
api/node_modules
key: node-modules-e2e-${{ runner.os }}-24.22.3-${{ hashFiles('package-lock.json') }}
- name: Install dependencies
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci
- name: Provision MongoDB memory-server binary
uses: ./.github/actions/mongodb-binary
- name: Verify Chrome is present
run: google-chrome --version
# These check this workflow's own wiring and scripts, so once per run is enough. They ran in
# every shard for about 8 seconds each; this lane finishes minutes before the shards do.
- name: Test Playwright matrix selection, the vote gate, the merge gate and the retried-pass reporter
run: node --test .github/scripts/playwright-matrix.test.mjs .github/scripts/playwright-shards.test.mjs .github/scripts/vote-gate.test.mjs .github/scripts/merge-gate.test.mjs e2e/reporters/retried.test.mjs
- name: Check MCP Apps result gating and same-repository PR eligibility
run: node --test e2e/mcp-apps/results.test.mjs e2e/mcp-apps/workflow.test.mjs
- name: Check real SDK server fixture
run: node --test e2e/mcp-apps/server.test.mjs
# The runner builds once, then runs all four browser policy phases against one disposable
# MongoDB. It also rejects a Playwright-successful phase if every test was skipped.
- name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota)
run: node e2e/mcp-apps/run.mjs
# Debugging aids that no job downloads: non-fatal and bounded, as the note on the
# e2e_shards uploads explains.
- name: Upload per-phase Playwright reports
if: ${{ !cancelled() }}
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-mcp-apps-reports
path: |
e2e/playwright-report/mcp-apps-*/**
e2e/.generated/mcp-apps-result-*.json
retention-days: 7
if-no-files-found: warn
- name: Upload MCP Apps traces and screenshots
if: failure()
timeout-minutes: 3
continue-on-error: true
uses: actions/upload-artifact@v6
with:
name: playwright-mcp-apps-results
path: e2e/specs/.test-results/mcp-apps-*/**
retention-days: 7
if-no-files-found: ignore
e2e:
name: e2e
# `!cancelled()`, not `always()`: the gate has to survive a failed shard to report it, but a
# run superseded by `cancel-in-progress` has nothing to adjudicate. Under `always()` it was
# still dispatched onto a fresh runner while every other job went `cancelled`, then read
# `needs.e2e_shards.result == 'cancelled'` and exited 1 — so each superseded push left this
# required check red instead of cancelled. Matches all three lanes it gates.
if: >-
!cancelled() &&
(github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'push' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' &&
github.event.pull_request != null &&
(contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) ||
github.event.pull_request.head.repo.full_name == github.repository)))
needs: [codegraph_select, shard_timings, e2e_shards, mcp_tool_list_changed, mcp_apps]
runs-on: ubuntu-latest
steps:
# A codegraph-skipped MCP lane reports `skipped`; that is a decision, not a failure.
- name: Verify every Playwright job passed
if: >-
needs.shard_timings.result != 'success' ||
needs.e2e_shards.result != 'success' ||
needs.mcp_apps.result != 'success' ||
(needs.mcp_tool_list_changed.result != 'success' &&
!(needs.mcp_tool_list_changed.result == 'skipped' && needs.codegraph_select.outputs.mcp_run == 'false'))
run: exit 1