🧭 ci: Assert Lighthouse Blocking Time at a Reference CPU on the Quietest Run (#18406)

Total blocking time is CPU time, and the hosted runner's CPU is not a constant: Lighthouse's
own benchmark of the runner moves by more than 2x between consecutive runs of one build on a
two-core runner, and blocking time moves with it. A median of three raw measurements against
a fixed budget therefore fails on the runner rather than on the page.

Rescale each run's main-thread tasks by that run's benchmark index to a reference CPU, assert
the lowest run instead of the median, and measure an over-budget batch once more before
failing. Budgets move to their own file so a fork with a heavier client changes numbers only.
This commit is contained in:
Danny Avila 2026-11-18 07:47:01 -05:00 • committed by GitHub
parent ff522e0771
commit 445668a371
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 315 additions and 71 deletions

View file

@ -4,9 +4,93 @@ import { tmpdir } from 'node:os';
import { inspect } from 'node:util';
import assert from 'node:assert/strict';
import { mkdtemp, writeFile, readFile, readdir, rm } from 'node:fs/promises';
import type Result from 'lighthouse/types/lhr/lhr.js';
import { runLighthouse } from './lighthouse/process.mts';
import type { Task } from './lighthouse/measure.mts';
import { discardLighthouseTraces, sanitizeLighthouseArtifacts } from './lighthouse/artifacts.mts';
import { blockingTime, measure, REFERENCE_BENCHMARK_INDEX } from './lighthouse/measure.mts';
import { runLighthouse } from './lighthouse/process.mts';
const blockingBudget = { 'total-blocking-time': 500 };
/** The page under test: two tasks after first paint, 300 ms of blocking at the reference CPU. */
const page: Task[] = [
{ startTime: 400, duration: 600 },
{ startTime: 1500, duration: 250 },
{ startTime: 2500, duration: 150 },
{ startTime: 3500, duration: 30 },
];
/** A run of `tasks` on a runner `slowdown` times slower than the reference. */
function run(tasks: Task[], slowdown = 1, lcp = 3000): Result {
return {
environment: { benchmarkIndex: REFERENCE_BENCHMARK_INDEX / slowdown },
audits: {
'first-contentful-paint': { numericValue: 1000 * slowdown },
interactive: { numericValue: 9000 * slowdown },
'largest-contentful-paint': { numericValue: lcp },
'main-thread-tasks': {
details: {
type: 'table',
items: tasks.map(({ startTime, duration }) => ({
startTime: startTime * slowdown,
duration: duration * slowdown,
})),
},
},
},
} as unknown as Result;
}
const slower = (tasks: Task[], factor: number): Task[] =>
tasks.map(({ startTime, duration }) => ({ startTime, duration: duration * factor }));
test('blocking time at a scale of 1 counts each task tail inside the paint window', () => {
const tasks = [
{ startTime: 0, duration: 200 },
{ startTime: 900, duration: 300 },
{ startTime: 1500, duration: 150 },
{ startTime: 2000, duration: 40 },
{ startTime: 2900, duration: 300 },
];
assert.equal(blockingTime(tasks, { start: 1000, end: 3000 }), 200 + 100 + 50);
});
test('a runner at any speed measures the same blocking time for the same page', () => {
const [reference] = measure([run(page)], blockingBudget);
assert.equal(reference.value, 300);
for (const slowdown of [0.6, 1.5, 3.6]) {
const [measured] = measure([run(page, slowdown)], blockingBudget);
assert.ok(Math.abs(measured.value - 300) < 1e-6, `${slowdown}x measured ${measured.value}`);
}
});
test('a faster runner counts the tasks that would block at the reference CPU', () => {
const [measured] = measure([run([{ startTime: 2000, duration: 80 }], 0.5)], blockingBudget);
assert.ok(Math.abs(measured.value - 30) < 1e-6, `measured ${measured.value}`);
});
test('one busy run cannot fail the blocking budget, and every run over it does', () => {
const [quiet] = measure([run(slower(page, 3)), run(page), run(slower(page, 2))], blockingBudget);
assert.equal(quiet.value, 300);
assert.equal(quiet.statistic, 'lowest, at reference CPU');
for (const slowdown of [1, 2.5]) {
const regressed = Array.from({ length: 3 }, () => run(slower(page, 2), slowdown));
const [measured] = measure(regressed, blockingBudget);
assert.ok(measured.value > measured.limit, `${slowdown}x measured ${measured.value}`);
}
});
test('other budgets keep the median, and a report without CPU evidence fails loudly', () => {
const runs = [run(page, 1, 5200), run(page, 1, 3100), run(page, 1, 4400)];
assert.deepEqual(measure(runs, { 'largest-contentful-paint': 4500 }), [
{ audit: 'largest-contentful-paint', statistic: 'median', value: 4400, limit: 4500 },
]);
assert.throws(() => measure(runs, { 'speed-index': 1 }), /numeric speed-index for every run/);
const unmeasured = run(page);
delete unmeasured.audits['main-thread-tasks'];
assert.throws(() => measure([unmeasured], blockingBudget), /no main-thread tasks/);
});
test('Lighthouse retains measured trace timings without publishing network logs or cookies', async (t) => {
const directory = await mkdtemp(path.join(tmpdir(), 'lighthouse-artifacts-'));

View file

@ -30,16 +30,33 @@ The runner makes three cold browser navigations to a populated conversation,
using the production client build and real authentication, config, file and message
routes. It uses desktop settings with `throttlingMethod: provided` so Lighthouse
does not replace the measured server delays with simulated network timing.
Median budgets are LCP **4,500 ms**, CLS **0.1**, and TBT **500 ms**, asserted against
the median of the three runs. These are lab regression budgets, not field web-vitals
percentiles; Lighthouse does not measure INP.
Budgets are LCP **4,500 ms**, CLS **0.1**, and TBT **500 ms** (`budgets.mts`). These are
lab regression budgets, not field web-vitals percentiles; Lighthouse does not measure INP.
LCP and CLS are asserted against the median of the runs. LCP is mostly the injected
database delay, so it barely moves with the runner.
TBT is CPU time, and a hosted runner's CPU is not a constant: three consecutive runs of
one build have measured 613, 191 and 159 ms on a two-core runner. The gate therefore
asserts blocking time **at a reference CPU, on the quietest run**:
- Each run's main-thread tasks are rescaled by Lighthouse's own CPU benchmark of that run
(`environment.benchmarkIndex` against `REFERENCE_BENCHMARK_INDEX` in `measure.mts`) and
blocking time is summed again, so a runner at half speed measures the same page.
- The lowest run is asserted, not the median. Contention only adds blocking time, so the
quietest run is the closest to what the page itself costs.
- A batch that is over any budget is measured once more, and the verdict uses all six runs.
A regression is over budget in every run; a busy stretch on a shared runner is not.
The runner prints both the measured and the reference-CPU blocking time for every run.
The test also requires the seeded transcript to be the LCP element, so a fast
login page, spinner, or empty shell cannot pass.
## When the gate fails
1. Read the failed audit's measured median and limit in the budget table the runner
prints immediately before asserting.
1. Read the failed audit's value and limit in the budget table the runner prints
immediately before asserting, and the per-run table above it. A run whose
`benchmarkIndex` is far below the others was measured on a busy runner; a blocking
time over budget at the reference CPU in all six runs is the page, not the runner.
2. Open a `.lighthouse/lhr-*.report.html` report. The console also prints API request
start/end times. A late request start suggests a browser dependency; a long
request suggests server work or serial database reads.
@ -65,16 +82,18 @@ before returning data. Do not raise a threshold to hide added round trips.
## Adding another scenario
`auditPage({ url, cookies, runs, budgets })` in `audit.ts` invokes the Lighthouse CLI
once per run, redacts authentication headers, prints API timings and asserts the median
once per run, redacts authentication headers, prints API timings and asserts the
budgets. It returns the reports so each scenario can assert its expected final URL and
LCP content; read the LCP element through the exported `lcpElement(report)` helper rather
than an audit id, because Lighthouse renames those between majors. `load.spec.ts` owns
local login state, conversation seeding and transcript assertions; the runner does not
depend on them.
A downstream fork can import this runner from a separate launch spec, supply
cookies from its own authentication fixture and pass its own `budgets` for
launch thresholds. The database delay hook can be preloaded by that fixture's server
A downstream fork that ships a different client keeps its own numbers in `budgets.mts`
and leaves the runner alone; calibrate them from the per-run tables of the fork's own
runs, since a heavier client carries more blocking time on every runner. A fork can also
import this runner from a separate launch spec, supply cookies from its own
authentication fixture and pass its own `budgets` for launch thresholds. The database delay hook can be preloaded by that fixture's server
configuration. Keep provider-specific authentication and provisioning in that spec.
For launch flows that navigate across documents, also measure entry-to-ready time
with Playwright: LCP resets on a new document, so final-page LCP alone does not

View file

@ -1,24 +1,26 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { expect } from '@playwright/test';
import type Result from 'lighthouse/types/lhr/lhr';
import type { Cookie } from '@playwright/test';
import { runLighthouse } from './process.mts';
import type { Measurement } from './measure.mts';
import type { Budgets } from './budgets.mts';
import { measure, referenceBlockingTime } from './measure.mts';
import { sanitizeLighthouseArtifacts } from './artifacts.mts';
/** Median budgets keyed by Lighthouse audit id. */
export type MedianBudgets = Record<string, number>;
import { runLighthouse } from './process.mts';
import { BUDGETS } from './budgets.mts';
const REPORTS_DIRECTORY = '.lighthouse';
const DEFAULT_RUNS = 3;
const DEFAULT_BUDGETS: MedianBudgets = {
'largest-contentful-paint': 4500,
'cumulative-layout-shift': 0.1,
'total-blocking-time': 500,
};
/**
* An over-budget batch is measured once more before it fails. A regression is over budget in
* every run; a busy stretch on a shared runner is not, and must not fail the gate alone.
*/
const BATCHES = 2;
/**
* Lighthouse 13 removed `largest-contentful-paint-element` and moved the LCP node into the
@ -33,28 +35,46 @@ export function lcpElement(report: Result): string {
return JSON.stringify(details);
}
function medianOf(results: Result[], audit: string): number {
const values = results
.map((report) => report.audits[audit]?.numericValue)
.filter((value): value is number => typeof value === 'number')
.sort((a, b) => a - b);
expect(values, `Lighthouse must report a numeric ${audit} for every run`).toHaveLength(
results.length,
);
const middle = values.length >> 1;
return values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2;
function readReports(directory: string): Result[] {
return fs
.readdirSync(directory)
.filter((file) => /^lhr-.*\.report\.json$/.test(file))
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
.map((file) => JSON.parse(fs.readFileSync(path.join(directory, file), 'utf8')) as Result);
}
function describeRuns(reports: Result[]): void {
for (const report of reports) {
console.log(
`${report.finalDisplayedUrl}: LCP ${report.audits['largest-contentful-paint'].displayValue}`,
);
const requests = report.audits['network-requests'].details;
if (requests?.type === 'table') {
console.table(
requests.items
.filter((item) => typeof item.url === 'string' && item.url.includes('/api/'))
.map((item) => ({
url: item.url,
startMs: item.networkRequestTime,
endMs: item.networkEndTime,
status: item.statusCode,
})),
);
}
expect(report.runtimeError, 'Lighthouse navigation must succeed').toBeUndefined();
}
}
export async function auditPage({
url,
cookies,
runs = DEFAULT_RUNS,
budgets = DEFAULT_BUDGETS,
budgets = BUDGETS,
}: {
url: string;
cookies: Cookie[];
runs?: number;
budgets?: MedianBudgets;
budgets?: Budgets;
}): Promise<Result[]> {
const cookie = cookies.map(({ name, value }) => `${name}=${value}`).join('; ');
const directory = path.resolve(REPORTS_DIRECTORY);
@ -76,51 +96,46 @@ export async function auditPage({
'--save-assets',
];
try {
for (let run = 1; run <= runs; run++) {
const output = path.join(directory, `lhr-${run}`);
await runLighthouse({ cli, flags, output, run, runs });
let results: Result[] = [];
let measured: Measurement[] = [];
for (let batch = 1; batch <= BATCHES; batch++) {
const total = runs * batch;
try {
for (let run = results.length + 1; run <= total; run++) {
const output = path.join(directory, `lhr-${run}`);
await runLighthouse({ cli, flags, output, run, runs: total });
}
} finally {
sanitizeLighthouseArtifacts(directory, cookies);
}
} finally {
sanitizeLighthouseArtifacts(directory, cookies);
}
const reports = fs.readdirSync(directory).filter((file) => /^lhr-.*\.report\.json$/.test(file));
expect(reports.length, 'Lighthouse must produce one report per run').toBe(runs);
const results = reports.map(
(file) => JSON.parse(fs.readFileSync(path.join(directory, file), 'utf8')) as Result,
);
for (const report of results) {
console.log(
`${report.finalDisplayedUrl}: LCP ${report.audits['largest-contentful-paint'].displayValue}`,
);
const requests = report.audits['network-requests'].details;
if (requests?.type === 'table') {
console.table(
requests.items
.filter((item) => typeof item.url === 'string' && item.url.includes('/api/'))
.map((item) => ({
url: item.url,
startMs: item.networkRequestTime,
endMs: item.networkEndTime,
status: item.statusCode,
})),
);
}
expect(report.runtimeError, 'Lighthouse navigation must succeed').toBeUndefined();
const reports = readReports(directory);
expect(reports.length, 'Lighthouse must produce one report per run').toBe(total);
describeRuns(reports.slice(results.length));
results = reports;
measured = measure(results, budgets);
if (batch === BATCHES || measured.every(({ value, limit }) => value <= limit)) break;
console.log(`Over budget after ${total} runs; measuring ${runs} more before failing.`);
}
console.log(
`Inspect ${REPORTS_DIRECTORY} HTML/JSON and e2e/lighthouse/README.md. Reuse loaded user/config data; overlap independent reads without bypassing authorization.`,
);
const measured = Object.entries(budgets).map(([audit, limit]) => ({
audit,
median: medianOf(results, audit),
limit,
}));
// A low benchmarkIndex is a slow or busy runner: read the measured times against it.
console.log(`Runner: ${os.availableParallelism()} logical CPUs`);
console.table(
results.map((report, index) => ({
run: index + 1,
benchmarkIndex: Math.round(report.environment.benchmarkIndex),
'largest-contentful-paint': Math.round(
report.audits['largest-contentful-paint'].numericValue ?? NaN,
),
'total-blocking-time': Math.round(report.audits['total-blocking-time'].numericValue ?? NaN),
'at reference CPU': Math.round(referenceBlockingTime(report)),
})),
);
console.table(measured);
for (const { audit, median, limit } of measured) {
expect(median, `Median ${audit} must stay within ${limit}`).toBeLessThanOrEqual(limit);
for (const { audit, statistic, value, limit } of measured) {
expect(value, `${audit} (${statistic}) must stay within ${limit}`).toBeLessThanOrEqual(limit);
}
return results;
}

View file

@ -0,0 +1,12 @@
/** Budgets keyed by Lighthouse audit id. */
export type Budgets = Record<string, number>;
/**
* The numbers live apart from the runner so a fork that ships a heavier client changes this
* file alone. `total-blocking-time` is asserted at the reference CPU: see `measure.mts`.
*/
export const BUDGETS: Budgets = {
'largest-contentful-paint': 4500,
'cumulative-layout-shift': 0.1,
'total-blocking-time': 500,
};

View file

@ -75,7 +75,9 @@ test('serial database latency stays within web-vitals budgets', async ({ page, b
}
throw error;
}
expect(reports, 'Lighthouse must produce all three reports').toHaveLength(3);
expect(reports.length, 'Lighthouse must produce at least three reports').toBeGreaterThanOrEqual(
3,
);
for (const report of reports) {
expect(report.finalDisplayedUrl, 'Do not measure a login redirect').toBe(url);
expect(

111
e2e/lighthouse/measure.mts Normal file
View file

@ -0,0 +1,111 @@
import type Result from 'lighthouse/types/lhr/lhr.js';
import type { Budgets } from './budgets.mts';
/**
* Lighthouse's CPU benchmark score on the hosted runners the budgets are calibrated against.
* Blocking time is restated at this speed: one build has measured 613, 191 and 159 ms of it in
* three consecutive runs, and the benchmark moves with whatever else the runner is doing.
*/
export const REFERENCE_BENCHMARK_INDEX = 2750;
/** Only the part of a main-thread task that runs past this blocks input. */
const BLOCKING_THRESHOLD_MS = 50;
const BLOCKING_AUDIT = 'total-blocking-time';
export interface Task {
startTime: number;
duration: number;
}
/** First contentful paint to time to interactive, in ms after navigation start. */
export interface Window {
start: number;
end: number;
}
export interface Measurement {
audit: string;
statistic: string;
value: number;
limit: number;
}
/**
* Total blocking time with every task `cpuScale` times as long. At a scale of 1 this is
* Lighthouse's own sum: each task's tail past the threshold, clipped to the window.
*/
export function blockingTime(tasks: Task[], window: Window, cpuScale = 1): number {
let total = 0;
for (const { startTime, duration } of tasks) {
const blocking = duration * cpuScale - BLOCKING_THRESHOLD_MS;
if (blocking <= 0) continue;
// A task too short to block on this runner has no tail, so its whole span stands in.
const tail = duration > BLOCKING_THRESHOLD_MS ? duration - BLOCKING_THRESHOLD_MS : duration;
const end = startTime + duration;
const counted = Math.min(end, window.end) - Math.max(end - tail, window.start);
if (counted > 0) total += blocking * (counted / tail);
}
return total;
}
/** One run's blocking time as the reference CPU would have measured it. */
export function referenceBlockingTime(report: Result): number {
const tasks = report.audits['main-thread-tasks']?.details;
const start = report.audits['first-contentful-paint']?.numericValue;
const end = report.audits['interactive']?.numericValue;
const benchmarkIndex = report.environment?.benchmarkIndex;
if (
tasks?.type !== 'table' ||
typeof start !== 'number' ||
typeof end !== 'number' ||
typeof benchmarkIndex !== 'number' ||
benchmarkIndex <= 0
) {
throw new Error(
'Lighthouse reported no main-thread tasks, paint window or CPU benchmark; check `main-thread-tasks`, `first-contentful-paint`, `interactive` and `environment.benchmarkIndex`.',
);
}
return blockingTime(
tasks.items.map((item) => ({
startTime: Number(item.startTime),
duration: Number(item.duration),
})),
{ start, end },
benchmarkIndex / REFERENCE_BENCHMARK_INDEX,
);
}
function median(results: Result[], audit: string): number {
const values = results
.map((report) => report.audits[audit]?.numericValue)
.filter((value): value is number => typeof value === 'number')
.sort((a, b) => a - b);
if (!values.length || values.length !== results.length) {
throw new Error(`Lighthouse must report a numeric ${audit} for every run`);
}
const middle = values.length >> 1;
return values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2;
}
/**
* Blocking time takes the lowest run rather than the median: a busy runner only ever adds
* blocking time, so the quietest run is the closest to what the page itself costs.
*/
export function measure(results: Result[], budgets: Budgets): Measurement[] {
return Object.entries(budgets).map(([audit, limit]) => {
if (audit !== BLOCKING_AUDIT) {
return { audit, statistic: 'median', value: median(results, audit), limit };
}
if (!results.length) {
throw new Error(`Lighthouse must report a numeric ${audit} for every run`);
}
return {
audit,
statistic: 'lowest, at reference CPU',
value: Math.min(...results.map(referenceBlockingTime)),
limit,
};
});
}

View file

@ -15,7 +15,8 @@ export default defineConfig({
...mockConfig,
testDir: 'lighthouse',
outputDir: 'lighthouse/.test-results',
timeout: 300_000,
// Covers the confirming batch an over-budget audit measures before it fails.
timeout: 600_000,
retries: 0,
reporter: [['line']],
webServer: servers.map((server) => ({