mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-18 14:08:21 +00:00
* 🧱 ci: Block the PWA Service Worker Across the Mock E2E Profiles Every mock test opens a fresh browser context, so the app's service worker installs and claims the page during each test's first load. A `page.route` installed within a few milliseconds of that claim is never consulted: the request it should hold goes to the server, and a test waiting on the route times out. 113 mock spec files install routes and seven blocked the worker themselves. Set `serviceWorkers: 'block'` in the mock profile's shared `use`, which every profile derived from it inherits. No spec asserts worker behavior; one that does can opt back in with `test.use({ serviceWorkers: 'allow' })`. * 🧪 test: Keep the Sandpack Preview Worker in the Plain Text Artifact Spec `serviceWorkers: 'block'` stops every service worker in the browser context, not only the app's. Sandpack's static template serves an artifact's files from a service worker inside its preview iframe, so with the profile blocking workers the preview stays empty and the spec that reads it fails every attempt. Opt that one spec back in and say in the profile's comment that the setting reaches third-party workers too.
708 lines
29 KiB
TypeScript
708 lines
29 KiB
TypeScript
import { defineConfig, devices } from '@playwright/test';
|
|
import { execFileSync } from 'child_process';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
import { getLocalE2EEnv, getE2EBaseURL } from './setup/env';
|
|
import { managementAuth } from './setup/agent-management';
|
|
|
|
const rootPath = path.resolve(__dirname, '..');
|
|
const replicaCount = Number(process.env.E2E_REPLICAS || '1');
|
|
if (replicaCount !== 1 && replicaCount !== 2) {
|
|
throw new Error(`E2E_REPLICAS must be 1 or 2, received ${process.env.E2E_REPLICAS}`);
|
|
}
|
|
const serverPath = path.resolve(
|
|
rootPath,
|
|
replicaCount === 2 ? 'e2e/setup/start-server-cluster.js' : 'e2e/setup/start-server.js',
|
|
);
|
|
const mcpHttpServerPath = path.resolve(rootPath, 'e2e/setup/fake-mcp-http-server.js');
|
|
const mcpOAuthServerPath = path.resolve(rootPath, 'e2e/setup/fake-mcp-oauth-server.js');
|
|
const dynamicMcpServerPath = path.resolve(rootPath, 'e2e/setup/fake-mcp-dynamic-network-server.js');
|
|
const mcpAppServerPath = path.resolve(rootPath, 'e2e/setup/fake-mcp-app-server.mjs');
|
|
/** Must match the `e2e-http` server URL in e2e/config/librechat.e2e.yaml. */
|
|
const MCP_HTTP_PORT = process.env.E2E_MCP_HTTP_PORT || '8765';
|
|
/** Must match the protected OAuth MCP fixture in e2e/config/librechat.e2e.yaml. */
|
|
const MCP_OAUTH_PORT = process.env.E2E_MCP_OAUTH_PORT || '8767';
|
|
/** Must match the dynamic Streamable HTTP and SSE URLs in the e2e config template. */
|
|
const MCP_DYNAMIC_PORT = process.env.E2E_MCP_DYNAMIC_PORT || '8766';
|
|
/** Used only by the dedicated MCP Apps browser profile. */
|
|
const MCP_APP_PORT = process.env.E2E_MCP_APP_PORT || '8768';
|
|
const MCP_STATE_PATH =
|
|
process.env.E2E_MCP_STATE_PATH ||
|
|
path.resolve(rootPath, 'e2e/specs/.test-results/mcp-tool-state.json');
|
|
const labelServerPath = path.resolve(rootPath, 'e2e/setup/fake-label-server.js');
|
|
/** The template's custom-endpoint `baseURL`s hard-code 8889;
|
|
* `writeRuntimeMockConfig` substitutes any override into the generated copy. */
|
|
const LABEL_PORT = process.env.E2E_LABEL_PORT || '8889';
|
|
const codeServerPath = path.resolve(rootPath, 'e2e/setup/fake-code-server.js');
|
|
const ragServerPath = path.resolve(rootPath, 'e2e/setup/fake-rag-server.js');
|
|
/** Ports the backend reaches via LIBRECHAT_CODE_BASEURL / RAG_API_URL below.
|
|
* Kept clear of the MCP (8765/8766) and label (8889) fixtures. */
|
|
const CODE_API_PORT = process.env.E2E_CODE_API_PORT || '8790';
|
|
const RAG_API_PORT = process.env.E2E_RAG_API_PORT || '8791';
|
|
/** The fake GitHub Enterprise host pull request lookups reach (`pullRequests.host`). The template
|
|
* hard-codes 8793; `writeRuntimeMockConfig` substitutes any override into the generated copy. */
|
|
const gitHostServerPath = path.resolve(rootPath, 'e2e/setup/fake-git-host-server.js');
|
|
const GIT_HOST_PORT = process.env.E2E_GIT_HOST_PORT || '8793';
|
|
const gitHostKeyPath = path.resolve(rootPath, 'e2e/.generated/git-host.key');
|
|
const gitHostCertPath = path.resolve(rootPath, 'e2e/.generated/git-host.crt');
|
|
const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js');
|
|
/** Model-fixture record mode: the run hook taps the REAL provider stream into a
|
|
* replayable fixture instead of overriding the model (e2e/setup/model-replay.js). */
|
|
const modelFixtureRecording = process.env.E2E_MODEL_FIXTURES === 'record';
|
|
const recordModelHookPath = path.resolve(rootPath, 'e2e/setup/record-model.js');
|
|
const recordProviderBaseURL =
|
|
process.env.E2E_RECORD_PROVIDER_BASE_URL || 'https://api.deepseek.com/v1';
|
|
const recordProviderModel = process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat';
|
|
if (modelFixtureRecording && !process.env.E2E_RECORD_PROVIDER_API_KEY) {
|
|
throw new Error('E2E_MODEL_FIXTURES=record requires E2E_RECORD_PROVIDER_API_KEY');
|
|
}
|
|
/**
|
|
* Each fixture belongs to exactly one spec, and a spec records only its own.
|
|
* Accepting an arbitrary name would leave a second fixture beside the
|
|
* committed one carrying the same prompts, and the server-side ambiguity
|
|
* check would then refuse to bind either — a successful recording run would
|
|
* disable the keyless lane.
|
|
*/
|
|
const RECORDABLE_FIXTURES = ['deepseek-two-turn', 'deepseek-tool-call'];
|
|
if (modelFixtureRecording && !process.env.E2E_MODEL_FIXTURE_NAME) {
|
|
throw new Error('E2E_MODEL_FIXTURES=record requires E2E_MODEL_FIXTURE_NAME');
|
|
}
|
|
if (
|
|
modelFixtureRecording &&
|
|
!RECORDABLE_FIXTURES.includes(process.env.E2E_MODEL_FIXTURE_NAME ?? '')
|
|
) {
|
|
throw new Error(
|
|
`E2E_MODEL_FIXTURE_NAME must be one of ${RECORDABLE_FIXTURES.join(', ')}; ` +
|
|
`received ${process.env.E2E_MODEL_FIXTURE_NAME}`,
|
|
);
|
|
}
|
|
/**
|
|
* Playwright documents `-c` as an alias for `--config`, so both spellings are
|
|
* parsed — recognising only the long form would let the short one slip past.
|
|
*
|
|
* Derived configs (`playwright.config.redis.ts`, `.mermaid.ts`) spread this
|
|
* config and then replace `testMatch`, discarding the record-mode restriction
|
|
* below — their specs would reach the paid provider and rewrite the selected
|
|
* fixture. The restriction cannot be enforced through a value a consumer can
|
|
* overwrite, so record mode refuses any config but this one.
|
|
*/
|
|
if (modelFixtureRecording) {
|
|
/** Only the process that parsed the CLI carries `--config`; Playwright
|
|
* workers do not, and must not be judged on an argument they never saw. */
|
|
const configFlagIndex = process.argv.findIndex(
|
|
(arg) =>
|
|
arg === '--config' || arg === '-c' || arg.startsWith('--config=') || arg.startsWith('-c='),
|
|
);
|
|
const configFlag = configFlagIndex === -1 ? undefined : process.argv[configFlagIndex];
|
|
let configPath: string | undefined;
|
|
if (configFlag?.includes('=')) {
|
|
configPath = configFlag.slice(configFlag.indexOf('=') + 1);
|
|
} else if (configFlag) {
|
|
configPath = process.argv[configFlagIndex + 1];
|
|
}
|
|
if (configPath && !/playwright\.config\.mock\.ts$/.test(configPath)) {
|
|
throw new Error(
|
|
`E2E_MODEL_FIXTURES=record only runs under playwright.config.mock.ts, not ${configPath}; ` +
|
|
'derived configs replace testMatch and would send their specs to the real provider',
|
|
);
|
|
}
|
|
}
|
|
const assistantsServerPath = path.resolve(rootPath, 'e2e/setup/fake-assistants-server.js');
|
|
const ASSISTANTS_PORT = process.env.E2E_ASSISTANTS_PORT || '8890';
|
|
const fluxServerPath = path.resolve(rootPath, 'e2e/setup/fake-flux-server.js');
|
|
const FLUX_PORT = process.env.E2E_FLUX_PORT || '8794';
|
|
const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml');
|
|
const configPath = path.resolve(rootPath, 'e2e/.generated/librechat.e2e.yaml');
|
|
const reportPath = path.resolve(rootPath, 'e2e/playwright-report');
|
|
const deploymentSkillsPath = path.resolve(rootPath, 'e2e/fixtures/deployment-skills');
|
|
const enableDynamicMcp = process.env.E2E_MCP_LIST_CHANGED === 'true';
|
|
const enableMcpApps = process.env.E2E_MCP_APPS === 'true';
|
|
const mcpAppsPolicy = process.env.E2E_MCP_APPS_POLICY;
|
|
if (mcpAppsPolicy && !['true', 'false', 'omitted'].includes(mcpAppsPolicy)) {
|
|
throw new Error(`E2E_MCP_APPS_POLICY must be true, false, or omitted; got ${mcpAppsPolicy}`);
|
|
}
|
|
|
|
function positiveIntegerEnv(name: string): number | undefined {
|
|
const raw = process.env[name];
|
|
if (!raw) {
|
|
return undefined;
|
|
}
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value <= 0) {
|
|
throw new Error(`${name} must be a positive integer; got ${raw}`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
const mcpAppResourceLimit = positiveIntegerEnv('E2E_MCP_APP_RESOURCE_LIMIT');
|
|
const mcpAppToolCallLimit = positiveIntegerEnv('E2E_MCP_APP_TOOL_CALL_LIMIT');
|
|
if ((mcpAppResourceLimit == null) !== (mcpAppToolCallLimit == null)) {
|
|
throw new Error('MCP App resource and tool-call fixture limits must be supplied together');
|
|
}
|
|
|
|
const baseURL = getE2EBaseURL();
|
|
const baseUrlObject = new URL(baseURL);
|
|
const mcpSandboxHost = baseUrlObject.hostname === 'localhost' ? '127.0.0.1' : 'localhost';
|
|
const defaultMcpSandboxUrl = new URL('/api/mcp/sandbox', baseUrlObject);
|
|
defaultMcpSandboxUrl.hostname = mcpSandboxHost;
|
|
const mcpSandboxUrl = process.env.E2E_MCP_SANDBOX_URL ?? defaultMcpSandboxUrl.href;
|
|
process.env.E2E_MCP_SANDBOX_URL = mcpSandboxUrl;
|
|
const chromiumChannel = process.env.E2E_CHROMIUM_CHANNEL || undefined;
|
|
|
|
/** WebAuthn rejects an IP address as the relying party, so passkey ceremonies reach
|
|
* the same server through its `localhost` name. */
|
|
const passkeyOrigin = (() => {
|
|
const url = new URL(baseURL);
|
|
url.hostname = 'localhost';
|
|
return url.origin;
|
|
})();
|
|
|
|
const vanillaOverrides = {
|
|
TENANT_ISOLATION_STRICT: 'false',
|
|
TRUST_TENANT_HEADER: 'true',
|
|
ALLOW_PASSKEY_LOGIN: 'true',
|
|
/** The password-reset scenario needs the reset routes; CI has no developer `.env`. */
|
|
ALLOW_PASSWORD_RESET: 'true',
|
|
PASSKEY_RP_ID: 'localhost',
|
|
PASSKEY_ORIGINS: passkeyOrigin,
|
|
/** Every project reruns the reset and passkey sign-in scenarios from one IP, and each
|
|
* login page load spends a passkey request on autofill; the defaults allow 2 and 20. */
|
|
RESET_PASSWORD_MAX: '100',
|
|
PASSKEY_MAX: '500',
|
|
OPENAI_API_KEY: 'user_provided',
|
|
OPENID_CLIENT_ID: '',
|
|
OPENID_ISSUER: '',
|
|
OPENID_AUTO_REDIRECT: 'false',
|
|
ALLOW_SOCIAL_LOGIN: 'false',
|
|
ALLOW_SOCIAL_REGISTRATION: 'false',
|
|
ALLOW_SHARED_LINKS_PUBLIC: 'true',
|
|
ENFORCE_TWO_FACTOR_AUTHENTICATION: 'false',
|
|
STREAM_KEEP_COMPLETED_JOBS: 'true',
|
|
FORK_IP_MAX: '100',
|
|
FORK_USER_MAX: '100',
|
|
/** A local `.env` may enable balance enforcement, which `neutralizeCredentialEnv`
|
|
* does not blank (not credential-shaped); the fresh e2e user has no balance
|
|
* record, so every streaming spec would be refused with a token_balance
|
|
* violation before the mock model runs. */
|
|
CHECK_BALANCE: 'false',
|
|
};
|
|
|
|
const externalCodeBaseUrl = process.env.E2E_CODE_BASEURL?.trim();
|
|
const codeApiKeyEnv: Record<string, string> = {};
|
|
if (!externalCodeBaseUrl) {
|
|
codeApiKeyEnv.LIBRECHAT_CODE_API_KEY = 'e2e-code-key';
|
|
} else if (process.env.E2E_CODE_API_KEY) {
|
|
codeApiKeyEnv.LIBRECHAT_CODE_API_KEY = process.env.E2E_CODE_API_KEY;
|
|
}
|
|
|
|
const baseEnv = {
|
|
...getLocalE2EEnv(),
|
|
CONFIG_PATH: configPath,
|
|
DEPLOYMENT_SKILLS_DIR: deploymentSkillsPath,
|
|
/** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model —
|
|
* or, in model-fixture record mode, to tap the real provider stream. */
|
|
LIBRECHAT_TEST_RUN_HOOK: modelFixtureRecording ? recordModelHookPath : fakeModelHookPath,
|
|
...(modelFixtureRecording
|
|
? {
|
|
E2E_MODEL_FIXTURE_NAME: process.env.E2E_MODEL_FIXTURE_NAME ?? '',
|
|
E2E_RECORD_PROVIDER_API_KEY: process.env.E2E_RECORD_PROVIDER_API_KEY ?? '',
|
|
}
|
|
: {}),
|
|
...(enableDynamicMcp ? { E2E_MCP_LIST_CHANGED: 'true', E2E_MCP_STATE_PATH: MCP_STATE_PATH } : {}),
|
|
...(enableMcpApps
|
|
? {
|
|
E2E_MCP_APPS: 'true',
|
|
MCP_SANDBOX_FRAME_ANCESTORS:
|
|
process.env.MCP_SANDBOX_FRAME_ANCESTORS ?? 'http://127.0.0.1:3080',
|
|
}
|
|
: {}),
|
|
/** The Assistants runtime uses the OpenAI SDK directly, outside the agents run hook. */
|
|
ASSISTANTS_API_KEY: 'e2e-mock-assistants-key',
|
|
ASSISTANTS_BASE_URL: `http://127.0.0.1:${ASSISTANTS_PORT}/v1`,
|
|
ASSISTANTS_MODELS: 'gpt-4o-mini',
|
|
/** The Flux tool talks to the fake BFL endpoint started below. */
|
|
FLUX_API_KEY: 'e2e-flux-key',
|
|
FLUX_API_BASE_URL: `http://127.0.0.1:${FLUX_PORT}`,
|
|
/** Enables the native Google provider for agents. The run hook replaces the model, so the
|
|
* key never reaches Google; it only lets attachments take the Google encoding path. */
|
|
GOOGLE_KEY: 'e2e-mock-google-key',
|
|
...(process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN
|
|
? { E2E_CODE_BRIDGE_ADMIN_TOKEN: process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN }
|
|
: {}),
|
|
/** Point code-env + RAG provisioning at the local fakes started below. */
|
|
LIBRECHAT_CODE_BASEURL: externalCodeBaseUrl || `http://127.0.0.1:${CODE_API_PORT}/v1`,
|
|
...(externalCodeBaseUrl ? { LIBRECHAT_CODE_BASEURL_STATEFUL: externalCodeBaseUrl } : {}),
|
|
...codeApiKeyEnv,
|
|
RAG_API_URL: `http://127.0.0.1:${RAG_API_PORT}`,
|
|
/** Pull request lookups go to the fake git host, which trusts any bearer token. */
|
|
E2E_GIT_HOST_TOKEN: 'e2e-git-host-token',
|
|
/** Resolved into the `e2e-memory` MCP server's env reference. */
|
|
E2E_MCP_DEPLOYMENT: 'e2e-deployment',
|
|
NODE_EXTRA_CA_CERTS: gitHostCertPath,
|
|
...vanillaOverrides,
|
|
};
|
|
|
|
const SECRET_KEY_PATTERN = /(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIALS|CLIENT_ID|_KEY)$/i;
|
|
const preservedCredentialEnvKeys = new Set([
|
|
...Object.keys(baseEnv),
|
|
'E2E_USER_PASSWORD',
|
|
'E2E_USER_B_PASSWORD',
|
|
]);
|
|
|
|
/**
|
|
* The custom endpoints in the template point their `baseURL` at the local fake
|
|
* label server; the fake model injected via `LIBRECHAT_TEST_RUN_HOOK` overrides
|
|
* the GRAPH before any request is made, so no real provider is contacted.
|
|
*
|
|
* Activity labels are the one exception: `run.generateActivityLabel()` bypasses
|
|
* the graph override and calls the endpoint's resolved client options, so that
|
|
* request does go out over HTTP — to `fake-label-server.js` on 127.0.0.1.
|
|
*/
|
|
function writeRuntimeMockConfig() {
|
|
const template = fs.readFileSync(configTemplatePath, 'utf8');
|
|
let config =
|
|
process.env.E2E_MODEL_SPECS_ENFORCE === 'true'
|
|
? template.replace('\n enforce: false\n', '\n enforce: true\n')
|
|
: template;
|
|
config = config.replace(
|
|
' agents:\n',
|
|
` agents:\n managementApi: ${JSON.stringify({ auth: managementAuth })}\n`,
|
|
);
|
|
const dynamicMcpConfig = enableDynamicMcp
|
|
? {
|
|
allowedDomain: '- http://127.0.0.1:8766',
|
|
stdioEnv: [
|
|
'E2E_MCP_LIST_CHANGED: "true"',
|
|
` E2E_MCP_STATE_PATH: ${JSON.stringify(MCP_STATE_PATH)}`,
|
|
].join('\n'),
|
|
networkServers: [
|
|
'e2e-streamable:',
|
|
' type: streamable-http',
|
|
' url: http://127.0.0.1:8766/mcp',
|
|
' title: E2E Streamable HTTP',
|
|
' description: Dynamic real-SDK Streamable HTTP fixture for mock end-to-end tests.',
|
|
' timeout: 30000',
|
|
' e2e-sse:',
|
|
' type: sse',
|
|
' url: http://127.0.0.1:8766/sse',
|
|
' title: E2E SSE',
|
|
' description: Dynamic real-SDK legacy SSE fixture for mock end-to-end tests.',
|
|
' timeout: 30000',
|
|
].join('\n'),
|
|
}
|
|
: { allowedDomain: '', stdioEnv: '', networkServers: '' };
|
|
/** Longer than the 30s default, so no fixture can time out sooner, and distinct from it, so the
|
|
* startup config shows whether the deployment's operation limits reach the client. */
|
|
const mcpAppOperationLimits = [' operationLimits:', ' timeoutMs: 45000'];
|
|
const mcpAppsConfig = enableMcpApps
|
|
? {
|
|
setting:
|
|
mcpAppsPolicy === 'omitted'
|
|
? ''
|
|
: `apps: ${mcpAppsPolicy === 'false' ? 'false' : 'true'}`,
|
|
sandbox: [
|
|
'mcpAppSandbox:',
|
|
` url: ${JSON.stringify(mcpSandboxUrl)}`,
|
|
...mcpAppOperationLimits,
|
|
].join('\n'),
|
|
allowedDomain: `- http://127.0.0.1:${MCP_APP_PORT}`,
|
|
server: [
|
|
'e2e-app:',
|
|
' type: streamable-http',
|
|
` url: http://127.0.0.1:${MCP_APP_PORT}/mcp`,
|
|
' title: E2E MCP App',
|
|
' description: Official-SDK MCP App fixture for browser integration tests.',
|
|
' timeout: 30000',
|
|
].join('\n '),
|
|
}
|
|
: {
|
|
setting: '',
|
|
sandbox: ['mcpAppSandbox:', ...mcpAppOperationLimits].join('\n'),
|
|
allowedDomain: '',
|
|
server: '',
|
|
};
|
|
const mcpAppsRateLimits =
|
|
mcpAppResourceLimit != null && mcpAppToolCallLimit != null
|
|
? [
|
|
' mcpApps:',
|
|
` resourcesPerMinute: ${mcpAppResourceLimit}`,
|
|
` toolCallsPerMinute: ${mcpAppToolCallLimit}`,
|
|
].join('\n')
|
|
: '';
|
|
const recordProviderBlock = modelFixtureRecording
|
|
? [
|
|
`- name: 'Replay Record Provider'`,
|
|
` apiKey: '\${E2E_RECORD_PROVIDER_API_KEY}'`,
|
|
` baseURL: '${recordProviderBaseURL}'`,
|
|
' models:',
|
|
' default:',
|
|
` - '${recordProviderModel}'`,
|
|
' fetch: false',
|
|
' titleConvo: false',
|
|
` modelDisplayLabel: 'Replay Record Provider'`,
|
|
].join('\n ')
|
|
: '# __E2E_MODEL_RECORD_PROVIDER__';
|
|
config = config
|
|
.replace('# __E2E_MODEL_RECORD_PROVIDER__', recordProviderBlock)
|
|
.replace(
|
|
'# __E2E_MODEL_RECORD_ADDED_ENDPOINT__',
|
|
modelFixtureRecording
|
|
? `- 'Replay Record Provider'`
|
|
: '# __E2E_MODEL_RECORD_ADDED_ENDPOINT__',
|
|
)
|
|
.replace('# __E2E_DYNAMIC_MCP_ALLOWED_DOMAIN__', dynamicMcpConfig.allowedDomain)
|
|
.replace('# __E2E_DYNAMIC_MCP_STDIO_ENV__', dynamicMcpConfig.stdioEnv)
|
|
.replace('# __E2E_DYNAMIC_MCP_NETWORK_SERVERS__', dynamicMcpConfig.networkServers)
|
|
.replace('# __E2E_MCP_APP_SANDBOX__', mcpAppsConfig.sandbox)
|
|
.replace('# __E2E_MCP_APPS_SETTING__', mcpAppsConfig.setting)
|
|
.replace('# __E2E_MCP_APPS_ALLOWED_DOMAIN__', mcpAppsConfig.allowedDomain)
|
|
.replace('# __E2E_MCP_APPS_SERVER__', mcpAppsConfig.server)
|
|
.replace('# __E2E_MCP_APP_RATE_LIMITS__', mcpAppsRateLimits);
|
|
const codeBridgeURL = process.env.E2E_CODE_BRIDGE_URL;
|
|
const codeBridgePairing = process.env.E2E_CODE_BRIDGE_ADMIN_TOKEN
|
|
? [
|
|
' owner: deployment',
|
|
' pairing:',
|
|
' workerId: e2e-vm',
|
|
' tokenEnv: E2E_CODE_BRIDGE_ADMIN_TOKEN',
|
|
]
|
|
: [];
|
|
config = config.replace(
|
|
'# __E2E_CODE_BRIDGE_CONFIG__',
|
|
codeBridgeURL
|
|
? [
|
|
' - stateful_code_sessions',
|
|
'statefulCodeSessions:',
|
|
' allowedEnvironments: ["conversation"]',
|
|
' environments:',
|
|
' - id: e2e-vm',
|
|
' name: E2E attached VM',
|
|
' type: attached',
|
|
` baseURL: ${JSON.stringify(codeBridgeURL)}`,
|
|
' default: true',
|
|
' configSchema:',
|
|
' permissions:',
|
|
' fileWrite:',
|
|
' allowed: [allow, ask, deny]',
|
|
' default: ask',
|
|
' commandExecution:',
|
|
' allowed: [ask, deny]',
|
|
' default: ask',
|
|
...codeBridgePairing,
|
|
].join('\n ')
|
|
: '# __E2E_CODE_BRIDGE_CONFIG__',
|
|
);
|
|
/** Keep the generated config in lockstep with the overridable label-server
|
|
* port: the template hard-codes 8889, so an `E2E_LABEL_PORT` override that
|
|
* moved only the server and its health check would report ready while
|
|
* every activity-label request went to the wrong port. */
|
|
if (LABEL_PORT !== '8889') {
|
|
config = config.split('127.0.0.1:8889').join(`127.0.0.1:${LABEL_PORT}`);
|
|
}
|
|
if (enableDynamicMcp && MCP_DYNAMIC_PORT !== '8766') {
|
|
config = config.split('127.0.0.1:8766').join(`127.0.0.1:${MCP_DYNAMIC_PORT}`);
|
|
}
|
|
if (GIT_HOST_PORT !== '8793') {
|
|
config = config.split('localhost:8793').join(`localhost:${GIT_HOST_PORT}`);
|
|
}
|
|
if (MCP_OAUTH_PORT !== '8767') {
|
|
config = config.split('127.0.0.1:8767').join(`127.0.0.1:${MCP_OAUTH_PORT}`);
|
|
}
|
|
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
|
fs.writeFileSync(configPath, config);
|
|
if (enableDynamicMcp) {
|
|
fs.mkdirSync(path.dirname(MCP_STATE_PATH), { recursive: true });
|
|
fs.writeFileSync(MCP_STATE_PATH, `${JSON.stringify({ revision: 0, tool: null })}\n`);
|
|
}
|
|
}
|
|
|
|
/** A kept certificate is replaced when it expires within this window, so no run outlives it. */
|
|
const GIT_HOST_CERT_MIN_SECONDS = 7 * 24 * 60 * 60;
|
|
|
|
function hasFreshGitHostCertificate(): boolean {
|
|
if (!fs.existsSync(gitHostKeyPath) || !fs.existsSync(gitHostCertPath)) {
|
|
return false;
|
|
}
|
|
try {
|
|
/** Exits non-zero when the certificate expires within the window, or cannot be read. */
|
|
execFileSync(
|
|
'openssl',
|
|
['x509', '-checkend', String(GIT_HOST_CERT_MIN_SECONDS), '-noout', '-in', gitHostCertPath],
|
|
{ stdio: 'ignore' },
|
|
);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The fake git host speaks TLS, as every pull request host must, with a certificate for
|
|
* `localhost` the app server trusts through `NODE_EXTRA_CA_CERTS`. Node reads that file once at
|
|
* startup, so it is made here, before any web server starts, and kept while it is valid for a
|
|
* week more. The config module loads again in every worker, which must not swap the pair under a
|
|
* running server, so only the main process ever replaces one.
|
|
*/
|
|
function ensureGitHostCertificate() {
|
|
const exists = fs.existsSync(gitHostKeyPath) && fs.existsSync(gitHostCertPath);
|
|
if (exists && process.env.TEST_WORKER_INDEX != null) {
|
|
return;
|
|
}
|
|
if (hasFreshGitHostCertificate()) {
|
|
return;
|
|
}
|
|
fs.mkdirSync(path.dirname(gitHostCertPath), { recursive: true });
|
|
execFileSync(
|
|
'openssl',
|
|
[
|
|
'req',
|
|
'-x509',
|
|
'-newkey',
|
|
'rsa:2048',
|
|
'-nodes',
|
|
'-keyout',
|
|
gitHostKeyPath,
|
|
'-out',
|
|
gitHostCertPath,
|
|
'-days',
|
|
'30',
|
|
'-subj',
|
|
'/CN=localhost',
|
|
'-addext',
|
|
'subjectAltName=DNS:localhost',
|
|
],
|
|
{ stdio: 'ignore' },
|
|
);
|
|
}
|
|
|
|
function neutralizeCredentialEnv(env: NodeJS.ProcessEnv, keep: Set<string>) {
|
|
for (const key of Object.keys(env)) {
|
|
if (!keep.has(key) && SECRET_KEY_PATTERN.test(key)) {
|
|
env[key] = '';
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Blank any credential-like vars from a local `.env` so they never reach the test server. */
|
|
function neutralizeDotenvSecrets(envFile: string, keep: Set<string>) {
|
|
if (!fs.existsSync(envFile)) {
|
|
return;
|
|
}
|
|
const lines = fs.readFileSync(envFile, 'utf8').split('\n');
|
|
for (const line of lines) {
|
|
const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=/);
|
|
if (!match) {
|
|
continue;
|
|
}
|
|
const key = match[1];
|
|
if (keep.has(key)) {
|
|
continue;
|
|
}
|
|
if (SECRET_KEY_PATTERN.test(key)) {
|
|
process.env[key] = '';
|
|
}
|
|
}
|
|
}
|
|
|
|
writeRuntimeMockConfig();
|
|
ensureGitHostCertificate();
|
|
neutralizeCredentialEnv(process.env, preservedCredentialEnvKeys);
|
|
Object.assign(process.env, baseEnv);
|
|
neutralizeDotenvSecrets(path.resolve(rootPath, '.env'), preservedCredentialEnvKeys);
|
|
|
|
export default defineConfig({
|
|
globalSetup: require.resolve('./setup/global-setup'),
|
|
globalTeardown: require.resolve('./setup/global-teardown.mock'),
|
|
testDir: 'specs/mock/',
|
|
/** Record mode swaps the fake model for a real provider, so it must never
|
|
* run the whole mock suite: every spec's prompts would reach the paid
|
|
* endpoint, and each fresh conversation would truncate and rewrite the one
|
|
* selected fixture, leaving whichever scenario ran last. Without this an
|
|
* unfiltered entry point (`npm run e2e:mock`) does exactly that. */
|
|
...(modelFixtureRecording ? { testMatch: /model-replay[a-z-]*\.spec\.ts$/ } : {}),
|
|
outputDir: 'specs/.test-results',
|
|
fullyParallel: true,
|
|
forbidOnly: !!process.env.CI,
|
|
retries: process.env.CI ? 2 : 0,
|
|
/** In CI a test that fails and then passes on a retry fails the run. The retries stay for
|
|
* what they show: a trace of each failed attempt, video of the first retry, and whether the
|
|
* failure is deterministic. Derived profiles inherit this with the rest of the config. */
|
|
failOnFlakyTests: !!process.env.CI,
|
|
workers: 1,
|
|
/** In CI the JSON results land beside the HTML report (reporters end in order, after the HTML
|
|
* reporter has replaced its folder), so the report artifact also carries the per-file
|
|
* timings `.github/scripts/playwright-shards.mjs record` rebuilds the shard plan from.
|
|
* The last reporter names each retried pass in the job log and the job summary. */
|
|
reporter: process.env.CI
|
|
? [
|
|
['html', { outputFolder: reportPath, open: 'never' }],
|
|
['line'],
|
|
['json', { outputFile: path.join(reportPath, 'results.json') }],
|
|
['./reporters/retried.mjs'],
|
|
]
|
|
: [['html', { outputFolder: reportPath }], ['list']],
|
|
use: {
|
|
baseURL,
|
|
video: 'on-first-retry',
|
|
trace: 'retain-on-failure',
|
|
ignoreHTTPSErrors: true,
|
|
headless: true,
|
|
storageState: path.resolve(process.cwd(), 'e2e/storageState.json'),
|
|
screenshot: 'only-on-failure',
|
|
/** Each test opens a fresh context, so the PWA service worker installs and claims the page
|
|
* during that test's first load. A `page.route` installed within a few milliseconds of the
|
|
* claim is never consulted, and the request it was meant to hold reaches the server. This
|
|
* blocks every worker in the context, including the one Sandpack's static preview serves
|
|
* an artifact's files from, so a spec that needs one opts back in with
|
|
* `test.use({ serviceWorkers: 'allow' })`. Derived profiles inherit this through `use`. */
|
|
serviceWorkers: 'block',
|
|
},
|
|
expect: {
|
|
timeout: 10000,
|
|
},
|
|
projects: [
|
|
{
|
|
name: chromiumChannel ?? 'chromium',
|
|
use: {
|
|
...devices['Desktop Chrome'],
|
|
...(chromiumChannel ? { channel: chromiumChannel } : {}),
|
|
},
|
|
},
|
|
],
|
|
webServer: [
|
|
{
|
|
// URL-based MCP fixture for the allowlist-override spec (its health route is GET /).
|
|
command: `node ${mcpHttpServerPath}`,
|
|
cwd: rootPath,
|
|
env: { ...process.env, E2E_MCP_HTTP_PORT: MCP_HTTP_PORT },
|
|
url: `http://127.0.0.1:${MCP_HTTP_PORT}/`,
|
|
stdout: 'pipe',
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
{
|
|
// Protected resource whose OAuth flow intentionally remains pending across navigation.
|
|
command: `node ${mcpOAuthServerPath}`,
|
|
cwd: rootPath,
|
|
env: { ...process.env, E2E_MCP_OAUTH_PORT: MCP_OAUTH_PORT },
|
|
url: `http://127.0.0.1:${MCP_OAUTH_PORT}/`,
|
|
stdout: 'pipe',
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
...(enableDynamicMcp
|
|
? [
|
|
{
|
|
// One real SDK server exposes both current HTTP and legacy SSE transports.
|
|
command: `node ${dynamicMcpServerPath}`,
|
|
cwd: rootPath,
|
|
env: {
|
|
...process.env,
|
|
E2E_MCP_DYNAMIC_PORT: MCP_DYNAMIC_PORT,
|
|
E2E_MCP_STATE_PATH: MCP_STATE_PATH,
|
|
},
|
|
url: `http://127.0.0.1:${MCP_DYNAMIC_PORT}/`,
|
|
stdout: 'pipe' as const,
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
]
|
|
: []),
|
|
...(enableMcpApps
|
|
? [
|
|
{
|
|
command: `node ${mcpAppServerPath}`,
|
|
cwd: rootPath,
|
|
env: { ...process.env, E2E_MCP_APP_PORT: MCP_APP_PORT },
|
|
url: `http://127.0.0.1:${MCP_APP_PORT}/`,
|
|
stdout: 'pipe' as const,
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
]
|
|
: []),
|
|
{
|
|
// Serves the activity-label model call (the custom endpoints' baseURL).
|
|
command: `node ${labelServerPath}`,
|
|
cwd: rootPath,
|
|
env: { ...process.env, E2E_LABEL_PORT: LABEL_PORT },
|
|
url: `http://127.0.0.1:${LABEL_PORT}/`,
|
|
stdout: 'pipe',
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
{
|
|
// Stateful provider-boundary fake for Assistant CRUD and streamed runs.
|
|
command: `node ${assistantsServerPath}`,
|
|
cwd: rootPath,
|
|
env: { ...process.env, E2E_ASSISTANTS_PORT: ASSISTANTS_PORT },
|
|
url: `http://127.0.0.1:${ASSISTANTS_PORT}/`,
|
|
stdout: 'pipe',
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
{
|
|
// Fake BFL endpoint for the Flux tool (FLUX_API_BASE_URL).
|
|
command: `node ${fluxServerPath}`,
|
|
cwd: rootPath,
|
|
env: { ...process.env, E2E_FLUX_PORT: FLUX_PORT },
|
|
url: `http://127.0.0.1:${FLUX_PORT}/`,
|
|
stdout: 'pipe',
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
{
|
|
// Fake code-execution API for file-provisioning specs (LIBRECHAT_CODE_BASEURL).
|
|
command: `node ${codeServerPath}`,
|
|
cwd: rootPath,
|
|
env: { ...process.env, E2E_CODE_API_PORT: CODE_API_PORT },
|
|
url: `http://127.0.0.1:${CODE_API_PORT}/health`,
|
|
stdout: 'pipe',
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
{
|
|
// Fake GitHub Enterprise host for the pull request scenarios (`pullRequests.host`).
|
|
command: `node ${gitHostServerPath}`,
|
|
cwd: rootPath,
|
|
env: {
|
|
...process.env,
|
|
E2E_GIT_HOST_PORT: GIT_HOST_PORT,
|
|
E2E_GIT_HOST_KEY: gitHostKeyPath,
|
|
E2E_GIT_HOST_CERT: gitHostCertPath,
|
|
},
|
|
url: `https://localhost:${GIT_HOST_PORT}/`,
|
|
ignoreHTTPSErrors: true,
|
|
stdout: 'pipe',
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
{
|
|
// Fake RAG (vector DB) API for file-provisioning specs (RAG_API_URL).
|
|
command: `node ${ragServerPath}`,
|
|
cwd: rootPath,
|
|
env: { ...process.env, E2E_RAG_API_PORT: RAG_API_PORT },
|
|
url: `http://127.0.0.1:${RAG_API_PORT}/health`,
|
|
stdout: 'pipe',
|
|
timeout: 60_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
{
|
|
// Start one LibreChat process, or a two-process topology behind a test-only proxy, after the
|
|
// network fixtures so inspection and persistent connections agree.
|
|
command: `node ${serverPath}`,
|
|
cwd: rootPath,
|
|
// Only the one-replica harness may assert the scheduler's single-process topology.
|
|
// The two-replica MCP suite must leave scheduled writes disabled.
|
|
env: {
|
|
...process.env,
|
|
...(replicaCount === 1 ? { SCHEDULES_SINGLE_PROCESS: 'true' } : {}),
|
|
},
|
|
url: baseURL,
|
|
stdout: 'pipe',
|
|
ignoreHTTPSErrors: true,
|
|
timeout: 120_000,
|
|
reuseExistingServer: false,
|
|
},
|
|
],
|
|
});
|