LibreChat/e2e/specs/mock/scenarios/tool-headers.spec.ts

473 lines
17 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { expect, test } from '@playwright/test';
import type { Page, Route } from '@playwright/test';
import { messagesView } from '../helpers';
/**
* Tool activity headers: a generated title is the only disclosure a lone call needs,
* file cards drop their timers and success summaries, durations and chevrons sit at the
* row's end, and actions whose row a title replaced move onto the title line.
* Conversations are routed fixtures, so every run sees the same rows.
*/
test.describe.configure({ timeout: 120_000 });
const NO_PARENT = '00000000-0000-0000-0000-000000000000';
const DONE = 'Tool headers fixture done.';
const unique = (prefix: string) => `${prefix}-${Date.now()}-${Math.floor(Math.random() * 1e6)}`;
const escapeRe = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
type Part = Record<string, unknown>;
const call = (
id: string,
name: string,
args: Record<string, unknown>,
output: string,
extra: Record<string, unknown> = {},
): Part => ({
type: 'tool_call',
tool_call: {
id,
name,
args: JSON.stringify(args),
output,
progress: 1,
type: 'tool_call',
runStepStatus: 'completed',
...extra,
},
});
const label = (text: string, ids: string[], status = 'ok'): Part => ({
type: 'activity_label',
activity_label: text,
tool_call_ids: ids,
status,
pending: false,
});
async function openConversation(page: Page, content: Part[], attachments?: Part[]) {
const conversationId = unique('e2e-tool-headers');
const now = new Date(0).toISOString();
const messageId = `${conversationId}-msg`;
const message = {
messageId,
conversationId,
parentMessageId: NO_PARENT,
isCreatedByUser: false,
sender: 'Assistant',
endpoint: 'Mock Provider A',
model: 'mock-model-a',
text: '',
content: [...content, { type: 'text', text: DONE }],
...(attachments ? { attachments: attachments.map((a) => ({ ...a, messageId })) } : {}),
createdAt: now,
updatedAt: now,
};
const conversation = {
conversationId,
title: 'Tool headers',
endpoint: 'Mock Provider A',
endpointType: 'custom',
model: 'mock-model-a',
createdAt: now,
updatedAt: now,
};
const convoIdRe = escapeRe(conversationId);
await page.route(new RegExp(`/api/convos/${convoIdRe}(?:\\?.*)?$`), (route: Route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify(conversation),
}),
);
await page.route(new RegExp(`/api/messages/${convoIdRe}(?:\\?.*)?$`), (route: Route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify([message]),
}),
);
await page.goto(`/c/${conversationId}`, { timeout: 30000 });
await expect(messagesView(page).getByText(DONE)).toBeVisible({ timeout: 20000 });
}
/** A reader opening a published link: the conversation's messages, served under a share id. */
async function openSharedConversation(page: Page, content: Part[]) {
const shareId = unique('e2e-shared-tool-headers');
const conversationId = `${shareId}-convo`;
const now = new Date(0).toISOString();
const userMessage = {
messageId: `${shareId}-user`,
conversationId,
parentMessageId: NO_PARENT,
isCreatedByUser: true,
sender: 'User',
text: 'Draft the release notes.',
createdAt: now,
updatedAt: now,
};
const reply = {
messageId: `${shareId}-msg`,
conversationId,
parentMessageId: userMessage.messageId,
isCreatedByUser: false,
sender: 'Assistant',
endpoint: 'Mock Provider A',
model: 'mock-model-a',
text: '',
content: [...content, { type: 'text', text: DONE }],
createdAt: now,
updatedAt: now,
};
const shareIdRe = escapeRe(shareId);
await page.route(new RegExp(`/api/share/${shareIdRe}/config(?:\\?.*)?$`), (route: Route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ appTitle: 'LibreChat' }),
}),
);
await page.route(new RegExp(`/api/share/${shareIdRe}(?:\\?.*)?$`), (route: Route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({
shareId,
conversationId,
title: 'Shared tool headers',
createdAt: now,
updatedAt: now,
messages: [userMessage, reply],
}),
}),
);
await page.goto(`/share/${shareId}`, { timeout: 30000 });
await expect(messagesView(page).getByText(DONE)).toBeVisible({ timeout: 20000 });
}
/** Opens the disclosure named `name` if it is folded, and returns it. */
async function expand(page: Page, name: RegExp) {
const button = messagesView(page).getByRole('button', { name }).first();
await expect(button).toBeVisible();
if ((await button.getAttribute('aria-expanded')) !== 'true') {
await button.click();
}
await expect(button).toHaveAttribute('aria-expanded', 'true');
return button;
}
test.describe('tool activity headers', () => {
test('a titled lone call shows its result under the title with no second row or rail @scenario:titled-lone-call-shows-result-directly', async ({
page,
}) => {
await openConversation(page, [
call(
'r1',
'read_file_mcp_filesystem',
{ path: 'README.md' },
'An open-source AI chat platform.',
),
label('Read the project README', ['r1']),
]);
const view = messagesView(page);
await expand(page, /Read the project README/);
await expect(view.getByText('An open-source AI chat platform.')).toBeVisible();
await expect(view.getByRole('button', { name: /^Ran read_file/ })).toHaveCount(0);
await expect(view.getByTestId('fold-rail')).toHaveCount(0);
});
test('a titled group of several calls keeps a row per call and its tree, without repeat counts @scenario:titled-group-keeps-rows-without-multipliers', async ({
page,
}) => {
const ids = ['w1', 'w2', 'w3'];
await openConversation(page, [
...ids.map((id, i) =>
call(id, 'get_weather_mcp_weather', { city: 'Milan' }, `Milan: ${18 + i}C`),
),
label('Checked the Milan weather', ids),
]);
const view = messagesView(page);
await expand(page, /Checked the Milan weather/);
await expect(view.getByRole('button', { name: /^Ran get_weather/ })).toHaveCount(3);
await expect(view.getByTestId('fold-rail').first()).toBeVisible();
await expect(view.getByText(/×\s*3|x3\b/)).toHaveCount(0);
});
test('a tool row shows its duration and an always-visible chevron at the row end @scenario:tool-row-duration-and-chevron-at-end', async ({
page,
}) => {
await openConversation(page, [
call('q1', 'run_query_mcp_mongo', { collection: 'messages' }, 'COLLSCAN', {
runStepDurationMs: 2400,
}),
call('q2', 'list_indexes_mcp_mongo', { collection: 'messages' }, '_id_', {
runStepDurationMs: 3100,
}),
]);
await expand(page, /^Ran 2 actions|^Ran run_query/);
const row = messagesView(page)
.getByRole('button', { name: /Ran run_query/ })
.first();
await expect(row).toBeVisible();
const duration = row.getByText('2.4s', { exact: true });
await expect(duration).toBeVisible();
await expect(row).not.toContainText('in 2.4s');
const [rowBox, durationBox] = await Promise.all([row.boundingBox(), duration.boundingBox()]);
expect(rowBox && durationBox).toBeTruthy();
/** Pinned to the end: the time's right edge sits within the last 48px of the row. */
expect(rowBox!.x + rowBox!.width - (durationBox!.x + durationBox!.width)).toBeLessThan(48);
await page.mouse.move(0, 0);
const chevronOpacity = await row
.locator('svg')
.last()
.evaluate((el) => getComputedStyle(el).opacity);
expect(chevronOpacity).toBe('1');
});
test('file read and edit cards show the full path with no timer or success summary @scenario:file-cards-show-full-path-without-timer-or-summary', async ({
page,
}) => {
await openConversation(page, [
call('f1', 'read_file', { file_path: 'src/index.ts' }, 'export {};\n', {
runStepDurationMs: 2100,
}),
call(
'f2',
'edit_file',
{
path: 'src/index.ts',
old_text: 'export {};',
new_text: "export { greet } from './greet';",
},
'Edited',
{ runStepDurationMs: 3100 },
),
label('Updated the index module', ['f1', 'f2']),
]);
const view = messagesView(page);
await expand(page, /Updated the index module/);
const read = await expand(page, /^Read index\.ts/);
const edit = await expand(page, /^Edited index\.ts/);
await expect(read).not.toContainText(/\d\.\ds/);
await expect(edit).not.toContainText(/\d\.\ds/);
await expect(view.getByText('src/index.ts', { exact: true })).toHaveCount(2);
await expect(view.getByText('Edited', { exact: true })).toHaveCount(0);
});
test('a failed edit keeps its error output visible @scenario:failed-edit-keeps-error-output', async ({
page,
}) => {
await openConversation(page, [
call(
'e1',
'edit_file',
{ path: 'src/a.ts', old_text: 'missing', new_text: 'replacement' },
'Error: old_text matched 0 locations in src/a.ts.',
{ runStepStatus: 'failed' },
),
]);
await expand(page, /a\.ts/);
await expect(messagesView(page).getByText(/matched 0 locations/)).toBeVisible();
});
test('a file card keeps a warning that follows its success summary @scenario:file-card-keeps-warning-after-summary', async ({
page,
}) => {
await openConversation(page, [
call(
'w1',
'create_file',
{ file_path: 'skills/demo/SKILL.md', content: '# Demo\n\nUse this skill for testing.' },
[
'Created skills/demo/SKILL.md (35 chars).',
'',
'Warnings:',
'- frontmatter.owner [UNKNOWN_KEY]: "owner" is not a recognized frontmatter key and is stored as-is',
].join('\n'),
),
]);
await expand(page, /SKILL\.md/);
await expect(messagesView(page).getByText(/not a recognized frontmatter key/)).toBeVisible();
});
test('a titled failed command opens on its error with the time on the command line @scenario:titled-failed-command-shows-error-and-time', async ({
page,
}) => {
await openConversation(page, [
call('b1', 'bash_tool', { command: 'false' }, 'Error: tool call failed: exit code 2', {
runStepStatus: 'failed',
runStepDurationMs: 10200,
}),
label('Checked the failing command', ['b1'], 'failed'),
]);
const view = messagesView(page);
await expand(page, /Checked the failing command/);
await expect(
view.getByText('Error: tool call failed: exit code 2', { exact: true }),
).toBeVisible();
const time = view.getByTestId('command-time').first();
await expect(time).toContainText('10.2s');
const command = view.getByText('false', { exact: true }).first();
const [timeBox, commandBox] = await Promise.all([time.boundingBox(), command.boundingBox()]);
/** Same line: the time no longer gets a strip of its own above the command. */
expect(
Math.abs(timeBox!.y + timeBox!.height / 2 - (commandBox!.y + commandBox!.height / 2)),
).toBeLessThan(8);
await expect(view.getByTestId('fold-rail')).toHaveCount(0);
});
test('a titled web search keeps its details on the title line @scenario:titled-web-search-details-on-title-line', async ({
page,
}) => {
const ws = call('s1', 'web_search', { query: 'css view transitions support' }, 'results');
await openConversation(
page,
[ws, label('Searched for View Transitions support', ['s1'])],
[
{
type: 'web_search',
toolCallId: 's1',
web_search: {
turn: 0,
organic: [{ link: 'https://developer.mozilla.org/', title: 'MDN on view transitions' }],
topStories: [],
},
},
],
);
const view = messagesView(page);
const header = await expand(page, /Searched for View Transitions support/);
await expect(view.getByRole('link', { name: /MDN on view transitions/ })).toBeVisible();
const details = view.getByRole('button', { name: 'Search details' });
await expect(details).toBeVisible();
const [headerBox, detailsBox] = await Promise.all([
header.boundingBox(),
details.boundingBox(),
]);
expect(
Math.abs(headerBox!.y + headerBox!.height / 2 - (detailsBox!.y + detailsBox!.height / 2)),
).toBeLessThan(8);
await details.focus();
await expect(page.getByText('css view transitions support')).toBeVisible();
});
test('a loaded skill shows its instructions and opens the skill @scenario:loaded-skill-shows-instructions-and-opens-skill', async ({
page,
}) => {
const skill = {
_id: 'e2e-skill-release-notes',
name: 'release-notes',
description: 'Drafts release notes.',
body: 'Group merged PRs by type.',
};
await page.route(/\/api\/skills(?:\?.*)?$/, (route: Route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ skills: [skill], has_more: false, after: null }),
}),
);
await page.route(new RegExp(`/api/skills/${skill._id}$`), (route: Route) =>
route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(skill) }),
);
await openConversation(page, [
call(
'k1',
'skill',
{ skillName: 'release-notes' },
'Skill "release-notes" loaded. Follow the instructions below.',
),
]);
const view = messagesView(page);
await expand(page, /release-notes/);
await expect(view.getByText('Group merged PRs by type.')).toBeVisible();
await expect(
view.getByText('Skill "release-notes" loaded. Follow the instructions below.'),
).toHaveCount(0);
await view.getByRole('button', { name: 'Open skill' }).click();
await expect(page).toHaveURL(new RegExp(`/skills/${skill._id}`));
});
test('a skill outside the library falls back to the tool output with no link @scenario:unknown-skill-falls-back-to-output', async ({
page,
}) => {
await page.route(/\/api\/skills(?:\?.*)?$/, (route: Route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ skills: [], has_more: false, after: null }),
}),
);
await openConversation(page, [
call('k2', 'skill', { skillName: 'deployment-only' }, 'Skill "deployment-only" loaded.'),
]);
const view = messagesView(page);
await expand(page, /deployment-only/);
await expect(view.getByText('Skill "deployment-only" loaded.')).toBeVisible();
await expect(view.getByRole('button', { name: 'Open skill' })).toHaveCount(0);
});
test("a shared skill call keeps its output instead of the viewer's same-named skill @scenario:shared-skill-call-keeps-output", async ({
page,
}) => {
const viewerSkill = {
_id: 'e2e-skill-viewer-release-notes',
name: 'release-notes',
description: "The viewer's own release notes skill.",
body: 'The viewer wrote these instructions.',
};
await page.route(/\/api\/skills(?:\?.*)?$/, (route: Route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ skills: [viewerSkill], has_more: false, after: null }),
}),
);
await page.route(new RegExp(`/api/skills/${viewerSkill._id}$`), (route: Route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify(viewerSkill),
}),
);
await openSharedConversation(page, [
call(
'k3',
'skill',
{ skillName: 'release-notes' },
'Skill "release-notes" loaded. Follow the instructions below.',
),
]);
const view = messagesView(page);
/** A lookup would start as the row opens, and the viewer's library answers at once, so
* waiting out a short window for the request is what lets the absence checks below fail
* against the old behavior instead of passing before its fetch lands. */
const lookup = page
.waitForRequest((request) => new URL(request.url()).pathname.startsWith('/api/skills'), {
timeout: 2000,
})
.then(
() => true,
() => false,
);
await expand(page, /release-notes/);
await expect(
view.getByText('Skill "release-notes" loaded. Follow the instructions below.'),
).toBeVisible();
expect(await lookup).toBe(false);
await expect(view.getByText('The viewer wrote these instructions.')).toHaveCount(0);
await expect(view.getByRole('button', { name: 'Open skill' })).toHaveCount(0);
});
test('an agent handoff opens like a tool disclosure and copies its instructions @scenario:handoff-reveals-instructions-with-copy', async ({
page,
}) => {
await openConversation(page, [
call('h1', 'lc_transfer_to_billing_agent', { instructions: 'User needs billing help' }, 'ok'),
]);
const view = messagesView(page);
await expand(page, /^Transferred to/);
await expect(view.getByText('User needs billing help')).toBeVisible();
await expect(view.getByRole('button', { name: /copy/i }).first()).toBeAttached();
});
});