LibreChat/examples/sdk-runtime/benchmark.cjs
Danny Avila 1b8e89d7d8
🧩 feat: Stream Host Models Through Remote SDK Runtimes (#18250)
* feat: stream authenticated host models through remote SDK runtimes

* fix: bind authorized tool schemas on remote host models
2026-11-17 09:36:04 -05:00

157 lines
4.8 KiB
JavaScript

const { performance } = require('node:perf_hooks');
const { randomUUID } = require('node:crypto');
const {
HostChatModel,
HostModelAdmission,
createHostModelService,
GraphEvents,
Providers,
ProviderTextAdmission,
createHostProviderTextService,
} = require('@librechat/agents');
const { HumanMessage } = require('@librechat/agents/langchain/messages');
const { createRemoteSdkRun } = require('@librechat/api/sdk');
const { convertToOpenAITool } = require('@langchain/core/utils/function_calling');
const { createExampleRun, createExampleModel, createExampleTool } = require('./agent.cjs');
const hostOwnedModel = process.env.SDK_BENCHMARK_HOST_MODEL === 'true';
const modelAdmission = new HostModelAdmission({ maxConcurrentCalls: 4, maxRequestBytes: 4194304 });
const protectedOutput = process.env.SDK_BENCHMARK_PROTECTION === 'true';
const admission = new ProviderTextAdmission({
maxConcurrentInspections: 4,
maxBufferedBytes: 4194304,
});
// A deterministic pass-through policy measures bridge cost, not production PII detection.
const policy = {
version: 1,
timeoutMs: 10000,
maxAttemptBytes: 1048576,
maxBufferedBytes: 4194304,
classify: () => 'prose',
inspect: ({ content }) => ({
version: 1,
ok: true,
value: { content, replacements: 0, categories: [] },
}),
};
async function measure(remote) {
const identity = {
runtimeId: process.env.SDK_RUNTIME_ID || 'remote-example',
agentId: process.env.SDK_BENCHMARK_AGENT_ID || 'benchmark',
runId: randomUUID(),
conversationId: randomUUID(),
userId: 'benchmark',
};
let firstEventMs;
const started = performance.now();
const config = {
runId: identity.runId,
...(protectedOutput && { providerTextProtection: policy }),
returnContent: true,
tokenCounter: () => 1,
customHandlers: {
[GraphEvents.ON_MESSAGE_DELTA]: {
handle: async () => {
firstEventMs ??= performance.now() - started;
},
},
},
graphConfig: {
type: 'standard',
llmConfig: { provider: Providers.OPENAI },
agentId: identity.agentId,
},
};
const model = hostOwnedModel
? createHostModelService({
identity,
model: createExampleModel(),
descriptor: {
version: 1,
reference: 'primary',
provider: Providers.OPENAI,
model: 'deterministic-host',
streaming: true,
tools: [convertToOpenAITool(createExampleTool())],
},
admission: modelAdmission,
limits: { timeoutMs: 10000, maxRequestBytes: 1048576, maxResponseBytes: 4194304 },
})
: undefined;
const run = remote
? await createRemoteSdkRun({
config,
identity,
hostServices: {
model,
...(protectedOutput && {
providerText: createHostProviderTextService({
identity,
policy,
admission,
inspectionTimeoutMs: 5000,
}),
}),
},
remote: {
url: process.env.SDK_RUNTIME_URL || 'ws://127.0.0.1:8804/sdk',
secret: process.env.SDK_RUNTIME_SECRET || '',
...(process.env.SDK_RUNTIME_RECOVERY === 'true' && {
recovery: { maxDisconnectMs: 15000, retryDelayMs: 250, maxJournalBytes: 67108864 },
}),
},
})
: await createExampleRun({
config,
identity,
hostModel: model
? new HostChatModel({ identity, ...model }).bindTools(model.descriptor.tools)
: undefined,
compileOptions: {},
signal: new AbortController().signal,
agent: { id: identity.agentId },
});
await run.processStream(
{ messages: [new HumanMessage('Inspect and review the fixture.')] },
{
version: 'v2',
configurable: { thread_id: identity.conversationId },
},
);
return { firstEventMs, completedMs: performance.now() - started };
}
async function main() {
await measure(false);
await measure(true);
const local = [],
remote = [];
for (let i = 0; i < 5; i++) {
local.push(await measure(false));
remote.push(await measure(true));
}
const median = (samples, key) => samples.map((item) => item[key]).sort((a, b) => a - b)[2];
console.log(
JSON.stringify(
{
mode: 'deterministic custom graph, five warm samples',
protectedOutput,
hostOwnedModel,
local,
remote,
median: {
localFirstEventMs: median(local, 'firstEventMs'),
remoteFirstEventMs: median(remote, 'firstEventMs'),
localCompletedMs: median(local, 'completedMs'),
remoteCompletedMs: median(remote, 'completedMs'),
},
},
null,
2,
),
);
}
main().catch((error) => {
console.error('Benchmark failed', { name: error.name });
process.exitCode = 1;
});