mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-18 22:14:27 +00:00
* fix(data-schemas): refresh FerretDB harness model coverage, fix compile errors, add bulkWrite differentials Track 2 of the search-stack plan (PLAN.md "FerretDB track"): - Replace the three hand-rolled 29-model MODEL_SCHEMAS maps in multiTenancy/sharding/orgOperations.ferretdb.spec.ts with a shared getModelSchemas(mongoose) helper (misc/ferretdb/schemas.ts) derived from the live createModels() registry, so coverage tracks all 37 current models automatically instead of drifting. Matches the reference pattern in misc/documentdb/compat.documentdb.spec.ts. - Fix the 3 compile-broken specs this uncovered: all three imported a `projectSchema` from '~/schema' that no longer exists (superseded by `chatProjectSchema`), which `tsc --noEmit` flags as TS2724 but the babel-based jest transform silently let through as `undefined`. Removing the hand-rolled maps removes the bad import as a side effect; verified clean with tsc across misc/ferretdb and misc/documentdb. - Add misc/ferretdb/bulkWrite.ferretdb.spec.ts: differential specs for the five bulkWrite flows the plan names as actually at risk (import via bulkSaveConvos/bulkSaveMessages, bulkWriteAclEntries, bulkIncrementTagCounts, Transaction.insertMany, file-TTL bulkWrite via extendFilesTTL). Each flow runs identical operations against a real mongodb-memory-server (always) and, when FERRETDB_URI is set, against FerretDB, asserting normalized result equality. Multi-document transactions already degrade via the existing supportsTransactions probe — not duplicated here. - Land the Spike A BSON-legibility findings (bson-legibility.md, bson-inventory.txt) from the bson-legibility-spike-6e38de worktree so decision 2's evidence is in-repo. Verified against a real FerretDB 2.7.0 + postgres-documentdb 17 stack (docker compose -f misc/ferretdb/docker-compose.ferretdb.yml): all 10 harness spec files pass individually, including all 10 bulkWrite.ferretdb tests (5 mongodb-memory-server baselines + 5 FerretDB differentials). Full packages/data-schemas src/ suite (1907 tests) unaffected. * 📝 docs: make the BSON projection findings self-contained The doc was written for readers who already knew the internal shorthand — it opened on "Spike A executed, Spike B scoped" and referred to Options 1/2/3 and "the handoff" without ever defining them, so a reader arriving from the repo could not follow the argument or act on the recommendation. Reframed around what the document actually investigates: the question is stated up front, the three candidate mechanisms are named in a table before they are compared, and the recommendation refers to them by name. No findings, numbers, or SQL changed. * 📝 docs: drop internal planning references from spec header * fix(data-schemas): keep FerretDB harness schema derivation side-effect free `getModelSchemas()` derived its map by calling `createModels(mongoose)`, which carried three consequences the harness did not want: - Model creation applies the tenant-isolation plugin to the module-level schema singletons, so every harness read and write inherited middleware that throws under `TENANT_ISOLATION_STRICT=true`. - Registering on the default connection meant the benchmark's own `mongoose.connect()` auto-created 37 collections in the URI's base database, adding a database and dozens of collections to the very catalog metrics it measures. - The unfiltered registry provisioned app-wide control-plane models (`SystemGrant`, `AuditLog`, `SkillSyncCredential`, `SkillSyncStatus`) into every org database. The helper now builds the registry on a throwaway Mongoose instance, returns schemas rebuilt from their own definition, options, and declared indexes, skips the four app-wide models (validated against the registry so a rename fails loudly), and memoizes the result. Also in this pass: - The "adds a new collection" migration test used `AuditLog`, which provisioning had already created, so it silently reused the production model and ignored its proposed schema. It now uses a fixture model absent from the registry and asserts the collection is missing beforehand and carries the proposed compound index afterwards. - `bulkWrite` flows run inside `runAsSystem()`; they drive production methods unscoped, as a cross-tenant maintenance job does, and otherwise fail closed under strict tenant isolation. - Phase 2's sparse-index assertion pinned a count the User schema no longer declares; it now checks that each index type round-trips.
84 lines
3.2 KiB
TypeScript
84 lines
3.2 KiB
TypeScript
import type { Schema } from 'mongoose';
|
|
import { createModels } from '~/models';
|
|
|
|
/**
|
|
* Registry entries that production keeps app-wide instead of provisioning into
|
|
* every org database, per the rationale in each factory under `src/models`:
|
|
* `SystemGrant` (cross-tenant control plane), `AuditLog` (append-only chain
|
|
* serialized per `chainKey`), `SkillSyncCredential` and `SkillSyncStatus`
|
|
* (app-wide GitHub skill sync). They are exactly the registry entries that opt
|
|
* out of the tenant-isolation plugin, so provisioning them per org would
|
|
* duplicate credentials and control-plane state into every tenant.
|
|
*/
|
|
export const APP_WIDE_MODELS: ReadonlySet<string> = new Set([
|
|
'AuditLog',
|
|
'SkillSyncCredential',
|
|
'SkillSyncStatus',
|
|
'SystemGrant',
|
|
]);
|
|
|
|
let cachedSchemas: Record<string, Schema> | undefined;
|
|
|
|
/**
|
|
* Rebuilds a schema from its own definition, options, and declared indexes.
|
|
*
|
|
* The registry schemas are module-level singletons that `createModels` mutates
|
|
* with application middleware (tenant isolation, audit chaining). This harness
|
|
* benchmarks raw per-database storage behavior and reads/writes without any
|
|
* tenant context, so it must not inherit that middleware — under
|
|
* `TENANT_ISOLATION_STRICT=true` those hooks throw before a single FerretDB
|
|
* assertion runs. The copy keeps every path, index, and option the collection
|
|
* layout depends on while carrying none of the hooks.
|
|
*/
|
|
function detachSchema(mongoose: typeof import('mongoose'), source: Schema): Schema {
|
|
const detached = new mongoose.Schema(source.obj, source.options);
|
|
const declared = new Set(detached.indexes().map(([fields]) => JSON.stringify(fields)));
|
|
|
|
for (const [fields, options] of source.indexes()) {
|
|
const key = JSON.stringify(fields);
|
|
if (declared.has(key)) {
|
|
continue;
|
|
}
|
|
detached.index(fields, options);
|
|
declared.add(key);
|
|
}
|
|
|
|
return detached;
|
|
}
|
|
|
|
/**
|
|
* Derives a `{ modelName: Schema }` map of every org-local LibreChat model from
|
|
* the live `createModels` registry instead of a hand-maintained list, so each
|
|
* FerretDB harness spec that provisions "one of every org collection" tracks
|
|
* the current model set automatically as models are added or removed.
|
|
*
|
|
* The registry is built on a throwaway Mongoose instance so nothing is
|
|
* registered on the default connection — the benchmark opens that connection
|
|
* itself, and default models would otherwise auto-create their collections in
|
|
* the URI's base database and skew catalog metrics and timings.
|
|
*/
|
|
export function getModelSchemas(mongoose: typeof import('mongoose')): Record<string, Schema> {
|
|
if (cachedSchemas) {
|
|
return cachedSchemas;
|
|
}
|
|
|
|
const registry = createModels(new mongoose.Mongoose());
|
|
|
|
const unknownAppWide = [...APP_WIDE_MODELS].filter((name) => !(name in registry));
|
|
if (unknownAppWide.length > 0) {
|
|
throw new Error(
|
|
`APP_WIDE_MODELS lists models missing from the registry: ${unknownAppWide.join(', ')}`,
|
|
);
|
|
}
|
|
|
|
const schemas: Record<string, Schema> = {};
|
|
for (const [name, model] of Object.entries(registry)) {
|
|
if (APP_WIDE_MODELS.has(name)) {
|
|
continue;
|
|
}
|
|
schemas[name] = detachSchema(mongoose, model.schema);
|
|
}
|
|
|
|
cachedSchemas = schemas;
|
|
return schemas;
|
|
}
|