mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-11-20 23:13:35 +00:00
🗂️ perf: Drop Schema Indexes That a Wider Index Already Covers
31 indexes are a key prefix of another index on the same collection that holds every document, so they serve no read the wider index cannot, yet every write maintains them. Remove the declarations and add a spec that fails when one is declared again, and that pins the eight field-level indexes kept because their name collides with a legacy unique index: that collision is what surfaces the tenant-index migration hint. Partial, sparse, hidden and collated indexes never count as a cover, and prefixes whose only cover is multikey (users/groups tenantId) stay. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
28d8780dad
commit
fe2cd45c43
23 changed files with 120 additions and 31 deletions
|
|
@ -16,7 +16,6 @@ const aclEntrySchema: Schema<IAclEntry> = new Schema<IAclEntry>(
|
|||
required: function (this: IAclEntry) {
|
||||
return this.principalType !== PrincipalType.PUBLIC;
|
||||
},
|
||||
index: true,
|
||||
},
|
||||
principalModel: {
|
||||
type: String,
|
||||
|
|
@ -33,7 +32,6 @@ const aclEntrySchema: Schema<IAclEntry> = new Schema<IAclEntry>(
|
|||
resourceId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
permBits: {
|
||||
type: Number,
|
||||
|
|
|
|||
|
|
@ -186,7 +186,6 @@ const agentSchema: Schema<IAgent> = new Schema<IAgent>(
|
|||
},
|
||||
tenantId: {
|
||||
type: String,
|
||||
index: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ const agentApiKeySchema: Schema<IAgentApiKey> = new Schema(
|
|||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: 'User',
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
name: {
|
||||
type: String,
|
||||
|
|
|
|||
|
|
@ -23,12 +23,10 @@ const agentCategorySchema: Schema<IAgentCategory> = new Schema<IAgentCategory>(
|
|||
order: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
index: true,
|
||||
},
|
||||
isActive: {
|
||||
type: Boolean,
|
||||
default: true,
|
||||
index: true,
|
||||
},
|
||||
custom: {
|
||||
type: Boolean,
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ const assistantSchema: Schema<IAssistant> = new Schema<IAssistant>(
|
|||
},
|
||||
tenantId: {
|
||||
type: String,
|
||||
index: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ const chatProjectSchema: Schema<IChatProjectDocument> = new Schema<IChatProjectD
|
|||
user: {
|
||||
type: String,
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
conversationCount: {
|
||||
type: Number,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ const configSchema: Schema<IConfig> = new Schema<IConfig>(
|
|||
type: String,
|
||||
enum: Object.values(PrincipalType),
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
principalId: {
|
||||
type: String,
|
||||
|
|
@ -24,7 +23,6 @@ const configSchema: Schema<IConfig> = new Schema<IConfig>(
|
|||
priority: {
|
||||
type: Number,
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
overrides: {
|
||||
type: Schema.Types.Mixed,
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ const conversationTag: Schema<IConversationTag> = new Schema<IConversationTag>(
|
|||
{
|
||||
tag: {
|
||||
type: String,
|
||||
index: true,
|
||||
},
|
||||
user: {
|
||||
type: String,
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ const convoSchema: Schema<IConversation> = new Schema(
|
|||
titleRevision: { type: Number },
|
||||
user: {
|
||||
type: String,
|
||||
index: true,
|
||||
meiliIndex: true,
|
||||
},
|
||||
messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }],
|
||||
|
|
@ -402,7 +401,6 @@ const convoSchema: Schema<IConversation> = new Schema(
|
|||
},
|
||||
tenantId: {
|
||||
type: String,
|
||||
index: true,
|
||||
},
|
||||
pinned: {
|
||||
type: Boolean,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ const toolFavoriteSchema: Schema<IToolFavorite> = new Schema(
|
|||
type: Schema.Types.ObjectId,
|
||||
ref: 'User',
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
itemType: {
|
||||
type: String,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ const MemoryEntrySchema: Schema<IMemoryEntry> = new Schema({
|
|||
userId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'User',
|
||||
index: true,
|
||||
required: true,
|
||||
},
|
||||
key: {
|
||||
|
|
|
|||
|
|
@ -12,13 +12,11 @@ const messageSchema: Schema<IMessage> = new Schema(
|
|||
},
|
||||
conversationId: {
|
||||
type: String,
|
||||
index: true,
|
||||
required: true,
|
||||
meiliIndex: true,
|
||||
},
|
||||
user: {
|
||||
type: String,
|
||||
index: true,
|
||||
required: true,
|
||||
default: null,
|
||||
meiliIndex: true,
|
||||
|
|
@ -327,7 +325,6 @@ const messageSchema: Schema<IMessage> = new Schema(
|
|||
},
|
||||
tenantId: {
|
||||
type: String,
|
||||
index: true,
|
||||
},
|
||||
},
|
||||
{ timestamps: true },
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ const queuedTurnSchema: Schema<IAgentQueuedTurnDocument> = new Schema(
|
|||
required: true,
|
||||
index: true,
|
||||
},
|
||||
tenantId: { type: String, index: true },
|
||||
tenantId: { type: String },
|
||||
conversationId: { type: String, required: true, maxlength: 256 },
|
||||
agentId: { type: String, required: true, maxlength: 256 },
|
||||
parentMessageId: {
|
||||
|
|
|
|||
117
packages/data-schemas/src/schema/redundantIndexes.spec.ts
Normal file
117
packages/data-schemas/src/schema/redundantIndexes.spec.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
import mongoose from 'mongoose';
|
||||
import type { IndexDefinition, IndexOptions, Schema } from 'mongoose';
|
||||
import { SUPERSEDED_INDEXES } from '../migrations/tenantIndexes';
|
||||
import { createModels } from '../models';
|
||||
|
||||
/** An index that holds an entry for every document and answers simple-collation reads. */
|
||||
const holdsEveryDocument = (options: IndexOptions) =>
|
||||
!options.sparse &&
|
||||
!options.hidden &&
|
||||
options.partialFilterExpression == null &&
|
||||
options.collation == null;
|
||||
|
||||
/** An index that only costs writes once another index serves its reads. */
|
||||
const isPlain = (options: IndexOptions) =>
|
||||
!options.unique && options.expireAfterSeconds == null && holdsEveryDocument(options);
|
||||
|
||||
/** The B-tree direction of a key; text, 2dsphere, hashed and wildcard keys serve other reads. */
|
||||
const direction = (value: unknown): 1 | -1 | undefined => {
|
||||
const dir = Number(value);
|
||||
return dir === 1 || dir === -1 ? dir : undefined;
|
||||
};
|
||||
|
||||
/** The name MongoDB gives an index declared without one. */
|
||||
const indexName = (keys: IndexDefinition) =>
|
||||
Object.entries(keys)
|
||||
.map(([field, dir]) => `${field}_${dir}`)
|
||||
.join('_');
|
||||
|
||||
/** True when `field`, or any parent of it, is an array, which makes an index on it multikey. */
|
||||
const isArrayPath = (schema: Schema, field: string) =>
|
||||
field
|
||||
.split('.')
|
||||
.some((_, i, parts) => schema.path(parts.slice(0, i + 1).join('.'))?.instance === 'Array');
|
||||
|
||||
/**
|
||||
* True when `long` serves every read `short` does: `short`'s keys lead `long` in order, all in
|
||||
* the same or all in the opposite direction (a scan runs either way), every key of `long` is a
|
||||
* B-tree key, and no trailing key is multikey, which would repeat each document once per array
|
||||
* element in a scan over the prefix.
|
||||
*/
|
||||
function covers(schema: Schema, long: IndexDefinition, short: IndexDefinition): boolean {
|
||||
const shortKeys = Object.entries(short);
|
||||
const longKeys = Object.entries(long);
|
||||
const first = direction(shortKeys[0]?.[1]);
|
||||
const lead = direction(longKeys[0]?.[1]);
|
||||
if (first === undefined || lead === undefined || shortKeys.length > longKeys.length) {
|
||||
return false;
|
||||
}
|
||||
const sign = first * lead;
|
||||
return (
|
||||
longKeys.every(([, dir]) => direction(dir) !== undefined) &&
|
||||
shortKeys.every(([field, dir], i) => {
|
||||
const expected = direction(dir);
|
||||
return (
|
||||
expected !== undefined &&
|
||||
longKeys[i][0] === field &&
|
||||
direction(longKeys[i][1]) === expected * sign
|
||||
);
|
||||
}) &&
|
||||
!longKeys.slice(shortKeys.length).some(([field]) => isArrayPath(schema, field))
|
||||
);
|
||||
}
|
||||
|
||||
/** Every schema's declared indexes with their declaration order, keyed by collection. */
|
||||
const declaredIndexes = () =>
|
||||
Object.values(createModels(mongoose)).map(({ schema, collection }) => ({
|
||||
schema,
|
||||
collection: collection.collectionName,
|
||||
legacy: SUPERSEDED_INDEXES[collection.collectionName] ?? [],
|
||||
entries: schema.indexes().map(([keys, options], at) => ({ keys, options, at })),
|
||||
}));
|
||||
|
||||
/**
|
||||
* A plain index whose keys lead another index that holds every document costs a write on each
|
||||
* insert and update for reads the wider index already serves; the wider keys make a scan over
|
||||
* the prefix alone touch more pages, which is the trade this test accepts. A field-level index
|
||||
* that shares its name with a legacy unique index stays: colliding with that index on startup
|
||||
* is what surfaces the tenant-index migration hint.
|
||||
*/
|
||||
describe('schema indexes', () => {
|
||||
it('declares no plain index that another index already serves', () => {
|
||||
const redundant = declaredIndexes().flatMap(({ schema, collection, legacy, entries }) =>
|
||||
entries
|
||||
.filter(({ keys, options }) => isPlain(options) && !legacy.includes(indexName(keys)))
|
||||
.filter(({ keys, at }) =>
|
||||
entries.some(
|
||||
(other) =>
|
||||
other.at !== at &&
|
||||
holdsEveryDocument(other.options) &&
|
||||
covers(schema, other.keys, keys) &&
|
||||
/** Of two identical declarations, only the later one is redundant. */
|
||||
(Object.keys(other.keys).length > Object.keys(keys).length || other.at < at),
|
||||
),
|
||||
)
|
||||
.map(({ keys }) => `${collection}: ${JSON.stringify(keys)}`),
|
||||
);
|
||||
expect(redundant).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps the field-level indexes whose name a legacy unique index shares', () => {
|
||||
const tripwires = declaredIndexes().flatMap(({ collection, legacy, entries }) =>
|
||||
entries
|
||||
.filter(({ keys, options }) => isPlain(options) && legacy.includes(indexName(keys)))
|
||||
.map(({ keys }) => `${collection}.${indexName(keys)}`),
|
||||
);
|
||||
expect(tripwires.sort()).toEqual([
|
||||
'accessroles.accessRoleId_1',
|
||||
'agentcategories.value_1',
|
||||
'conversations.conversationId_1',
|
||||
'mcpservers.serverName_1',
|
||||
'messages.messageId_1',
|
||||
'presets.presetId_1',
|
||||
'roles.name_1',
|
||||
'users.email_1',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -21,7 +21,6 @@ const scheduleSchema: Schema<IScheduleDocument> = new Schema(
|
|||
id: {
|
||||
type: String,
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
user: {
|
||||
type: Schema.Types.ObjectId,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ const scheduleRunSchema: Schema<IScheduleRunDocument> = new Schema(
|
|||
type: Schema.Types.ObjectId,
|
||||
ref: 'User',
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
tenantId: {
|
||||
type: String,
|
||||
|
|
|
|||
|
|
@ -54,7 +54,6 @@ const shareSchema: Schema<ISharedLink> = new Schema(
|
|||
},
|
||||
user: {
|
||||
type: String,
|
||||
index: true,
|
||||
},
|
||||
messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }],
|
||||
shareId: {
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@ const skillSchema: Schema<ISkillDocument> = new Schema(
|
|||
name: {
|
||||
type: String,
|
||||
required: true,
|
||||
index: true,
|
||||
maxlength: [SKILL_NAME_MAX_LENGTH, `Name cannot exceed ${SKILL_NAME_MAX_LENGTH} characters`],
|
||||
validate: {
|
||||
validator: function (value: string): boolean {
|
||||
|
|
@ -161,13 +160,11 @@ const skillSchema: Schema<ISkillDocument> = new Schema(
|
|||
category: {
|
||||
type: String,
|
||||
default: '',
|
||||
index: true,
|
||||
},
|
||||
author: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'User',
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
authorName: {
|
||||
type: String,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ const skillFileSchema: Schema<ISkillFileDocument> = new Schema(
|
|||
type: Schema.Types.ObjectId,
|
||||
ref: 'Skill',
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
relativePath: {
|
||||
type: String,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ const skillSyncCredentialSchema: Schema<ISkillSyncCredentialDocument> = new Sche
|
|||
type: String,
|
||||
enum: ['github'],
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
credentialKey: {
|
||||
type: String,
|
||||
|
|
|
|||
|
|
@ -77,7 +77,6 @@ const skillSyncStatusSchema: Schema<ISkillSyncStatusDocument> = new Schema(
|
|||
type: String,
|
||||
enum: ['github'],
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
sourceId: {
|
||||
type: String,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import type { ISkillUploadIntentDocument } from '~/types/skillDeletion';
|
|||
const skillUploadIntentSchema: Schema<ISkillUploadIntentDocument> =
|
||||
new Schema<ISkillUploadIntentDocument>(
|
||||
{
|
||||
skillId: { type: Schema.Types.ObjectId, required: true, index: true },
|
||||
skillId: { type: Schema.Types.ObjectId, required: true },
|
||||
fileId: { type: String, required: true, unique: true, index: true },
|
||||
source: { type: String, required: true },
|
||||
storageKey: { type: String, required: true },
|
||||
|
|
@ -27,7 +27,6 @@ const skillUploadIntentSchema: Schema<ISkillUploadIntentDocument> =
|
|||
{ timestamps: true },
|
||||
);
|
||||
|
||||
skillUploadIntentSchema.index({ skillId: 1, state: 1 });
|
||||
skillUploadIntentSchema.index({ skillId: 1, state: 1, cleanedAt: 1 });
|
||||
skillUploadIntentSchema.index({ state: 1, lastCleanupAttemptAt: 1, createdAt: 1 });
|
||||
// The committed cleanup branch must not scan healthy live uploads just to preserve ordering.
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
|
|||
// Zero is reserved for a staging row that is visible before sequence allocation.
|
||||
laneSequence: { type: Number, required: true, min: 0 },
|
||||
envelope: { type: Schema.Types.Mixed, required: true },
|
||||
user: { type: Schema.Types.ObjectId, ref: 'User', required: true, index: true },
|
||||
user: { type: Schema.Types.ObjectId, ref: 'User', required: true },
|
||||
tenantId: { type: String, index: true },
|
||||
status: {
|
||||
type: String,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue