From fe2cd45c43cd85e5489c62d3334ce482fb33276a Mon Sep 17 00:00:00 2001 From: Julien Millau Date: Mon, 2 Nov 2026 13:50:58 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=82=EF=B8=8F=20perf:=20Drop=20Schema?= =?UTF-8?q?=20Indexes=20That=20a=20Wider=20Index=20Already=20Covers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 31 indexes are a key prefix of another index on the same collection that holds every document, so they serve no read the wider index cannot, yet every write maintains them. Remove the declarations and add a spec that fails when one is declared again, and that pins the eight field-level indexes kept because their name collides with a legacy unique index: that collision is what surfaces the tenant-index migration hint. Partial, sparse, hidden and collated indexes never count as a cover, and prefixes whose only cover is multikey (users/groups tenantId) stay. Co-Authored-By: Claude Fable 5.1 --- packages/data-schemas/src/schema/aclEntry.ts | 2 - packages/data-schemas/src/schema/agent.ts | 1 - .../data-schemas/src/schema/agentApiKey.ts | 1 - .../data-schemas/src/schema/agentCategory.ts | 2 - packages/data-schemas/src/schema/assistant.ts | 1 - .../data-schemas/src/schema/chatProject.ts | 1 - packages/data-schemas/src/schema/config.ts | 2 - .../src/schema/conversationTag.ts | 1 - packages/data-schemas/src/schema/convo.ts | 2 - packages/data-schemas/src/schema/favorite.ts | 1 - packages/data-schemas/src/schema/memory.ts | 1 - packages/data-schemas/src/schema/message.ts | 3 - .../data-schemas/src/schema/queuedTurn.ts | 2 +- .../src/schema/redundantIndexes.spec.ts | 117 ++++++++++++++++++ packages/data-schemas/src/schema/schedule.ts | 1 - .../data-schemas/src/schema/scheduleRun.ts | 1 - packages/data-schemas/src/schema/share.ts | 1 - packages/data-schemas/src/schema/skill.ts | 3 - packages/data-schemas/src/schema/skillFile.ts | 1 - .../src/schema/skillSyncCredential.ts | 1 - .../src/schema/skillSyncStatus.ts | 1 - .../src/schema/skillUploadIntent.ts | 3 +- .../src/schema/triggerDelivery.ts | 2 +- 23 files changed, 120 insertions(+), 31 deletions(-) create mode 100644 packages/data-schemas/src/schema/redundantIndexes.spec.ts diff --git a/packages/data-schemas/src/schema/aclEntry.ts b/packages/data-schemas/src/schema/aclEntry.ts index 562d4e5b21..98bb7e37f7 100644 --- a/packages/data-schemas/src/schema/aclEntry.ts +++ b/packages/data-schemas/src/schema/aclEntry.ts @@ -16,7 +16,6 @@ const aclEntrySchema: Schema = new Schema( required: function (this: IAclEntry) { return this.principalType !== PrincipalType.PUBLIC; }, - index: true, }, principalModel: { type: String, @@ -33,7 +32,6 @@ const aclEntrySchema: Schema = new Schema( resourceId: { type: Schema.Types.ObjectId, required: true, - index: true, }, permBits: { type: Number, diff --git a/packages/data-schemas/src/schema/agent.ts b/packages/data-schemas/src/schema/agent.ts index be6e471585..e394e19b2e 100644 --- a/packages/data-schemas/src/schema/agent.ts +++ b/packages/data-schemas/src/schema/agent.ts @@ -186,7 +186,6 @@ const agentSchema: Schema = new Schema( }, tenantId: { type: String, - index: true, }, }, { diff --git a/packages/data-schemas/src/schema/agentApiKey.ts b/packages/data-schemas/src/schema/agentApiKey.ts index 50334f5f5c..40b2251306 100644 --- a/packages/data-schemas/src/schema/agentApiKey.ts +++ b/packages/data-schemas/src/schema/agentApiKey.ts @@ -18,7 +18,6 @@ const agentApiKeySchema: Schema = new Schema( type: mongoose.Schema.Types.ObjectId, ref: 'User', required: true, - index: true, }, name: { type: String, diff --git a/packages/data-schemas/src/schema/agentCategory.ts b/packages/data-schemas/src/schema/agentCategory.ts index e8d611fb05..b17b704679 100644 --- a/packages/data-schemas/src/schema/agentCategory.ts +++ b/packages/data-schemas/src/schema/agentCategory.ts @@ -23,12 +23,10 @@ const agentCategorySchema: Schema = new Schema( order: { type: Number, default: 0, - index: true, }, isActive: { type: Boolean, default: true, - index: true, }, custom: { type: Boolean, diff --git a/packages/data-schemas/src/schema/assistant.ts b/packages/data-schemas/src/schema/assistant.ts index 44a108c052..1a7cb216d6 100644 --- a/packages/data-schemas/src/schema/assistant.ts +++ b/packages/data-schemas/src/schema/assistant.ts @@ -35,7 +35,6 @@ const assistantSchema: Schema = new Schema( }, tenantId: { type: String, - index: true, }, }, { diff --git a/packages/data-schemas/src/schema/chatProject.ts b/packages/data-schemas/src/schema/chatProject.ts index 7c196f002d..a72ab1486d 100644 --- a/packages/data-schemas/src/schema/chatProject.ts +++ b/packages/data-schemas/src/schema/chatProject.ts @@ -37,7 +37,6 @@ const chatProjectSchema: Schema = new Schema = new Schema( type: String, enum: Object.values(PrincipalType), required: true, - index: true, }, principalId: { type: String, @@ -24,7 +23,6 @@ const configSchema: Schema = new Schema( priority: { type: Number, required: true, - index: true, }, overrides: { type: Schema.Types.Mixed, diff --git a/packages/data-schemas/src/schema/conversationTag.ts b/packages/data-schemas/src/schema/conversationTag.ts index f3d0c8c0ef..356fa6252a 100644 --- a/packages/data-schemas/src/schema/conversationTag.ts +++ b/packages/data-schemas/src/schema/conversationTag.ts @@ -13,7 +13,6 @@ const conversationTag: Schema = new Schema( { tag: { type: String, - index: true, }, user: { type: String, diff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.ts index 3a24aae0f5..3ff9107152 100644 --- a/packages/data-schemas/src/schema/convo.ts +++ b/packages/data-schemas/src/schema/convo.ts @@ -34,7 +34,6 @@ const convoSchema: Schema = new Schema( titleRevision: { type: Number }, user: { type: String, - index: true, meiliIndex: true, }, messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }], @@ -402,7 +401,6 @@ const convoSchema: Schema = new Schema( }, tenantId: { type: String, - index: true, }, pinned: { type: Boolean, diff --git a/packages/data-schemas/src/schema/favorite.ts b/packages/data-schemas/src/schema/favorite.ts index 7efafec438..154c39dc5b 100644 --- a/packages/data-schemas/src/schema/favorite.ts +++ b/packages/data-schemas/src/schema/favorite.ts @@ -8,7 +8,6 @@ const toolFavoriteSchema: Schema = new Schema( type: Schema.Types.ObjectId, ref: 'User', required: true, - index: true, }, itemType: { type: String, diff --git a/packages/data-schemas/src/schema/memory.ts b/packages/data-schemas/src/schema/memory.ts index 346489b8de..1182f6f394 100644 --- a/packages/data-schemas/src/schema/memory.ts +++ b/packages/data-schemas/src/schema/memory.ts @@ -11,7 +11,6 @@ const MemoryEntrySchema: Schema = new Schema({ userId: { type: Schema.Types.ObjectId, ref: 'User', - index: true, required: true, }, key: { diff --git a/packages/data-schemas/src/schema/message.ts b/packages/data-schemas/src/schema/message.ts index fe6c19de07..fc05de0fb5 100644 --- a/packages/data-schemas/src/schema/message.ts +++ b/packages/data-schemas/src/schema/message.ts @@ -12,13 +12,11 @@ const messageSchema: Schema = new Schema( }, conversationId: { type: String, - index: true, required: true, meiliIndex: true, }, user: { type: String, - index: true, required: true, default: null, meiliIndex: true, @@ -327,7 +325,6 @@ const messageSchema: Schema = new Schema( }, tenantId: { type: String, - index: true, }, }, { timestamps: true }, diff --git a/packages/data-schemas/src/schema/queuedTurn.ts b/packages/data-schemas/src/schema/queuedTurn.ts index 314d51398d..d43c9bfd89 100644 --- a/packages/data-schemas/src/schema/queuedTurn.ts +++ b/packages/data-schemas/src/schema/queuedTurn.ts @@ -64,7 +64,7 @@ const queuedTurnSchema: Schema = new Schema( required: true, index: true, }, - tenantId: { type: String, index: true }, + tenantId: { type: String }, conversationId: { type: String, required: true, maxlength: 256 }, agentId: { type: String, required: true, maxlength: 256 }, parentMessageId: { diff --git a/packages/data-schemas/src/schema/redundantIndexes.spec.ts b/packages/data-schemas/src/schema/redundantIndexes.spec.ts new file mode 100644 index 0000000000..bf687a307c --- /dev/null +++ b/packages/data-schemas/src/schema/redundantIndexes.spec.ts @@ -0,0 +1,117 @@ +import mongoose from 'mongoose'; +import type { IndexDefinition, IndexOptions, Schema } from 'mongoose'; +import { SUPERSEDED_INDEXES } from '../migrations/tenantIndexes'; +import { createModels } from '../models'; + +/** An index that holds an entry for every document and answers simple-collation reads. */ +const holdsEveryDocument = (options: IndexOptions) => + !options.sparse && + !options.hidden && + options.partialFilterExpression == null && + options.collation == null; + +/** An index that only costs writes once another index serves its reads. */ +const isPlain = (options: IndexOptions) => + !options.unique && options.expireAfterSeconds == null && holdsEveryDocument(options); + +/** The B-tree direction of a key; text, 2dsphere, hashed and wildcard keys serve other reads. */ +const direction = (value: unknown): 1 | -1 | undefined => { + const dir = Number(value); + return dir === 1 || dir === -1 ? dir : undefined; +}; + +/** The name MongoDB gives an index declared without one. */ +const indexName = (keys: IndexDefinition) => + Object.entries(keys) + .map(([field, dir]) => `${field}_${dir}`) + .join('_'); + +/** True when `field`, or any parent of it, is an array, which makes an index on it multikey. */ +const isArrayPath = (schema: Schema, field: string) => + field + .split('.') + .some((_, i, parts) => schema.path(parts.slice(0, i + 1).join('.'))?.instance === 'Array'); + +/** + * True when `long` serves every read `short` does: `short`'s keys lead `long` in order, all in + * the same or all in the opposite direction (a scan runs either way), every key of `long` is a + * B-tree key, and no trailing key is multikey, which would repeat each document once per array + * element in a scan over the prefix. + */ +function covers(schema: Schema, long: IndexDefinition, short: IndexDefinition): boolean { + const shortKeys = Object.entries(short); + const longKeys = Object.entries(long); + const first = direction(shortKeys[0]?.[1]); + const lead = direction(longKeys[0]?.[1]); + if (first === undefined || lead === undefined || shortKeys.length > longKeys.length) { + return false; + } + const sign = first * lead; + return ( + longKeys.every(([, dir]) => direction(dir) !== undefined) && + shortKeys.every(([field, dir], i) => { + const expected = direction(dir); + return ( + expected !== undefined && + longKeys[i][0] === field && + direction(longKeys[i][1]) === expected * sign + ); + }) && + !longKeys.slice(shortKeys.length).some(([field]) => isArrayPath(schema, field)) + ); +} + +/** Every schema's declared indexes with their declaration order, keyed by collection. */ +const declaredIndexes = () => + Object.values(createModels(mongoose)).map(({ schema, collection }) => ({ + schema, + collection: collection.collectionName, + legacy: SUPERSEDED_INDEXES[collection.collectionName] ?? [], + entries: schema.indexes().map(([keys, options], at) => ({ keys, options, at })), + })); + +/** + * A plain index whose keys lead another index that holds every document costs a write on each + * insert and update for reads the wider index already serves; the wider keys make a scan over + * the prefix alone touch more pages, which is the trade this test accepts. A field-level index + * that shares its name with a legacy unique index stays: colliding with that index on startup + * is what surfaces the tenant-index migration hint. + */ +describe('schema indexes', () => { + it('declares no plain index that another index already serves', () => { + const redundant = declaredIndexes().flatMap(({ schema, collection, legacy, entries }) => + entries + .filter(({ keys, options }) => isPlain(options) && !legacy.includes(indexName(keys))) + .filter(({ keys, at }) => + entries.some( + (other) => + other.at !== at && + holdsEveryDocument(other.options) && + covers(schema, other.keys, keys) && + /** Of two identical declarations, only the later one is redundant. */ + (Object.keys(other.keys).length > Object.keys(keys).length || other.at < at), + ), + ) + .map(({ keys }) => `${collection}: ${JSON.stringify(keys)}`), + ); + expect(redundant).toEqual([]); + }); + + it('keeps the field-level indexes whose name a legacy unique index shares', () => { + const tripwires = declaredIndexes().flatMap(({ collection, legacy, entries }) => + entries + .filter(({ keys, options }) => isPlain(options) && legacy.includes(indexName(keys))) + .map(({ keys }) => `${collection}.${indexName(keys)}`), + ); + expect(tripwires.sort()).toEqual([ + 'accessroles.accessRoleId_1', + 'agentcategories.value_1', + 'conversations.conversationId_1', + 'mcpservers.serverName_1', + 'messages.messageId_1', + 'presets.presetId_1', + 'roles.name_1', + 'users.email_1', + ]); + }); +}); diff --git a/packages/data-schemas/src/schema/schedule.ts b/packages/data-schemas/src/schema/schedule.ts index 7adf6cd9c7..3e2df4f1d1 100644 --- a/packages/data-schemas/src/schema/schedule.ts +++ b/packages/data-schemas/src/schema/schedule.ts @@ -21,7 +21,6 @@ const scheduleSchema: Schema = new Schema( id: { type: String, required: true, - index: true, }, user: { type: Schema.Types.ObjectId, diff --git a/packages/data-schemas/src/schema/scheduleRun.ts b/packages/data-schemas/src/schema/scheduleRun.ts index 5dc918ee4b..2df5ac5035 100644 --- a/packages/data-schemas/src/schema/scheduleRun.ts +++ b/packages/data-schemas/src/schema/scheduleRun.ts @@ -14,7 +14,6 @@ const scheduleRunSchema: Schema = new Schema( type: Schema.Types.ObjectId, ref: 'User', required: true, - index: true, }, tenantId: { type: String, diff --git a/packages/data-schemas/src/schema/share.ts b/packages/data-schemas/src/schema/share.ts index bacc3cd7e1..3303132569 100644 --- a/packages/data-schemas/src/schema/share.ts +++ b/packages/data-schemas/src/schema/share.ts @@ -54,7 +54,6 @@ const shareSchema: Schema = new Schema( }, user: { type: String, - index: true, }, messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }], shareId: { diff --git a/packages/data-schemas/src/schema/skill.ts b/packages/data-schemas/src/schema/skill.ts index 7a18f369fe..ffb8931459 100644 --- a/packages/data-schemas/src/schema/skill.ts +++ b/packages/data-schemas/src/schema/skill.ts @@ -51,7 +51,6 @@ const skillSchema: Schema = new Schema( name: { type: String, required: true, - index: true, maxlength: [SKILL_NAME_MAX_LENGTH, `Name cannot exceed ${SKILL_NAME_MAX_LENGTH} characters`], validate: { validator: function (value: string): boolean { @@ -161,13 +160,11 @@ const skillSchema: Schema = new Schema( category: { type: String, default: '', - index: true, }, author: { type: Schema.Types.ObjectId, ref: 'User', required: true, - index: true, }, authorName: { type: String, diff --git a/packages/data-schemas/src/schema/skillFile.ts b/packages/data-schemas/src/schema/skillFile.ts index e4c86c066b..ee7d14c428 100644 --- a/packages/data-schemas/src/schema/skillFile.ts +++ b/packages/data-schemas/src/schema/skillFile.ts @@ -14,7 +14,6 @@ const skillFileSchema: Schema = new Schema( type: Schema.Types.ObjectId, ref: 'Skill', required: true, - index: true, }, relativePath: { type: String, diff --git a/packages/data-schemas/src/schema/skillSyncCredential.ts b/packages/data-schemas/src/schema/skillSyncCredential.ts index 3ff90a1d79..833f3f06d5 100644 --- a/packages/data-schemas/src/schema/skillSyncCredential.ts +++ b/packages/data-schemas/src/schema/skillSyncCredential.ts @@ -7,7 +7,6 @@ const skillSyncCredentialSchema: Schema = new Sche type: String, enum: ['github'], required: true, - index: true, }, credentialKey: { type: String, diff --git a/packages/data-schemas/src/schema/skillSyncStatus.ts b/packages/data-schemas/src/schema/skillSyncStatus.ts index 6e6e3828dd..0e24865b83 100644 --- a/packages/data-schemas/src/schema/skillSyncStatus.ts +++ b/packages/data-schemas/src/schema/skillSyncStatus.ts @@ -77,7 +77,6 @@ const skillSyncStatusSchema: Schema = new Schema( type: String, enum: ['github'], required: true, - index: true, }, sourceId: { type: String, diff --git a/packages/data-schemas/src/schema/skillUploadIntent.ts b/packages/data-schemas/src/schema/skillUploadIntent.ts index 37ce30e1e0..e7746c2852 100644 --- a/packages/data-schemas/src/schema/skillUploadIntent.ts +++ b/packages/data-schemas/src/schema/skillUploadIntent.ts @@ -4,7 +4,7 @@ import type { ISkillUploadIntentDocument } from '~/types/skillDeletion'; const skillUploadIntentSchema: Schema = new Schema( { - skillId: { type: Schema.Types.ObjectId, required: true, index: true }, + skillId: { type: Schema.Types.ObjectId, required: true }, fileId: { type: String, required: true, unique: true, index: true }, source: { type: String, required: true }, storageKey: { type: String, required: true }, @@ -27,7 +27,6 @@ const skillUploadIntentSchema: Schema = { timestamps: true }, ); -skillUploadIntentSchema.index({ skillId: 1, state: 1 }); skillUploadIntentSchema.index({ skillId: 1, state: 1, cleanedAt: 1 }); skillUploadIntentSchema.index({ state: 1, lastCleanupAttemptAt: 1, createdAt: 1 }); // The committed cleanup branch must not scan healthy live uploads just to preserve ordering. diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts index 54e8306c39..e81d3f2a50 100644 --- a/packages/data-schemas/src/schema/triggerDelivery.ts +++ b/packages/data-schemas/src/schema/triggerDelivery.ts @@ -121,7 +121,7 @@ const triggerDeliverySchema: Schema = new Schema( // Zero is reserved for a staging row that is visible before sequence allocation. laneSequence: { type: Number, required: true, min: 0 }, envelope: { type: Schema.Types.Mixed, required: true }, - user: { type: Schema.Types.ObjectId, ref: 'User', required: true, index: true }, + user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, tenantId: { type: String, index: true }, status: { type: String,