diff --git a/packages/data-schemas/src/schema/aclEntry.ts b/packages/data-schemas/src/schema/aclEntry.ts index 562d4e5b21..98bb7e37f7 100644 --- a/packages/data-schemas/src/schema/aclEntry.ts +++ b/packages/data-schemas/src/schema/aclEntry.ts @@ -16,7 +16,6 @@ const aclEntrySchema: Schema = new Schema( required: function (this: IAclEntry) { return this.principalType !== PrincipalType.PUBLIC; }, - index: true, }, principalModel: { type: String, @@ -33,7 +32,6 @@ const aclEntrySchema: Schema = new Schema( resourceId: { type: Schema.Types.ObjectId, required: true, - index: true, }, permBits: { type: Number, diff --git a/packages/data-schemas/src/schema/agent.ts b/packages/data-schemas/src/schema/agent.ts index be6e471585..e394e19b2e 100644 --- a/packages/data-schemas/src/schema/agent.ts +++ b/packages/data-schemas/src/schema/agent.ts @@ -186,7 +186,6 @@ const agentSchema: Schema = new Schema( }, tenantId: { type: String, - index: true, }, }, { diff --git a/packages/data-schemas/src/schema/agentApiKey.ts b/packages/data-schemas/src/schema/agentApiKey.ts index 50334f5f5c..40b2251306 100644 --- a/packages/data-schemas/src/schema/agentApiKey.ts +++ b/packages/data-schemas/src/schema/agentApiKey.ts @@ -18,7 +18,6 @@ const agentApiKeySchema: Schema = new Schema( type: mongoose.Schema.Types.ObjectId, ref: 'User', required: true, - index: true, }, name: { type: String, diff --git a/packages/data-schemas/src/schema/agentCategory.ts b/packages/data-schemas/src/schema/agentCategory.ts index e8d611fb05..b17b704679 100644 --- a/packages/data-schemas/src/schema/agentCategory.ts +++ b/packages/data-schemas/src/schema/agentCategory.ts @@ -23,12 +23,10 @@ const agentCategorySchema: Schema = new Schema( order: { type: Number, default: 0, - index: true, }, isActive: { type: Boolean, default: true, - index: true, }, custom: { type: Boolean, diff --git a/packages/data-schemas/src/schema/assistant.ts b/packages/data-schemas/src/schema/assistant.ts index 44a108c052..1a7cb216d6 100644 --- a/packages/data-schemas/src/schema/assistant.ts +++ b/packages/data-schemas/src/schema/assistant.ts @@ -35,7 +35,6 @@ const assistantSchema: Schema = new Schema( }, tenantId: { type: String, - index: true, }, }, { diff --git a/packages/data-schemas/src/schema/chatProject.ts b/packages/data-schemas/src/schema/chatProject.ts index 7c196f002d..a72ab1486d 100644 --- a/packages/data-schemas/src/schema/chatProject.ts +++ b/packages/data-schemas/src/schema/chatProject.ts @@ -37,7 +37,6 @@ const chatProjectSchema: Schema = new Schema = new Schema( type: String, enum: Object.values(PrincipalType), required: true, - index: true, }, principalId: { type: String, @@ -24,7 +23,6 @@ const configSchema: Schema = new Schema( priority: { type: Number, required: true, - index: true, }, overrides: { type: Schema.Types.Mixed, diff --git a/packages/data-schemas/src/schema/conversationTag.ts b/packages/data-schemas/src/schema/conversationTag.ts index f3d0c8c0ef..356fa6252a 100644 --- a/packages/data-schemas/src/schema/conversationTag.ts +++ b/packages/data-schemas/src/schema/conversationTag.ts @@ -13,7 +13,6 @@ const conversationTag: Schema = new Schema( { tag: { type: String, - index: true, }, user: { type: String, diff --git a/packages/data-schemas/src/schema/convo.ts b/packages/data-schemas/src/schema/convo.ts index 3a24aae0f5..3ff9107152 100644 --- a/packages/data-schemas/src/schema/convo.ts +++ b/packages/data-schemas/src/schema/convo.ts @@ -34,7 +34,6 @@ const convoSchema: Schema = new Schema( titleRevision: { type: Number }, user: { type: String, - index: true, meiliIndex: true, }, messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }], @@ -402,7 +401,6 @@ const convoSchema: Schema = new Schema( }, tenantId: { type: String, - index: true, }, pinned: { type: Boolean, diff --git a/packages/data-schemas/src/schema/favorite.ts b/packages/data-schemas/src/schema/favorite.ts index 7efafec438..154c39dc5b 100644 --- a/packages/data-schemas/src/schema/favorite.ts +++ b/packages/data-schemas/src/schema/favorite.ts @@ -8,7 +8,6 @@ const toolFavoriteSchema: Schema = new Schema( type: Schema.Types.ObjectId, ref: 'User', required: true, - index: true, }, itemType: { type: String, diff --git a/packages/data-schemas/src/schema/memory.ts b/packages/data-schemas/src/schema/memory.ts index 346489b8de..1182f6f394 100644 --- a/packages/data-schemas/src/schema/memory.ts +++ b/packages/data-schemas/src/schema/memory.ts @@ -11,7 +11,6 @@ const MemoryEntrySchema: Schema = new Schema({ userId: { type: Schema.Types.ObjectId, ref: 'User', - index: true, required: true, }, key: { diff --git a/packages/data-schemas/src/schema/message.ts b/packages/data-schemas/src/schema/message.ts index fe6c19de07..fc05de0fb5 100644 --- a/packages/data-schemas/src/schema/message.ts +++ b/packages/data-schemas/src/schema/message.ts @@ -12,13 +12,11 @@ const messageSchema: Schema = new Schema( }, conversationId: { type: String, - index: true, required: true, meiliIndex: true, }, user: { type: String, - index: true, required: true, default: null, meiliIndex: true, @@ -327,7 +325,6 @@ const messageSchema: Schema = new Schema( }, tenantId: { type: String, - index: true, }, }, { timestamps: true }, diff --git a/packages/data-schemas/src/schema/queuedTurn.ts b/packages/data-schemas/src/schema/queuedTurn.ts index 314d51398d..d43c9bfd89 100644 --- a/packages/data-schemas/src/schema/queuedTurn.ts +++ b/packages/data-schemas/src/schema/queuedTurn.ts @@ -64,7 +64,7 @@ const queuedTurnSchema: Schema = new Schema( required: true, index: true, }, - tenantId: { type: String, index: true }, + tenantId: { type: String }, conversationId: { type: String, required: true, maxlength: 256 }, agentId: { type: String, required: true, maxlength: 256 }, parentMessageId: { diff --git a/packages/data-schemas/src/schema/redundantIndexes.spec.ts b/packages/data-schemas/src/schema/redundantIndexes.spec.ts new file mode 100644 index 0000000000..bf687a307c --- /dev/null +++ b/packages/data-schemas/src/schema/redundantIndexes.spec.ts @@ -0,0 +1,117 @@ +import mongoose from 'mongoose'; +import type { IndexDefinition, IndexOptions, Schema } from 'mongoose'; +import { SUPERSEDED_INDEXES } from '../migrations/tenantIndexes'; +import { createModels } from '../models'; + +/** An index that holds an entry for every document and answers simple-collation reads. */ +const holdsEveryDocument = (options: IndexOptions) => + !options.sparse && + !options.hidden && + options.partialFilterExpression == null && + options.collation == null; + +/** An index that only costs writes once another index serves its reads. */ +const isPlain = (options: IndexOptions) => + !options.unique && options.expireAfterSeconds == null && holdsEveryDocument(options); + +/** The B-tree direction of a key; text, 2dsphere, hashed and wildcard keys serve other reads. */ +const direction = (value: unknown): 1 | -1 | undefined => { + const dir = Number(value); + return dir === 1 || dir === -1 ? dir : undefined; +}; + +/** The name MongoDB gives an index declared without one. */ +const indexName = (keys: IndexDefinition) => + Object.entries(keys) + .map(([field, dir]) => `${field}_${dir}`) + .join('_'); + +/** True when `field`, or any parent of it, is an array, which makes an index on it multikey. */ +const isArrayPath = (schema: Schema, field: string) => + field + .split('.') + .some((_, i, parts) => schema.path(parts.slice(0, i + 1).join('.'))?.instance === 'Array'); + +/** + * True when `long` serves every read `short` does: `short`'s keys lead `long` in order, all in + * the same or all in the opposite direction (a scan runs either way), every key of `long` is a + * B-tree key, and no trailing key is multikey, which would repeat each document once per array + * element in a scan over the prefix. + */ +function covers(schema: Schema, long: IndexDefinition, short: IndexDefinition): boolean { + const shortKeys = Object.entries(short); + const longKeys = Object.entries(long); + const first = direction(shortKeys[0]?.[1]); + const lead = direction(longKeys[0]?.[1]); + if (first === undefined || lead === undefined || shortKeys.length > longKeys.length) { + return false; + } + const sign = first * lead; + return ( + longKeys.every(([, dir]) => direction(dir) !== undefined) && + shortKeys.every(([field, dir], i) => { + const expected = direction(dir); + return ( + expected !== undefined && + longKeys[i][0] === field && + direction(longKeys[i][1]) === expected * sign + ); + }) && + !longKeys.slice(shortKeys.length).some(([field]) => isArrayPath(schema, field)) + ); +} + +/** Every schema's declared indexes with their declaration order, keyed by collection. */ +const declaredIndexes = () => + Object.values(createModels(mongoose)).map(({ schema, collection }) => ({ + schema, + collection: collection.collectionName, + legacy: SUPERSEDED_INDEXES[collection.collectionName] ?? [], + entries: schema.indexes().map(([keys, options], at) => ({ keys, options, at })), + })); + +/** + * A plain index whose keys lead another index that holds every document costs a write on each + * insert and update for reads the wider index already serves; the wider keys make a scan over + * the prefix alone touch more pages, which is the trade this test accepts. A field-level index + * that shares its name with a legacy unique index stays: colliding with that index on startup + * is what surfaces the tenant-index migration hint. + */ +describe('schema indexes', () => { + it('declares no plain index that another index already serves', () => { + const redundant = declaredIndexes().flatMap(({ schema, collection, legacy, entries }) => + entries + .filter(({ keys, options }) => isPlain(options) && !legacy.includes(indexName(keys))) + .filter(({ keys, at }) => + entries.some( + (other) => + other.at !== at && + holdsEveryDocument(other.options) && + covers(schema, other.keys, keys) && + /** Of two identical declarations, only the later one is redundant. */ + (Object.keys(other.keys).length > Object.keys(keys).length || other.at < at), + ), + ) + .map(({ keys }) => `${collection}: ${JSON.stringify(keys)}`), + ); + expect(redundant).toEqual([]); + }); + + it('keeps the field-level indexes whose name a legacy unique index shares', () => { + const tripwires = declaredIndexes().flatMap(({ collection, legacy, entries }) => + entries + .filter(({ keys, options }) => isPlain(options) && legacy.includes(indexName(keys))) + .map(({ keys }) => `${collection}.${indexName(keys)}`), + ); + expect(tripwires.sort()).toEqual([ + 'accessroles.accessRoleId_1', + 'agentcategories.value_1', + 'conversations.conversationId_1', + 'mcpservers.serverName_1', + 'messages.messageId_1', + 'presets.presetId_1', + 'roles.name_1', + 'users.email_1', + ]); + }); +}); diff --git a/packages/data-schemas/src/schema/schedule.ts b/packages/data-schemas/src/schema/schedule.ts index 7adf6cd9c7..3e2df4f1d1 100644 --- a/packages/data-schemas/src/schema/schedule.ts +++ b/packages/data-schemas/src/schema/schedule.ts @@ -21,7 +21,6 @@ const scheduleSchema: Schema = new Schema( id: { type: String, required: true, - index: true, }, user: { type: Schema.Types.ObjectId, diff --git a/packages/data-schemas/src/schema/scheduleRun.ts b/packages/data-schemas/src/schema/scheduleRun.ts index 5dc918ee4b..2df5ac5035 100644 --- a/packages/data-schemas/src/schema/scheduleRun.ts +++ b/packages/data-schemas/src/schema/scheduleRun.ts @@ -14,7 +14,6 @@ const scheduleRunSchema: Schema = new Schema( type: Schema.Types.ObjectId, ref: 'User', required: true, - index: true, }, tenantId: { type: String, diff --git a/packages/data-schemas/src/schema/share.ts b/packages/data-schemas/src/schema/share.ts index bacc3cd7e1..3303132569 100644 --- a/packages/data-schemas/src/schema/share.ts +++ b/packages/data-schemas/src/schema/share.ts @@ -54,7 +54,6 @@ const shareSchema: Schema = new Schema( }, user: { type: String, - index: true, }, messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }], shareId: { diff --git a/packages/data-schemas/src/schema/skill.ts b/packages/data-schemas/src/schema/skill.ts index 7a18f369fe..ffb8931459 100644 --- a/packages/data-schemas/src/schema/skill.ts +++ b/packages/data-schemas/src/schema/skill.ts @@ -51,7 +51,6 @@ const skillSchema: Schema = new Schema( name: { type: String, required: true, - index: true, maxlength: [SKILL_NAME_MAX_LENGTH, `Name cannot exceed ${SKILL_NAME_MAX_LENGTH} characters`], validate: { validator: function (value: string): boolean { @@ -161,13 +160,11 @@ const skillSchema: Schema = new Schema( category: { type: String, default: '', - index: true, }, author: { type: Schema.Types.ObjectId, ref: 'User', required: true, - index: true, }, authorName: { type: String, diff --git a/packages/data-schemas/src/schema/skillFile.ts b/packages/data-schemas/src/schema/skillFile.ts index e4c86c066b..ee7d14c428 100644 --- a/packages/data-schemas/src/schema/skillFile.ts +++ b/packages/data-schemas/src/schema/skillFile.ts @@ -14,7 +14,6 @@ const skillFileSchema: Schema = new Schema( type: Schema.Types.ObjectId, ref: 'Skill', required: true, - index: true, }, relativePath: { type: String, diff --git a/packages/data-schemas/src/schema/skillSyncCredential.ts b/packages/data-schemas/src/schema/skillSyncCredential.ts index 3ff90a1d79..833f3f06d5 100644 --- a/packages/data-schemas/src/schema/skillSyncCredential.ts +++ b/packages/data-schemas/src/schema/skillSyncCredential.ts @@ -7,7 +7,6 @@ const skillSyncCredentialSchema: Schema = new Sche type: String, enum: ['github'], required: true, - index: true, }, credentialKey: { type: String, diff --git a/packages/data-schemas/src/schema/skillSyncStatus.ts b/packages/data-schemas/src/schema/skillSyncStatus.ts index 6e6e3828dd..0e24865b83 100644 --- a/packages/data-schemas/src/schema/skillSyncStatus.ts +++ b/packages/data-schemas/src/schema/skillSyncStatus.ts @@ -77,7 +77,6 @@ const skillSyncStatusSchema: Schema = new Schema( type: String, enum: ['github'], required: true, - index: true, }, sourceId: { type: String, diff --git a/packages/data-schemas/src/schema/skillUploadIntent.ts b/packages/data-schemas/src/schema/skillUploadIntent.ts index 37ce30e1e0..e7746c2852 100644 --- a/packages/data-schemas/src/schema/skillUploadIntent.ts +++ b/packages/data-schemas/src/schema/skillUploadIntent.ts @@ -4,7 +4,7 @@ import type { ISkillUploadIntentDocument } from '~/types/skillDeletion'; const skillUploadIntentSchema: Schema = new Schema( { - skillId: { type: Schema.Types.ObjectId, required: true, index: true }, + skillId: { type: Schema.Types.ObjectId, required: true }, fileId: { type: String, required: true, unique: true, index: true }, source: { type: String, required: true }, storageKey: { type: String, required: true }, @@ -27,7 +27,6 @@ const skillUploadIntentSchema: Schema = { timestamps: true }, ); -skillUploadIntentSchema.index({ skillId: 1, state: 1 }); skillUploadIntentSchema.index({ skillId: 1, state: 1, cleanedAt: 1 }); skillUploadIntentSchema.index({ state: 1, lastCleanupAttemptAt: 1, createdAt: 1 }); // The committed cleanup branch must not scan healthy live uploads just to preserve ordering. diff --git a/packages/data-schemas/src/schema/triggerDelivery.ts b/packages/data-schemas/src/schema/triggerDelivery.ts index 54e8306c39..e81d3f2a50 100644 --- a/packages/data-schemas/src/schema/triggerDelivery.ts +++ b/packages/data-schemas/src/schema/triggerDelivery.ts @@ -121,7 +121,7 @@ const triggerDeliverySchema: Schema = new Schema( // Zero is reserved for a staging row that is visible before sequence allocation. laneSequence: { type: Number, required: true, min: 0 }, envelope: { type: Schema.Types.Mixed, required: true }, - user: { type: Schema.Types.ObjectId, ref: 'User', required: true, index: true }, + user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, tenantId: { type: String, index: true }, status: { type: String,