🗂️ perf: Drop Schema Indexes That a Wider Index Already Covers

31 indexes are a key prefix of another index on the same collection that
holds every document, so they serve no read the wider index cannot, yet
every write maintains them. Remove the declarations and add a spec that
fails when one is declared again, and that pins the eight field-level
indexes kept because their name collides with a legacy unique index: that
collision is what surfaces the tenant-index migration hint.

Partial, sparse, hidden and collated indexes never count as a cover, and
prefixes whose only cover is multikey (users/groups tenantId) stay.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Julien Millau 2026-11-02 13:50:58 +01:00
parent 28d8780dad
commit fe2cd45c43
23 changed files with 120 additions and 31 deletions

View file

@ -16,7 +16,6 @@ const aclEntrySchema: Schema<IAclEntry> = new Schema<IAclEntry>(
required: function (this: IAclEntry) {
return this.principalType !== PrincipalType.PUBLIC;
},
index: true,
},
principalModel: {
type: String,
@ -33,7 +32,6 @@ const aclEntrySchema: Schema<IAclEntry> = new Schema<IAclEntry>(
resourceId: {
type: Schema.Types.ObjectId,
required: true,
index: true,
},
permBits: {
type: Number,

View file

@ -186,7 +186,6 @@ const agentSchema: Schema<IAgent> = new Schema<IAgent>(
},
tenantId: {
type: String,
index: true,
},
},
{

View file

@ -18,7 +18,6 @@ const agentApiKeySchema: Schema<IAgentApiKey> = new Schema(
type: mongoose.Schema.Types.ObjectId,
ref: 'User',
required: true,
index: true,
},
name: {
type: String,

View file

@ -23,12 +23,10 @@ const agentCategorySchema: Schema<IAgentCategory> = new Schema<IAgentCategory>(
order: {
type: Number,
default: 0,
index: true,
},
isActive: {
type: Boolean,
default: true,
index: true,
},
custom: {
type: Boolean,

View file

@ -35,7 +35,6 @@ const assistantSchema: Schema<IAssistant> = new Schema<IAssistant>(
},
tenantId: {
type: String,
index: true,
},
},
{

View file

@ -37,7 +37,6 @@ const chatProjectSchema: Schema<IChatProjectDocument> = new Schema<IChatProjectD
user: {
type: String,
required: true,
index: true,
},
conversationCount: {
type: Number,

View file

@ -8,7 +8,6 @@ const configSchema: Schema<IConfig> = new Schema<IConfig>(
type: String,
enum: Object.values(PrincipalType),
required: true,
index: true,
},
principalId: {
type: String,
@ -24,7 +23,6 @@ const configSchema: Schema<IConfig> = new Schema<IConfig>(
priority: {
type: Number,
required: true,
index: true,
},
overrides: {
type: Schema.Types.Mixed,

View file

@ -13,7 +13,6 @@ const conversationTag: Schema<IConversationTag> = new Schema<IConversationTag>(
{
tag: {
type: String,
index: true,
},
user: {
type: String,

View file

@ -34,7 +34,6 @@ const convoSchema: Schema<IConversation> = new Schema(
titleRevision: { type: Number },
user: {
type: String,
index: true,
meiliIndex: true,
},
messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }],
@ -402,7 +401,6 @@ const convoSchema: Schema<IConversation> = new Schema(
},
tenantId: {
type: String,
index: true,
},
pinned: {
type: Boolean,

View file

@ -8,7 +8,6 @@ const toolFavoriteSchema: Schema<IToolFavorite> = new Schema(
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
index: true,
},
itemType: {
type: String,

View file

@ -11,7 +11,6 @@ const MemoryEntrySchema: Schema<IMemoryEntry> = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'User',
index: true,
required: true,
},
key: {

View file

@ -12,13 +12,11 @@ const messageSchema: Schema<IMessage> = new Schema(
},
conversationId: {
type: String,
index: true,
required: true,
meiliIndex: true,
},
user: {
type: String,
index: true,
required: true,
default: null,
meiliIndex: true,
@ -327,7 +325,6 @@ const messageSchema: Schema<IMessage> = new Schema(
},
tenantId: {
type: String,
index: true,
},
},
{ timestamps: true },

View file

@ -64,7 +64,7 @@ const queuedTurnSchema: Schema<IAgentQueuedTurnDocument> = new Schema(
required: true,
index: true,
},
tenantId: { type: String, index: true },
tenantId: { type: String },
conversationId: { type: String, required: true, maxlength: 256 },
agentId: { type: String, required: true, maxlength: 256 },
parentMessageId: {

View file

@ -0,0 +1,117 @@
import mongoose from 'mongoose';
import type { IndexDefinition, IndexOptions, Schema } from 'mongoose';
import { SUPERSEDED_INDEXES } from '../migrations/tenantIndexes';
import { createModels } from '../models';
/** An index that holds an entry for every document and answers simple-collation reads. */
const holdsEveryDocument = (options: IndexOptions) =>
!options.sparse &&
!options.hidden &&
options.partialFilterExpression == null &&
options.collation == null;
/** An index that only costs writes once another index serves its reads. */
const isPlain = (options: IndexOptions) =>
!options.unique && options.expireAfterSeconds == null && holdsEveryDocument(options);
/** The B-tree direction of a key; text, 2dsphere, hashed and wildcard keys serve other reads. */
const direction = (value: unknown): 1 | -1 | undefined => {
const dir = Number(value);
return dir === 1 || dir === -1 ? dir : undefined;
};
/** The name MongoDB gives an index declared without one. */
const indexName = (keys: IndexDefinition) =>
Object.entries(keys)
.map(([field, dir]) => `${field}_${dir}`)
.join('_');
/** True when `field`, or any parent of it, is an array, which makes an index on it multikey. */
const isArrayPath = (schema: Schema, field: string) =>
field
.split('.')
.some((_, i, parts) => schema.path(parts.slice(0, i + 1).join('.'))?.instance === 'Array');
/**
* True when `long` serves every read `short` does: `short`'s keys lead `long` in order, all in
* the same or all in the opposite direction (a scan runs either way), every key of `long` is a
* B-tree key, and no trailing key is multikey, which would repeat each document once per array
* element in a scan over the prefix.
*/
function covers(schema: Schema, long: IndexDefinition, short: IndexDefinition): boolean {
const shortKeys = Object.entries(short);
const longKeys = Object.entries(long);
const first = direction(shortKeys[0]?.[1]);
const lead = direction(longKeys[0]?.[1]);
if (first === undefined || lead === undefined || shortKeys.length > longKeys.length) {
return false;
}
const sign = first * lead;
return (
longKeys.every(([, dir]) => direction(dir) !== undefined) &&
shortKeys.every(([field, dir], i) => {
const expected = direction(dir);
return (
expected !== undefined &&
longKeys[i][0] === field &&
direction(longKeys[i][1]) === expected * sign
);
}) &&
!longKeys.slice(shortKeys.length).some(([field]) => isArrayPath(schema, field))
);
}
/** Every schema's declared indexes with their declaration order, keyed by collection. */
const declaredIndexes = () =>
Object.values(createModels(mongoose)).map(({ schema, collection }) => ({
schema,
collection: collection.collectionName,
legacy: SUPERSEDED_INDEXES[collection.collectionName] ?? [],
entries: schema.indexes().map(([keys, options], at) => ({ keys, options, at })),
}));
/**
* A plain index whose keys lead another index that holds every document costs a write on each
* insert and update for reads the wider index already serves; the wider keys make a scan over
* the prefix alone touch more pages, which is the trade this test accepts. A field-level index
* that shares its name with a legacy unique index stays: colliding with that index on startup
* is what surfaces the tenant-index migration hint.
*/
describe('schema indexes', () => {
it('declares no plain index that another index already serves', () => {
const redundant = declaredIndexes().flatMap(({ schema, collection, legacy, entries }) =>
entries
.filter(({ keys, options }) => isPlain(options) && !legacy.includes(indexName(keys)))
.filter(({ keys, at }) =>
entries.some(
(other) =>
other.at !== at &&
holdsEveryDocument(other.options) &&
covers(schema, other.keys, keys) &&
/** Of two identical declarations, only the later one is redundant. */
(Object.keys(other.keys).length > Object.keys(keys).length || other.at < at),
),
)
.map(({ keys }) => `${collection}: ${JSON.stringify(keys)}`),
);
expect(redundant).toEqual([]);
});
it('keeps the field-level indexes whose name a legacy unique index shares', () => {
const tripwires = declaredIndexes().flatMap(({ collection, legacy, entries }) =>
entries
.filter(({ keys, options }) => isPlain(options) && legacy.includes(indexName(keys)))
.map(({ keys }) => `${collection}.${indexName(keys)}`),
);
expect(tripwires.sort()).toEqual([
'accessroles.accessRoleId_1',
'agentcategories.value_1',
'conversations.conversationId_1',
'mcpservers.serverName_1',
'messages.messageId_1',
'presets.presetId_1',
'roles.name_1',
'users.email_1',
]);
});
});

View file

@ -21,7 +21,6 @@ const scheduleSchema: Schema<IScheduleDocument> = new Schema(
id: {
type: String,
required: true,
index: true,
},
user: {
type: Schema.Types.ObjectId,

View file

@ -14,7 +14,6 @@ const scheduleRunSchema: Schema<IScheduleRunDocument> = new Schema(
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
index: true,
},
tenantId: {
type: String,

View file

@ -54,7 +54,6 @@ const shareSchema: Schema<ISharedLink> = new Schema(
},
user: {
type: String,
index: true,
},
messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }],
shareId: {

View file

@ -51,7 +51,6 @@ const skillSchema: Schema<ISkillDocument> = new Schema(
name: {
type: String,
required: true,
index: true,
maxlength: [SKILL_NAME_MAX_LENGTH, `Name cannot exceed ${SKILL_NAME_MAX_LENGTH} characters`],
validate: {
validator: function (value: string): boolean {
@ -161,13 +160,11 @@ const skillSchema: Schema<ISkillDocument> = new Schema(
category: {
type: String,
default: '',
index: true,
},
author: {
type: Schema.Types.ObjectId,
ref: 'User',
required: true,
index: true,
},
authorName: {
type: String,

View file

@ -14,7 +14,6 @@ const skillFileSchema: Schema<ISkillFileDocument> = new Schema(
type: Schema.Types.ObjectId,
ref: 'Skill',
required: true,
index: true,
},
relativePath: {
type: String,

View file

@ -7,7 +7,6 @@ const skillSyncCredentialSchema: Schema<ISkillSyncCredentialDocument> = new Sche
type: String,
enum: ['github'],
required: true,
index: true,
},
credentialKey: {
type: String,

View file

@ -77,7 +77,6 @@ const skillSyncStatusSchema: Schema<ISkillSyncStatusDocument> = new Schema(
type: String,
enum: ['github'],
required: true,
index: true,
},
sourceId: {
type: String,

View file

@ -4,7 +4,7 @@ import type { ISkillUploadIntentDocument } from '~/types/skillDeletion';
const skillUploadIntentSchema: Schema<ISkillUploadIntentDocument> =
new Schema<ISkillUploadIntentDocument>(
{
skillId: { type: Schema.Types.ObjectId, required: true, index: true },
skillId: { type: Schema.Types.ObjectId, required: true },
fileId: { type: String, required: true, unique: true, index: true },
source: { type: String, required: true },
storageKey: { type: String, required: true },
@ -27,7 +27,6 @@ const skillUploadIntentSchema: Schema<ISkillUploadIntentDocument> =
{ timestamps: true },
);
skillUploadIntentSchema.index({ skillId: 1, state: 1 });
skillUploadIntentSchema.index({ skillId: 1, state: 1, cleanedAt: 1 });
skillUploadIntentSchema.index({ state: 1, lastCleanupAttemptAt: 1, createdAt: 1 });
// The committed cleanup branch must not scan healthy live uploads just to preserve ordering.

View file

@ -121,7 +121,7 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
// Zero is reserved for a staging row that is visible before sequence allocation.
laneSequence: { type: Number, required: true, min: 0 },
envelope: { type: Schema.Types.Mixed, required: true },
user: { type: Schema.Types.ObjectId, ref: 'User', required: true, index: true },
user: { type: Schema.Types.ObjectId, ref: 'User', required: true },
tenantId: { type: String, index: true },
status: {
type: String,