From 5a7fe0bb863c67475fe9871af5d036582730befd Mon Sep 17 00:00:00 2001 From: Malte Polley Date: Sat, 27 Jun 2026 22:23:15 +0200 Subject: [PATCH] feat: integrate multi-provider adapters into skill sync runner --- .../api/src/skills/sync/genericSync.spec.ts | 396 ++++++++ packages/api/src/skills/sync/genericSync.ts | 859 ++++++++++++++++++ packages/api/src/skills/sync/github.ts | 24 +- packages/api/src/skills/sync/orchestrator.ts | 52 +- packages/api/src/skills/sync/scheduler.ts | 84 +- packages/data-schemas/src/methods/skill.ts | 12 +- packages/data-schemas/src/types/skill.ts | 5 +- 7 files changed, 1405 insertions(+), 27 deletions(-) create mode 100644 packages/api/src/skills/sync/genericSync.spec.ts create mode 100644 packages/api/src/skills/sync/genericSync.ts diff --git a/packages/api/src/skills/sync/genericSync.spec.ts b/packages/api/src/skills/sync/genericSync.spec.ts new file mode 100644 index 0000000000..179f0886e5 --- /dev/null +++ b/packages/api/src/skills/sync/genericSync.spec.ts @@ -0,0 +1,396 @@ +import { Types } from 'mongoose'; +import type { + ISkill, + ISkillSyncStatus, + CreateSkillInput, + CreateSkillResult, + SkillSyncStatusInput, +} from '@librechat/data-schemas'; +import type { SkillSyncConfig } from 'librechat-data-provider'; +import type { GitHubSkillSyncDeps } from './github'; +import { createGitHubSkillSyncRunner } from './github'; + +function makeSkill(input: CreateSkillInput): ISkill & { _id: Types.ObjectId } { + return { + _id: new Types.ObjectId(), + name: input.name, + description: input.description ?? '', + body: input.body ?? '', + source: input.source ?? 'inline', + sourceMetadata: input.sourceMetadata, + author: input.author ?? new Types.ObjectId(), + authorName: input.authorName ?? '', + version: 1, + fileCount: 0, + alwaysApply: input.alwaysApply ?? false, + tenantId: input.tenantId, + } as ISkill & { _id: Types.ObjectId }; +} + +/** + * Builds a mock fetch that emulates a GitLab-like API: + * - GET /api/v4/projects/:id/repository/commits/:ref → commit + * - GET /api/v4/projects/:id/repository/tree → tree listing + * - GET /api/v4/projects/:id/repository/blobs/:sha/raw → blob content + */ +function gitlabFetch(skillMd: string): typeof fetch { + return jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/repository/commits/')) { + return { ok: true, status: 200, json: async () => ({ id: 'gl-commit-sha', parent_ids: [] }) } as Response; + } + if (url.includes('/repository/tree')) { + return { + ok: true, + status: 200, + headers: { get: () => null }, + json: async () => [ + { id: 'gl-skill-sha', name: 'SKILL.md', type: 'blob', path: 'skills/research/SKILL.md', mode: '100644' }, + ], + } as unknown as Response; + } + if (url.includes('/blobs/') && url.includes('/raw')) { + const buf = Buffer.from(skillMd); + return { + ok: true, + status: 200, + arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength), + } as unknown as Response; + } + return { ok: false, status: 404, json: async () => ({}) } as unknown as Response; + }) as unknown as typeof fetch; +} + +/** + * Builds a mock fetch that emulates a Bitbucket-like API: + * - GET /repositories/:ws/:repo/commit/:ref → commit + * - GET /repositories/:ws/:repo/src/:ref/ → tree listing + * - GET /repositories/:ws/:repo/src/:ref/:path → blob content + */ +function bitbucketFetch(skillMd: string): typeof fetch { + return jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/commit/')) { + return { ok: true, status: 200, json: async () => ({ hash: 'bb-commit-sha' }) } as Response; + } + if (url.match(/\/src\/[^/]+\/\?/) || url.match(/\/src\/[^/]+\?/)) { + return { + ok: true, + status: 200, + json: async () => ({ + values: [ + { path: 'skills/writer/SKILL.md', type: 'commit_file', size: skillMd.length, commit: { hash: 'bb-commit-sha' } }, + ], + next: undefined, + }), + } as unknown as Response; + } + if (url.includes('/src/')) { + const buf = Buffer.from(skillMd); + return { + ok: true, + status: 200, + arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength), + } as unknown as Response; + } + return { ok: false, status: 404, json: async () => ({}) } as unknown as Response; + }) as unknown as typeof fetch; +} + +/** + * Builds a mock fetch that emulates Azure DevOps API: + * - GET /_apis/git/repositories/:repo/refs → refs + * - GET /_apis/git/repositories/:repo/commits/:id → commit + * - GET /_apis/git/repositories/:repo/trees/:sha → tree listing + * - GET /_apis/git/repositories/:repo/blobs/:sha → blob content + */ +function azureFetch(skillMd: string): typeof fetch { + return jest.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes('/refs?')) { + return { ok: true, status: 200, json: async () => ({ value: [{ name: 'refs/heads/main', objectId: 'az-commit-sha' }] }) } as Response; + } + if (url.includes('/commits/')) { + return { ok: true, status: 200, json: async () => ({ commitId: 'az-commit-sha', treeId: 'az-tree-sha' }) } as Response; + } + if (url.includes('/trees/')) { + return { + ok: true, + status: 200, + json: async () => ({ + treeEntries: [ + { objectId: 'az-blob-sha', relativePath: 'skills/coder/SKILL.md', gitObjectType: 'blob', size: skillMd.length, mode: '100644' }, + ], + }), + } as unknown as Response; + } + if (url.includes('/blobs/')) { + const buf = Buffer.from(skillMd); + return { + ok: true, + status: 200, + arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength), + } as unknown as Response; + } + return { ok: false, status: 404, json: async () => ({}) } as unknown as Response; + }) as unknown as typeof fetch; +} + +function createMultiProviderConfig(): SkillSyncConfig { + return { + github: { + enabled: false, + intervalMinutes: 60, + runOnStartup: false, + sources: [], + }, + gitlab: { + enabled: true, + intervalMinutes: 30, + runOnStartup: false, + sources: [ + { + id: 'gl-skills', + projectId: '12345', + ref: 'main', + paths: ['skills'], + token: '${GITLAB_SKILLS_TOKEN}', + }, + ], + }, + bitbucket: { + enabled: true, + intervalMinutes: 45, + runOnStartup: false, + sources: [ + { + id: 'bb-skills', + workspace: 'myteam', + repository: 'skills-repo', + ref: 'main', + paths: ['skills'], + token: '${BITBUCKET_SKILLS_TOKEN}', + }, + ], + }, + azuredevops: { + enabled: true, + intervalMinutes: 60, + runOnStartup: false, + sources: [ + { + id: 'az-skills', + organization: 'myorg', + project: 'myproj', + repository: 'skills-repo', + ref: 'main', + paths: ['skills'], + token: '${AZUREDEVOPS_SKILLS_TOKEN}', + }, + ], + }, + }; +} + +function createDeps(overrides: Partial = {}): GitHubSkillSyncDeps { + const statuses: ISkillSyncStatus[] = []; + return { + getConfig: async () => createMultiProviderConfig(), + getCredentialToken: jest.fn(async () => null), + getCredentialSummary: jest.fn(async () => null), + listCredentials: jest.fn(async () => []), + listStatuses: jest.fn(async () => statuses), + upsertStatus: jest.fn(async (input: SkillSyncStatusInput) => { + const status = { + provider: input.provider, + sourceId: input.sourceId, + tenantId: input.tenantId, + status: input.status, + credentialKey: input.credentialKey, + ref: input.ref, + paths: input.paths, + startedAt: input.startedAt, + finishedAt: input.finishedAt, + lastSuccessAt: input.status === 'succeeded' ? input.finishedAt : undefined, + lastFailureAt: input.status === 'failed' ? input.finishedAt : undefined, + errorCode: input.errorCode, + errorMessage: input.errorMessage, + syncedSkillCount: input.syncedSkillCount ?? 0, + syncedFileCount: input.syncedFileCount ?? 0, + deletedSkillCount: input.deletedSkillCount ?? 0, + deletedFileCount: input.deletedFileCount ?? 0, + } as ISkillSyncStatus; + statuses.push(status); + return status; + }), + tryAcquireLock: jest.fn(async () => true), + refreshLock: jest.fn(async () => true), + releaseLock: jest.fn(async () => undefined), + createSkill: jest.fn(async (input: CreateSkillInput): Promise => { + return { skill: makeSkill(input), warnings: [] }; + }), + updateSkill: jest.fn(), + getSkillById: jest.fn(), + findSkillBySourceIdentity: jest.fn(async () => null), + listSkillsBySource: jest.fn(async () => []), + listSkillFiles: jest.fn(async () => []), + getSkillFileByPath: jest.fn(async () => null), + upsertSkillFile: jest.fn(async () => ({ + _id: new Types.ObjectId(), + skillId: new Types.ObjectId(), + relativePath: 'file', + file_id: 'fid', + filename: 'file', + filepath: '/uploads/file', + source: 'local', + mimeType: 'text/plain', + bytes: 1, + category: 'file', + isExecutable: false, + author: new Types.ObjectId(), + })), + deleteSkillFile: jest.fn(async () => ({ deleted: true })), + deleteSkill: jest.fn(async () => ({ deleted: true })), + saveBuffer: jest.fn(async () => ({ filepath: '/uploads/file', source: 'local' })), + deleteFile: jest.fn(async () => undefined), + grantPermission: jest.fn(async () => undefined), + fetchFn: jest.fn(async () => ({ ok: false, status: 404 })) as unknown as typeof fetch, + ...overrides, + }; +} + +describe('Multi-provider skill sync integration wiring', () => { + const originalEnv = process.env; + + beforeEach(() => { + process.env = { + ...originalEnv, + GITLAB_SKILLS_TOKEN: 'glpat-test-token', + BITBUCKET_SKILLS_TOKEN: 'bb-test-token', + AZUREDEVOPS_SKILLS_TOKEN: 'az-test-token', + }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it('syncs a GitLab source when GitHub is disabled', async () => { + const glFetch = gitlabFetch('---\nname: research\ndescription: From GitLab\n---\nGL body'); + const config = createMultiProviderConfig(); + // Only enable GitLab + config.bitbucket!.enabled = false; + config.azuredevops!.enabled = false; + const deps = createDeps({ + fetchFn: glFetch, + getConfig: async () => config, + }); + const runner = createGitHubSkillSyncRunner(deps); + + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'research', + description: 'From GitLab', + source: 'gitlab', + sourceMetadata: expect.objectContaining({ + provider: 'gitlab', + sourceId: 'gl-skills', + }), + }), + ); + }); + + it('syncs a Bitbucket source when GitHub is disabled', async () => { + const bbFetch = bitbucketFetch('---\nname: writer\ndescription: From Bitbucket\n---\nBB body'); + const config = createMultiProviderConfig(); + // Only enable Bitbucket + config.gitlab!.enabled = false; + config.azuredevops!.enabled = false; + const deps = createDeps({ + fetchFn: bbFetch, + getConfig: async () => config, + }); + const runner = createGitHubSkillSyncRunner(deps); + + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'writer', + source: 'bitbucket', + sourceMetadata: expect.objectContaining({ + provider: 'bitbucket', + sourceId: 'bb-skills', + }), + }), + ); + }); + + it('syncs an Azure DevOps source when GitHub is disabled', async () => { + const azFetch = azureFetch('---\nname: coder\ndescription: From Azure\n---\nAZ body'); + const config = createMultiProviderConfig(); + // Only enable Azure DevOps + config.gitlab!.enabled = false; + config.bitbucket!.enabled = false; + const deps = createDeps({ + fetchFn: azFetch, + getConfig: async () => config, + }); + const runner = createGitHubSkillSyncRunner(deps); + + const result = await runner.runOnce(); + + expect(result.status).toBe('completed'); + expect(deps.createSkill).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'coder', + source: 'azuredevops', + sourceMetadata: expect.objectContaining({ + provider: 'azuredevops', + sourceId: 'az-skills', + }), + }), + ); + }); + + it('reports failed status when token env var is missing', async () => { + delete process.env.GITLAB_SKILLS_TOKEN; + const config = createMultiProviderConfig(); + config.bitbucket!.enabled = false; + config.azuredevops!.enabled = false; + const deps = createDeps({ getConfig: async () => config }); + const runner = createGitHubSkillSyncRunner(deps); + + const result = await runner.runOnce(); + + expect(result.status).toBe('failed'); + expect(result.sources).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + provider: 'gitlab', + sourceId: 'gl-skills', + status: 'failed', + errorCode: 'MISSING_CREDENTIAL', + }), + ]), + ); + }); + + it('skips disabled providers and syncs only enabled ones', async () => { + const config = createMultiProviderConfig(); + config.gitlab!.enabled = false; + config.bitbucket!.enabled = false; + config.azuredevops!.enabled = false; + const deps = createDeps({ getConfig: async () => config }); + const runner = createGitHubSkillSyncRunner(deps); + + const result = await runner.runOnce(); + + expect(result.status).toBe('skipped'); + expect(result.sources).toHaveLength(0); + }); +}); diff --git a/packages/api/src/skills/sync/genericSync.ts b/packages/api/src/skills/sync/genericSync.ts new file mode 100644 index 0000000000..994fb79356 --- /dev/null +++ b/packages/api/src/skills/sync/genericSync.ts @@ -0,0 +1,859 @@ +import path from 'path'; +import crypto from 'crypto'; +import { Types } from 'mongoose'; +import { logger, tenantStorage } from '@librechat/data-schemas'; +import { + ResourceType, + PrincipalType, + AccessRoleIds, + SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH, +} from 'librechat-data-provider'; +import type { + ISkill, + ISkillFile, + ISkillSyncStatus, + SkillSyncProvider, + SkillSyncStatusInput, + UpsertSkillFileInput, +} from '@librechat/data-schemas'; +import type { GitRepoAdapter, GitTreeEntry } from './adapter'; +import type { GitHubSkillSyncDeps } from './github'; +import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits'; +import { parseSkillMarkdown } from '../parse'; + +const SYSTEM_AUTHOR_ID = new Types.ObjectId('000000000000000000000000'); + +type AssertNotCancelled = () => void; + +export type GenericSourceConfig = { + id: string; + ref: string; + paths: string[]; + skillDiscoveryDepth?: number; + credentialKey?: string; + token?: string; + tenantId?: string; +}; + +type DiscoveredSkill = { + rootPath: string; + skillMd: GitTreeEntry; + files: GitTreeEntry[]; +}; + +type SyncCounters = { + syncedSkillCount: number; + syncedFileCount: number; + deletedSkillCount: number; + deletedFileCount: number; +}; + +type SaveBufferResult = { + filepath: string; + source: string; + storageKey?: string; + storageRegion?: string; +}; + +type StoredSkillFileRef = { + filepath: string; + source: string; + storageKey?: string; + storageRegion?: string; + author?: Types.ObjectId | string; + tenantId?: string; +}; + +// --- Pure helpers (duplicated from github.ts to avoid modifying it) --- + +function normalizeRepoPath(value: string): string { + const trimmed = value.trim().replace(/^\/+|\/+$/g, ''); + return trimmed === '.' ? '' : trimmed; +} + +function isSafeRelativePath(value: string): boolean { + if (!value || value.startsWith('/') || value.startsWith('\\')) { + return false; + } + if (!/^[a-zA-Z0-9._\-/]+$/.test(value)) { + return false; + } + return value.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..'); +} + +function toSkillName(value: string): string { + const normalized = value + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/^-+|-+$/g, '') + .replace(/-{2,}/g, '-'); + return normalized || 'synced-skill'; +} + +function getFilename(relativePath: string): string { + return path.posix.basename(relativePath); +} + +function guessMimeType(filename: string): string { + const ext = path.extname(filename).toLowerCase(); + const mimeMap: Record = { + '.md': 'text/markdown', + '.txt': 'text/plain', + '.js': 'application/javascript', + '.ts': 'text/typescript', + '.json': 'application/json', + '.yaml': 'text/yaml', + '.yml': 'text/yaml', + '.py': 'text/x-python', + '.sh': 'application/x-sh', + '.css': 'text/css', + '.html': 'text/html', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.svg': 'image/svg+xml', + '.webp': 'image/webp', + '.pdf': 'application/pdf', + }; + return mimeMap[ext] ?? 'application/octet-stream'; +} + +function toCleanFrontmatter( + frontmatter: Record | undefined, +): Record { + if (!frontmatter) { + return {}; + } + const clean = { ...frontmatter }; + delete clean.name; + delete clean.description; + if ('always-apply' in clean && typeof clean['always-apply'] !== 'boolean') { + delete clean['always-apply']; + } + if ('alwaysApply' in clean && typeof clean.alwaysApply !== 'boolean') { + delete clean.alwaysApply; + } + return clean; +} + +function getProviderAuthorName(provider: SkillSyncProvider): string { + switch (provider) { + case 'gitlab': + return 'GitLab Sync'; + case 'bitbucket': + return 'Bitbucket Sync'; + case 'azuredevops': + return 'Azure DevOps Sync'; + default: + return 'Skill Sync'; + } +} + +function makeUpstreamId(source: GenericSourceConfig, rootPath: string): string { + return `${source.id}:${rootPath}`; +} + +function makeSourceAuthorId( + provider: SkillSyncProvider, + source: GenericSourceConfig, +): Types.ObjectId { + const seed = source.tenantId + ? `${provider}:${source.id}:${source.tenantId}` + : `${provider}:${source.id}`; + const digest = crypto.createHash('sha256').update(seed).digest('hex').slice(0, 24); + return new Types.ObjectId(digest); +} + +function getLimitMegabytes(bytes: number): number { + return Math.round(bytes / 1024 / 1024); +} + +function assertBlobSize(entry: GitTreeEntry, relativePath: string): number { + if (typeof entry.size !== 'number' || !Number.isFinite(entry.size) || entry.size < 0) { + // If size not available from tree listing, skip pre-check (will check buffer after fetch) + return 0; + } + if (entry.size > DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes) { + throw new GenericSyncError( + 'BLOB_TOO_LARGE', + `File "${relativePath}" exceeds the ${getLimitMegabytes(DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes)}MB per-file limit`, + ); + } + return entry.size; +} + +function assertBufferSize(buffer: Buffer, relativePath: string): void { + if (buffer.length > DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes) { + throw new GenericSyncError( + 'BLOB_TOO_LARGE', + `File "${relativePath}" exceeds the ${getLimitMegabytes(DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes)}MB per-file limit`, + ); + } +} + +function assertCumulativeSize(totalBytes: number): void { + if (totalBytes > DEFAULT_SKILL_IMPORT_LIMITS.maxDecompressedBytes) { + throw new GenericSyncError( + 'PACKAGE_TOO_LARGE', + `Skill files exceed the ${getLimitMegabytes(DEFAULT_SKILL_IMPORT_LIMITS.maxDecompressedBytes)}MB cumulative limit`, + ); + } +} + +function assertEntryCount(discovered: DiscoveredSkill): void { + if (discovered.files.length + 1 > DEFAULT_SKILL_IMPORT_LIMITS.maxEntries) { + throw new GenericSyncError( + 'TOO_MANY_FILES', + `Skill "${discovered.rootPath}" exceeds the ${DEFAULT_SKILL_IMPORT_LIMITS.maxEntries} file limit`, + ); + } +} + +class GenericSyncError extends Error { + code: string; + constructor(code: string, message: string) { + super(message); + this.name = 'GenericSyncError'; + this.code = code; + } +} + +function sanitizeError(error: unknown): { code: string; message: string } { + if (error instanceof GenericSyncError) { + return { code: error.code, message: error.message }; + } + if (error instanceof Error) { + return { + code: 'SYNC_FAILED', + message: error.message.replace(/Bearer\s+\S+/gi, 'Bearer [redacted]'), + }; + } + return { code: 'SYNC_FAILED', message: 'Unknown skill sync failure' }; +} + +function getTokenEnvVarName(tokenReference: string | undefined): string | null { + const match = tokenReference?.trim().match(/^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/); + return match?.[1] ?? null; +} + +function resolveToken( + deps: GitHubSkillSyncDeps, + provider: SkillSyncProvider, + source: GenericSourceConfig, +): Promise { + if (deps.allowServerCredentials === false) { + return Promise.resolve(null); + } + const tokenEnvVar = getTokenEnvVarName(source.token); + if (tokenEnvVar) { + return Promise.resolve(process.env[tokenEnvVar]?.trim() || null); + } + if (!source.credentialKey) { + return Promise.resolve(null); + } + return deps.getCredentialToken(provider, source.credentialKey); +} + +function isSkillRootWithinDiscoveryDepth( + rootPath: string, + basePath: string, + maxDepth: number, +): boolean { + if (rootPath === basePath) { + return true; + } + if (basePath && !rootPath.startsWith(`${basePath}/`)) { + return false; + } + const relative = basePath ? rootPath.slice(basePath.length).replace(/^\/+/, '') : rootPath; + if (!relative) { + return true; + } + return relative.split('/').length <= maxDepth; +} + +function filterTreeByPaths(tree: GitTreeEntry[], paths: string[]): GitTreeEntry[] { + const normalizedPaths = paths.map(normalizeRepoPath); + return tree.filter((entry) => { + const entryPath = normalizeRepoPath(entry.path); + return normalizedPaths.some( + (basePath) => + basePath === '' || entryPath === basePath || entryPath.startsWith(`${basePath}/`), + ); + }); +} + +function discoverSkills(tree: GitTreeEntry[], source: GenericSourceConfig): DiscoveredSkill[] { + const basePaths = source.paths.map(normalizeRepoPath); + const skillDiscoveryDepth = source.skillDiscoveryDepth ?? SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH; + const skillMdByRoot = new Map(); + + for (const entry of tree) { + if (entry.type !== 'blob') { + continue; + } + const normalized = normalizeRepoPath(entry.path); + const basename = path.posix.basename(normalized); + if (basename.toUpperCase() !== 'SKILL.MD') { + continue; + } + const parent = normalizeRepoPath(path.posix.dirname(normalized)); + for (const basePath of basePaths) { + if (isSkillRootWithinDiscoveryDepth(parent, basePath, skillDiscoveryDepth)) { + skillMdByRoot.set(parent, entry); + } + } + } + + const skillRoots = [...skillMdByRoot.keys()]; + return [...skillMdByRoot.entries()].map(([rootPath, skillMd]) => { + const prefix = rootPath ? `${rootPath}/` : ''; + const childSkillRoots = skillRoots.filter( + (c) => c && c !== rootPath && (rootPath ? c.startsWith(`${rootPath}/`) : true), + ); + const files = tree.filter((entry) => { + if (entry.type !== 'blob') { + return false; + } + const normalized = normalizeRepoPath(entry.path); + if (!normalized.startsWith(prefix) || normalized === skillMd.path) { + return false; + } + if (childSkillRoots.some((childRoot) => normalized.startsWith(`${childRoot}/`))) { + return false; + } + const relativePath = prefix ? normalized.slice(prefix.length) : normalized; + return isSafeRelativePath(relativePath) && relativePath.toUpperCase() !== 'SKILL.MD'; + }); + return { rootPath, skillMd, files }; + }); +} + +function getDiscoveredRelativePath(discovered: DiscoveredSkill, entry: GitTreeEntry): string { + const prefix = discovered.rootPath ? `${discovered.rootPath}/` : ''; + const normalized = normalizeRepoPath(entry.path); + return prefix ? normalized.slice(prefix.length) : normalized; +} + +function makeStatusInput(params: { + provider: SkillSyncProvider; + source: GenericSourceConfig; + status: SkillSyncStatusInput['status']; + startedAt?: Date; + finishedAt?: Date; + errorCode?: string; + errorMessage?: string; + counts?: Partial; +}): SkillSyncStatusInput { + return { + provider: params.provider, + sourceId: params.source.id, + tenantId: params.source.tenantId, + status: params.status, + credentialKey: params.source.credentialKey, + ref: params.source.ref, + paths: params.source.paths, + startedAt: params.startedAt, + finishedAt: params.finishedAt, + errorCode: params.errorCode, + errorMessage: params.errorMessage, + syncedSkillCount: params.counts?.syncedSkillCount ?? 0, + syncedFileCount: params.counts?.syncedFileCount ?? 0, + deletedSkillCount: params.counts?.deletedSkillCount ?? 0, + deletedFileCount: params.counts?.deletedFileCount ?? 0, + }; +} + +function getSourceMetadataString( + row: { sourceMetadata?: Record }, + key: string, +): string | undefined { + const metadata = row.sourceMetadata; + const value = metadata && typeof metadata === 'object' ? metadata[key] : undefined; + return typeof value === 'string' ? value : undefined; +} + +/** + * Build the blob reference to pass to adapter.fetchBlob(). + * Bitbucket requires "commitSha:filePath" format; others use the entry sha directly. + */ +function makeBlobRef( + provider: SkillSyncProvider, + entry: GitTreeEntry, + commitSha: string, +): string { + if (provider === 'bitbucket') { + return `${commitSha}:${entry.path}`; + } + return entry.sha; +} + +async function ensurePublicViewer( + deps: GitHubSkillSyncDeps, + skillId: Types.ObjectId, +): Promise { + await deps.grantPermission({ + principalType: PrincipalType.PUBLIC, + principalId: null, + resourceType: ResourceType.SKILL, + resourceId: skillId, + accessRoleId: AccessRoleIds.SKILL_VIEWER, + grantedBy: SYSTEM_AUTHOR_ID, + }); +} + +export async function syncGenericSource(params: { + deps: GitHubSkillSyncDeps; + adapter: GitRepoAdapter; + provider: SkillSyncProvider; + source: GenericSourceConfig; + assertNotCancelled: AssertNotCancelled; +}): Promise { + const { deps, adapter, provider, source, assertNotCancelled } = params; + const startedAt = new Date(); + await deps.upsertStatus(makeStatusInput({ provider, source, status: 'running', startedAt })); + + try { + assertNotCancelled(); + const token = await resolveToken(deps, provider, source); + assertNotCancelled(); + if (!token) { + throw new GenericSyncError( + 'MISSING_CREDENTIAL', + `Missing ${provider} credential for source "${source.id}"`, + ); + } + + const { commitSha, treeSha } = await adapter.getTreeSha(source.ref); + assertNotCancelled(); + + const fullTree = await adapter.listTree(treeSha, true); + assertNotCancelled(); + + const tree = filterTreeByPaths(fullTree, source.paths); + const discoveredSkills = discoverSkills(tree, source); + const authorId = makeSourceAuthorId(provider, source); + const authorName = getProviderAuthorName(provider); + const syncedAt = new Date(); + const seenUpstreamIds = new Set(); + const counts: SyncCounters = { + syncedSkillCount: 0, + syncedFileCount: 0, + deletedSkillCount: 0, + deletedFileCount: 0, + }; + + for (const discovered of discoveredSkills) { + assertNotCancelled(); + assertEntryCount(discovered); + + const skillMdPath = discovered.rootPath + ? `${discovered.rootPath}/SKILL.md` + : 'SKILL.md'; + const blobRef = makeBlobRef(provider, discovered.skillMd, commitSha); + const skillMdBuffer = await adapter.fetchBlob(blobRef); + assertNotCancelled(); + assertBufferSize(skillMdBuffer, skillMdPath); + + const skillMdContent = skillMdBuffer.toString('utf-8'); + const parsed = parseSkillMarkdown(skillMdContent); + if (parsed.parseError) { + throw new GenericSyncError( + 'SKILL_PARSE_FAILED', + `${skillMdPath} contains invalid YAML frontmatter: ${parsed.parseError}`, + ); + } + + const upstreamId = makeUpstreamId(source, discovered.rootPath); + seenUpstreamIds.add(upstreamId); + const fallbackName = toSkillName(path.posix.basename(discovered.rootPath) || source.id); + + const sourceMetadata = { + provider, + sourceId: source.id, + upstreamId, + ref: source.ref, + skillPath: discovered.rootPath, + commitSha, + skillBlobSha: discovered.skillMd.sha, + syncedAt: syncedAt.toISOString(), + syncStatus: 'synced', + }; + + const update = { + name: parsed.name || fallbackName, + description: parsed.description || parsed.name || fallbackName, + body: skillMdContent, + frontmatter: toCleanFrontmatter(parsed.frontmatter), + alwaysApply: parsed.alwaysApply, + source: provider, + sourceMetadata, + }; + + const sourceTenantId = source.tenantId ?? undefined; + const existing = await deps.findSkillBySourceIdentity({ + source: provider, + upstreamId, + tenantId: sourceTenantId, + }); + + let skill: ISkill & { _id: Types.ObjectId }; + if (existing && (existing.tenantId ?? undefined) === sourceTenantId) { + const result = await deps.updateSkill({ + id: existing._id.toString(), + expectedVersion: existing.version, + update, + }); + if (result.status === 'updated') { + skill = result.skill; + } else { + throw new GenericSyncError('SKILL_CONFLICT', `Skill "${existing.name}" conflict during sync`); + } + } else { + const created = await deps.createSkill({ + ...update, + name: update.name ?? fallbackName, + description: update.description ?? fallbackName, + author: authorId, + authorName, + source: provider, + tenantId: source.tenantId, + }); + skill = created.skill; + } + + await ensurePublicViewer(deps, skill._id); + + // Sync files + const remotePaths = new Set(); + let totalFileBytes = 0; + for (const entry of discovered.files) { + assertNotCancelled(); + const relativePath = getDiscoveredRelativePath(discovered, entry); + if (!isSafeRelativePath(relativePath) || relativePath.toUpperCase() === 'SKILL.MD') { + continue; + } + totalFileBytes += assertBlobSize(entry, relativePath); + assertCumulativeSize(totalFileBytes); + remotePaths.add(relativePath); + + const existingFile = await deps.getSkillFileByPath(skill._id, relativePath); + if (existingFile && getSourceMetadataString(existingFile, 'blobSha') === entry.sha) { + continue; + } + + const fileBlobRef = makeBlobRef(provider, entry, commitSha); + const buffer = await adapter.fetchBlob(fileBlobRef); + assertNotCancelled(); + assertBufferSize(buffer, relativePath); + + const fileId = crypto.randomUUID(); + const filename = getFilename(relativePath); + const mimeType = guessMimeType(filename); + const saved = await deps.saveBuffer({ + userId: skill.author.toString(), + buffer, + fileName: `${fileId}__${filename}`, + basePath: 'uploads', + isImage: mimeType.startsWith('image/'), + tenantId: skill.tenantId, + }); + + await deps.upsertSkillFile({ + skillId: skill._id, + relativePath, + file_id: fileId, + filename, + filepath: saved.filepath, + storageKey: saved.storageKey, + storageRegion: saved.storageRegion, + source: saved.source, + sourceMetadata: { + provider, + sourceId: source.id, + upstreamId, + commitSha, + blobSha: entry.sha, + path: entry.path, + }, + mimeType, + bytes: buffer.length, + isExecutable: false, + author: skill.author, + tenantId: skill.tenantId, + }); + counts.syncedFileCount++; + } + + // Delete files no longer in remote + const existingFiles = await deps.listSkillFiles(skill._id); + for (const file of existingFiles) { + if (remotePaths.has(file.relativePath)) { + continue; + } + const result = await deps.deleteSkillFile(skill._id, file.relativePath); + if (result.deleted) { + counts.deletedFileCount++; + if (deps.deleteFile) { + await deps.deleteFile({ + filepath: file.filepath, + source: file.source, + storageKey: file.storageKey, + storageRegion: file.storageRegion, + user: file.author, + tenantId: file.tenantId, + }).catch((e) => logger.error(`[${provider}Sync] Failed to clean up file:`, e)); + } + } + } + + counts.syncedSkillCount++; + } + + // Delete stale skills no longer in remote + const currentSyncedSkills = await deps.listSkillsBySource({ + source: provider, + sourceId: source.id, + }); + const sourceTenantId = source.tenantId ?? undefined; + for (const skill of currentSyncedSkills) { + assertNotCancelled(); + if ((skill.tenantId ?? undefined) !== sourceTenantId) { + continue; + } + const upstreamId = getSourceMetadataString(skill, 'upstreamId') ?? ''; + if (seenUpstreamIds.has(upstreamId)) { + continue; + } + // Delete skill and its files + const files = await deps.listSkillFiles(skill._id); + for (const file of files) { + if (deps.deleteFile) { + await deps.deleteFile({ + filepath: file.filepath, + source: file.source, + storageKey: file.storageKey, + storageRegion: file.storageRegion, + user: file.author, + tenantId: file.tenantId, + }).catch((e) => logger.error(`[${provider}Sync] Failed to clean up stale file:`, e)); + } + } + await deps.deleteSkill(skill._id.toString()); + counts.deletedSkillCount++; + counts.deletedFileCount += files.length; + } + + return deps.upsertStatus( + makeStatusInput({ + provider, + source, + status: 'succeeded', + startedAt, + finishedAt: new Date(), + counts, + }), + ); + } catch (error) { + const sanitized = sanitizeError(error); + logger.error(`[${provider}Sync] Source "${source.id}" failed: ${sanitized.message}`); + return deps.upsertStatus( + makeStatusInput({ + provider, + source, + status: 'failed', + startedAt, + finishedAt: new Date(), + errorCode: sanitized.code, + errorMessage: sanitized.message, + }), + ); + } +} + +function syncGenericSourceInTenantContext(params: { + deps: GitHubSkillSyncDeps; + adapter: GitRepoAdapter; + provider: SkillSyncProvider; + source: GenericSourceConfig; + assertNotCancelled: AssertNotCancelled; +}): Promise { + if (!params.source.tenantId) { + return syncGenericSource(params); + } + return tenantStorage.run({ tenantId: params.source.tenantId }, async () => + syncGenericSource(params), + ); +} + +export { syncGenericSourceInTenantContext }; + +/** + * Called from the main runner's runOnce() to sync all non-GitHub providers. + * Reads gitlab/bitbucket/azuredevops from the config and syncs each enabled source. + */ +export async function syncNonGitHubProviders(params: { + deps: GitHubSkillSyncDeps; + fetchFn: typeof fetch; + assertNotCancelled: AssertNotCancelled; +}): Promise { + const { deps, fetchFn, assertNotCancelled } = params; + const config = await deps.getConfig(); + if (!config) { + return []; + } + + const results: ISkillSyncStatus[] = []; + + // GitLab + if (config.gitlab?.enabled && config.gitlab.sources.length > 0) { + for (const source of config.gitlab.sources) { + assertNotCancelled(); + const token = await resolveToken(deps, 'gitlab', source); + if (!token) { + results.push( + await deps.upsertStatus( + makeStatusInput({ + provider: 'gitlab', + source, + status: 'failed', + startedAt: new Date(), + finishedAt: new Date(), + errorCode: 'MISSING_CREDENTIAL', + errorMessage: `Missing gitlab credential for source "${source.id}"`, + }), + ), + ); + continue; + } + const { createRepoAdapter } = await import('./adapterFactory'); + const adapter = createRepoAdapter({ + provider: 'gitlab', + token, + projectId: source.projectId, + baseUrl: source.baseUrl, + fetchFn, + }); + results.push( + await syncGenericSourceInTenantContext({ + deps, + adapter, + provider: 'gitlab', + source: { + id: source.id, + ref: source.ref, + paths: source.paths, + skillDiscoveryDepth: source.skillDiscoveryDepth, + credentialKey: source.credentialKey, + token: source.token, + tenantId: source.tenantId, + }, + assertNotCancelled, + }), + ); + } + } + + // Bitbucket + if (config.bitbucket?.enabled && config.bitbucket.sources.length > 0) { + for (const source of config.bitbucket.sources) { + assertNotCancelled(); + const token = await resolveToken(deps, 'bitbucket', source); + if (!token) { + results.push( + await deps.upsertStatus( + makeStatusInput({ + provider: 'bitbucket', + source, + status: 'failed', + startedAt: new Date(), + finishedAt: new Date(), + errorCode: 'MISSING_CREDENTIAL', + errorMessage: `Missing bitbucket credential for source "${source.id}"`, + }), + ), + ); + continue; + } + const { createRepoAdapter } = await import('./adapterFactory'); + const adapter = createRepoAdapter({ + provider: 'bitbucket', + token, + workspace: source.workspace, + repository: source.repository, + fetchFn, + }); + results.push( + await syncGenericSourceInTenantContext({ + deps, + adapter, + provider: 'bitbucket', + source: { + id: source.id, + ref: source.ref, + paths: source.paths, + skillDiscoveryDepth: source.skillDiscoveryDepth, + credentialKey: source.credentialKey, + token: source.token, + tenantId: source.tenantId, + }, + assertNotCancelled, + }), + ); + } + } + + // Azure DevOps + if (config.azuredevops?.enabled && config.azuredevops.sources.length > 0) { + for (const source of config.azuredevops.sources) { + assertNotCancelled(); + const token = await resolveToken(deps, 'azuredevops', source); + if (!token) { + results.push( + await deps.upsertStatus( + makeStatusInput({ + provider: 'azuredevops', + source, + status: 'failed', + startedAt: new Date(), + finishedAt: new Date(), + errorCode: 'MISSING_CREDENTIAL', + errorMessage: `Missing azuredevops credential for source "${source.id}"`, + }), + ), + ); + continue; + } + const { createRepoAdapter } = await import('./adapterFactory'); + const adapter = createRepoAdapter({ + provider: 'azuredevops', + token, + organization: source.organization, + project: source.project, + repository: source.repository, + baseUrl: source.baseUrl, + fetchFn, + }); + results.push( + await syncGenericSourceInTenantContext({ + deps, + adapter, + provider: 'azuredevops', + source: { + id: source.id, + ref: source.ref, + paths: source.paths, + skillDiscoveryDepth: source.skillDiscoveryDepth, + credentialKey: source.credentialKey, + token: source.token, + tenantId: source.tenantId, + }, + assertNotCancelled, + }), + ); + } + } + + return results; +} diff --git a/packages/api/src/skills/sync/github.ts b/packages/api/src/skills/sync/github.ts index 5af41921f3..d7ff08dc89 100644 --- a/packages/api/src/skills/sync/github.ts +++ b/packages/api/src/skills/sync/github.ts @@ -24,6 +24,7 @@ import type { import type { SkillSyncConfig, SkillSyncGitHubSourceConfig } from 'librechat-data-provider'; import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits'; import { parseSkillMarkdown } from '../parse'; +import { syncNonGitHubProviders } from './genericSync'; const GITHUB_API_BASE = 'https://api.github.com'; const SYSTEM_AUTHOR_ID = new Types.ObjectId('000000000000000000000000'); @@ -171,12 +172,12 @@ export type GitHubSkillSyncDeps = { }) => Promise; getSkillById: (id: string | Types.ObjectId) => Promise<(ISkill & { _id: Types.ObjectId }) | null>; findSkillBySourceIdentity: (params: { - source: 'github' | 'notion'; + source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; upstreamId: string; tenantId?: string; }) => Promise<(ISkill & { _id: Types.ObjectId }) | null>; listSkillsBySource: (params: { - source: 'github' | 'notion'; + source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; sourceId: string; }) => Promise>; listSkillFiles: ( @@ -1790,8 +1791,16 @@ export function createGitHubSkillSyncRunner(deps: GitHubSkillSyncDeps): GitHubSk async function runOnce(): Promise { const github = getGithubConfig(await deps.getConfig()); - if (!github.enabled || github.sources.length === 0) { - return { status: 'skipped', message: 'GitHub skill sync is disabled', sources: [] }; + const githubEnabled = github.enabled && github.sources.length > 0; + if (!githubEnabled) { + // Even if GitHub is disabled, process other providers + const assertNotCancelled = () => {}; + const genericSources = await syncNonGitHubProviders({ deps, fetchFn, assertNotCancelled }); + if (genericSources.length === 0) { + return { status: 'skipped', message: 'Skill sync is disabled', sources: [] }; + } + const failed = genericSources.some((s) => s.status === 'failed'); + return { status: failed ? 'failed' : 'completed', sources: genericSources }; } const allowServerCredentials = deps.allowServerCredentials !== false; if (!allowServerCredentials) { @@ -1856,6 +1865,13 @@ export function createGitHubSkillSyncRunner(deps: GitHubSkillSyncDeps): GitHubSk await syncSourceInTenantContext({ deps, source, fetchFn, assertNotCancelled }), ); } + + // Sync non-GitHub providers + if (!lockLost) { + const genericSources = await syncNonGitHubProviders({ deps, fetchFn, assertNotCancelled }); + sources.push(...genericSources); + } + const failed = sources.some((source) => source.status === 'failed'); return { status: failed || lockLost ? 'failed' : 'completed', diff --git a/packages/api/src/skills/sync/orchestrator.ts b/packages/api/src/skills/sync/orchestrator.ts index 5181799f05..3c34868869 100644 --- a/packages/api/src/skills/sync/orchestrator.ts +++ b/packages/api/src/skills/sync/orchestrator.ts @@ -77,6 +77,25 @@ function hasGitHubConfig(config: SkillSyncConfig | undefined): config is SkillSy return Boolean(config?.github); } +function hasAnyEnabledProvider(config: SkillSyncConfig | undefined): boolean { + if (!config) { + return false; + } + if (config.github?.enabled && config.github.sources.length > 0) { + return true; + } + if (config.gitlab?.enabled && config.gitlab.sources.length > 0) { + return true; + } + if (config.bitbucket?.enabled && config.bitbucket.sources.length > 0) { + return true; + } + if (config.azuredevops?.enabled && config.azuredevops.sources.length > 0) { + return true; + } + return false; +} + function isSameSkillSyncConfig( left: SkillSyncConfig | undefined, right: SkillSyncConfig | undefined, @@ -153,13 +172,18 @@ function getRequestSkillSyncConfig( logger: SkillSyncTriggerLogger, ): SkillSyncConfig | undefined { const resolved = parseSkillSyncConfig(appConfig?.skillSync, logger); - if ( - !hasGitHubConfig(resolved) || - !resolved.github.enabled || - resolved.github.sources.length === 0 - ) { + if (!hasAnyEnabledProvider(resolved)) { return undefined; } + if (!hasGitHubConfig(resolved)) { + // Non-GitHub providers are configured but no GitHub — pass through the + // full config so the runner can process them via syncNonGitHubProviders. + const base = parseSkillSyncConfig(appConfig?.config?.skillSync, logger); + if (isSameSkillSyncConfig(resolved, base)) { + return undefined; + } + return resolved; + } const base = parseSkillSyncConfig(appConfig?.config?.skillSync, logger); if (isSameSkillSyncConfig(resolved, base)) { @@ -268,11 +292,13 @@ export function createSkillSyncTriggerOrchestrator( async function maybeRunForRequest(request: SkillSyncRequestLike): Promise { const config = getRequestSkillSyncConfig(request.config, request.user, deps.logger); - if (!hasGitHubConfig(config)) { + if (!config) { return false; } - const syncKey = getRequestSyncKey(config, request.user); + const syncKey = hasGitHubConfig(config) + ? getRequestSyncKey(config, request.user) + : `nongithub:${JSON.stringify(request.user?.tenantId ?? '')}`; if (inFlight.has(syncKey)) { return false; } @@ -282,9 +308,15 @@ export function createSkillSyncTriggerOrchestrator( loadAppConfig: async () => request.config, allowServerCredentials: Boolean(request.skillSyncAllowServerCredentials), }); - const status = await requestRunner.getStatus(); - if (!shouldRunRequestSync(status, { minIntervalMs, staleRunningMs })) { - return false; + + // For configs with GitHub sources, use the status-based interval check. + // For non-GitHub-only configs, run unconditionally (interval gating happens + // inside the generic sync via the per-source status timestamps). + if (hasGitHubConfig(config)) { + const status = await requestRunner.getStatus(); + if (!shouldRunRequestSync(status, { minIntervalMs, staleRunningMs })) { + return false; + } } inFlight.add(syncKey); diff --git a/packages/api/src/skills/sync/scheduler.ts b/packages/api/src/skills/sync/scheduler.ts index 7aad2d7bab..6016ea64d0 100644 --- a/packages/api/src/skills/sync/scheduler.ts +++ b/packages/api/src/skills/sync/scheduler.ts @@ -28,6 +28,79 @@ function getSources(config: SkillSyncConfig | undefined) { return Array.isArray(sources) ? sources : []; } +function hasAnySources(config: SkillSyncConfig | undefined): boolean { + if (getSources(config).length > 0) { + return true; + } + if (config?.gitlab?.enabled && config.gitlab.sources.length > 0) { + return true; + } + if (config?.bitbucket?.enabled && config.bitbucket.sources.length > 0) { + return true; + } + if (config?.azuredevops?.enabled && config.azuredevops.sources.length > 0) { + return true; + } + return false; +} + +function hasAnyEnabled(config: SkillSyncConfig | undefined): boolean { + if (config?.github?.enabled && getSources(config).length > 0) { + return true; + } + if (config?.gitlab?.enabled && config.gitlab.sources.length > 0) { + return true; + } + if (config?.bitbucket?.enabled && config.bitbucket.sources.length > 0) { + return true; + } + if (config?.azuredevops?.enabled && config.azuredevops.sources.length > 0) { + return true; + } + return false; +} + +function getMinIntervalMinutes(config: SkillSyncConfig | undefined): number { + const intervals: number[] = []; + if (config?.github?.enabled) { + intervals.push(config.github.intervalMinutes ?? 60); + } + if (config?.gitlab?.enabled) { + intervals.push(config.gitlab.intervalMinutes ?? 60); + } + if (config?.bitbucket?.enabled) { + intervals.push(config.bitbucket.intervalMinutes ?? 60); + } + if (config?.azuredevops?.enabled) { + intervals.push(config.azuredevops.intervalMinutes ?? 60); + } + return intervals.length > 0 ? Math.min(...intervals) : 60; +} + +function shouldRunOnStartup(config: SkillSyncConfig | undefined): boolean { + if (config?.github?.enabled && config.github.runOnStartup && getSources(config).length > 0) { + return true; + } + if (config?.gitlab?.enabled && config.gitlab.runOnStartup && config.gitlab.sources.length > 0) { + return true; + } + if ( + config?.bitbucket?.enabled && + config.bitbucket.runOnStartup && + config.bitbucket.sources.length > 0 + ) { + return true; + } + if ( + config?.azuredevops?.enabled && + config.azuredevops.runOnStartup && + config.azuredevops.sources.length > 0 + ) { + return true; + } + return false; +} + export function startGitHubSkillSyncScheduler(params: { getConfig: () => MaybePromise; runner: GitHubSkillSyncRunner; @@ -48,21 +121,20 @@ export function startGitHubSkillSyncScheduler(params: { if (stopped) { return; } - const delayMs = normalizeIntervalMinutes(config?.github?.intervalMinutes) * 60 * 1000; + const delayMs = normalizeIntervalMinutes(getMinIntervalMinutes(config)) * 60 * 1000; timer = setTimeout(tick, delayMs); timer.unref?.(); }; const runIfEnabled = async () => { const config = await getConfig(); - const github = config?.github; - if (!github?.enabled || getSources(config).length === 0) { + if (!hasAnyEnabled(config)) { return config; } try { await params.runner.runOnce(); } catch (error) { - logger.error('[GitHubSkillSync] Scheduled run failed:', error); + logger.error('[SkillSync] Scheduled run failed:', error); } return getConfig(); }; @@ -87,9 +159,9 @@ export function startGitHubSkillSyncScheduler(params: { void (async () => { const config = await getConfig(); - if (config?.github?.enabled && config.github.runOnStartup && getSources(config).length > 0) { + if (shouldRunOnStartup(config)) { void params.runner.runOnce().catch((error) => { - logger.error('[GitHubSkillSync] Scheduled startup run failed:', error); + logger.error('[SkillSync] Scheduled startup run failed:', error); }); } scheduleNext(config); diff --git a/packages/data-schemas/src/methods/skill.ts b/packages/data-schemas/src/methods/skill.ts index 764fab4b4a..fa684db843 100644 --- a/packages/data-schemas/src/methods/skill.ts +++ b/packages/data-schemas/src/methods/skill.ts @@ -480,7 +480,7 @@ export type CreateSkillInput = { category?: string; author: Types.ObjectId; authorName: string; - source?: 'inline' | 'github' | 'notion'; + source?: 'inline' | 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; sourceMetadata?: Record; /** * When `true`, the skill is auto-primed into every turn. Callers pass this @@ -499,7 +499,7 @@ export type UpdateSkillInput = { frontmatter?: Record; category?: string; alwaysApply?: boolean; - source?: 'inline' | 'github' | 'notion'; + source?: 'inline' | 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; sourceMetadata?: Record; }; @@ -936,12 +936,12 @@ export function createSkillMethods( deleteSkill: (id: string) => Promise<{ deleted: boolean }>; deleteUserSkills: (userId: Types.ObjectId | string) => Promise; findSkillBySourceIdentity: (params: { - source: 'github' | 'notion'; + source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; upstreamId: string; tenantId?: string; }) => Promise<(ISkill & { _id: Types.ObjectId }) | null>; listSkillsBySource: (params: { - source: 'github' | 'notion'; + source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; sourceId: string; }) => Promise>; listSkillFiles: ( @@ -1592,7 +1592,7 @@ export function createSkillMethods( } async function findSkillBySourceIdentity(params: { - source: 'github' | 'notion'; + source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; upstreamId: string; tenantId?: string; }): Promise<(ISkill & { _id: Types.ObjectId }) | null> { @@ -1609,7 +1609,7 @@ export function createSkillMethods( } async function listSkillsBySource(params: { - source: 'github' | 'notion'; + source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; sourceId: string; }): Promise> { const Skill = mongoose.models.Skill as Model; diff --git a/packages/data-schemas/src/types/skill.ts b/packages/data-schemas/src/types/skill.ts index e10c5bfda7..768d1ae7a2 100644 --- a/packages/data-schemas/src/types/skill.ts +++ b/packages/data-schemas/src/types/skill.ts @@ -72,9 +72,12 @@ export interface ISkill { * Provenance of this skill's canonical definition. * - `inline` — authored inside LibreChat. * - `github` — mirrored from a configured GitHub skill sync source. + * - `gitlab` — mirrored from a configured GitLab skill sync source. + * - `bitbucket` — mirrored from a configured Bitbucket skill sync source. + * - `azuredevops` — mirrored from a configured Azure DevOps skill sync source. * - `notion` — reserved for future external sync integrations. */ - source: 'inline' | 'github' | 'notion'; + source: 'inline' | 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion'; /** * Provenance payload keyed by `source`, including upstream identifiers * such as GitHub source id, path, and commit/blob SHAs.