mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-09-01 03:27:01 +00:00
feat: integrate multi-provider adapters into skill sync runner
This commit is contained in:
Malte Polley
parent
9fef1ec3b6
commit
5a7fe0bb86
7 changed files with 1405 additions and 27 deletions
396
packages/api/src/skills/sync/genericSync.spec.ts
Normal file
396
packages/api/src/skills/sync/genericSync.spec.ts
Normal file
|
|
@ -0,0 +1,396 @@
|
|||
import { Types } from 'mongoose';
|
||||
import type {
|
||||
ISkill,
|
||||
ISkillSyncStatus,
|
||||
CreateSkillInput,
|
||||
CreateSkillResult,
|
||||
SkillSyncStatusInput,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { SkillSyncConfig } from 'librechat-data-provider';
|
||||
import type { GitHubSkillSyncDeps } from './github';
|
||||
import { createGitHubSkillSyncRunner } from './github';
|
||||
|
||||
function makeSkill(input: CreateSkillInput): ISkill & { _id: Types.ObjectId } {
|
||||
return {
|
||||
_id: new Types.ObjectId(),
|
||||
name: input.name,
|
||||
description: input.description ?? '',
|
||||
body: input.body ?? '',
|
||||
source: input.source ?? 'inline',
|
||||
sourceMetadata: input.sourceMetadata,
|
||||
author: input.author ?? new Types.ObjectId(),
|
||||
authorName: input.authorName ?? '',
|
||||
version: 1,
|
||||
fileCount: 0,
|
||||
alwaysApply: input.alwaysApply ?? false,
|
||||
tenantId: input.tenantId,
|
||||
} as ISkill & { _id: Types.ObjectId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a mock fetch that emulates a GitLab-like API:
|
||||
* - GET /api/v4/projects/:id/repository/commits/:ref → commit
|
||||
* - GET /api/v4/projects/:id/repository/tree → tree listing
|
||||
* - GET /api/v4/projects/:id/repository/blobs/:sha/raw → blob content
|
||||
*/
|
||||
function gitlabFetch(skillMd: string): typeof fetch {
|
||||
return jest.fn(async (input: RequestInfo | URL) => {
|
||||
const url = input.toString();
|
||||
if (url.includes('/repository/commits/')) {
|
||||
return { ok: true, status: 200, json: async () => ({ id: 'gl-commit-sha', parent_ids: [] }) } as Response;
|
||||
}
|
||||
if (url.includes('/repository/tree')) {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
headers: { get: () => null },
|
||||
json: async () => [
|
||||
{ id: 'gl-skill-sha', name: 'SKILL.md', type: 'blob', path: 'skills/research/SKILL.md', mode: '100644' },
|
||||
],
|
||||
} as unknown as Response;
|
||||
}
|
||||
if (url.includes('/blobs/') && url.includes('/raw')) {
|
||||
const buf = Buffer.from(skillMd);
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength),
|
||||
} as unknown as Response;
|
||||
}
|
||||
return { ok: false, status: 404, json: async () => ({}) } as unknown as Response;
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a mock fetch that emulates a Bitbucket-like API:
|
||||
* - GET /repositories/:ws/:repo/commit/:ref → commit
|
||||
* - GET /repositories/:ws/:repo/src/:ref/ → tree listing
|
||||
* - GET /repositories/:ws/:repo/src/:ref/:path → blob content
|
||||
*/
|
||||
function bitbucketFetch(skillMd: string): typeof fetch {
|
||||
return jest.fn(async (input: RequestInfo | URL) => {
|
||||
const url = input.toString();
|
||||
if (url.includes('/commit/')) {
|
||||
return { ok: true, status: 200, json: async () => ({ hash: 'bb-commit-sha' }) } as Response;
|
||||
}
|
||||
if (url.match(/\/src\/[^/]+\/\?/) || url.match(/\/src\/[^/]+\?/)) {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
values: [
|
||||
{ path: 'skills/writer/SKILL.md', type: 'commit_file', size: skillMd.length, commit: { hash: 'bb-commit-sha' } },
|
||||
],
|
||||
next: undefined,
|
||||
}),
|
||||
} as unknown as Response;
|
||||
}
|
||||
if (url.includes('/src/')) {
|
||||
const buf = Buffer.from(skillMd);
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength),
|
||||
} as unknown as Response;
|
||||
}
|
||||
return { ok: false, status: 404, json: async () => ({}) } as unknown as Response;
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a mock fetch that emulates Azure DevOps API:
|
||||
* - GET /_apis/git/repositories/:repo/refs → refs
|
||||
* - GET /_apis/git/repositories/:repo/commits/:id → commit
|
||||
* - GET /_apis/git/repositories/:repo/trees/:sha → tree listing
|
||||
* - GET /_apis/git/repositories/:repo/blobs/:sha → blob content
|
||||
*/
|
||||
function azureFetch(skillMd: string): typeof fetch {
|
||||
return jest.fn(async (input: RequestInfo | URL) => {
|
||||
const url = input.toString();
|
||||
if (url.includes('/refs?')) {
|
||||
return { ok: true, status: 200, json: async () => ({ value: [{ name: 'refs/heads/main', objectId: 'az-commit-sha' }] }) } as Response;
|
||||
}
|
||||
if (url.includes('/commits/')) {
|
||||
return { ok: true, status: 200, json: async () => ({ commitId: 'az-commit-sha', treeId: 'az-tree-sha' }) } as Response;
|
||||
}
|
||||
if (url.includes('/trees/')) {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
treeEntries: [
|
||||
{ objectId: 'az-blob-sha', relativePath: 'skills/coder/SKILL.md', gitObjectType: 'blob', size: skillMd.length, mode: '100644' },
|
||||
],
|
||||
}),
|
||||
} as unknown as Response;
|
||||
}
|
||||
if (url.includes('/blobs/')) {
|
||||
const buf = Buffer.from(skillMd);
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength),
|
||||
} as unknown as Response;
|
||||
}
|
||||
return { ok: false, status: 404, json: async () => ({}) } as unknown as Response;
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
function createMultiProviderConfig(): SkillSyncConfig {
|
||||
return {
|
||||
github: {
|
||||
enabled: false,
|
||||
intervalMinutes: 60,
|
||||
runOnStartup: false,
|
||||
sources: [],
|
||||
},
|
||||
gitlab: {
|
||||
enabled: true,
|
||||
intervalMinutes: 30,
|
||||
runOnStartup: false,
|
||||
sources: [
|
||||
{
|
||||
id: 'gl-skills',
|
||||
projectId: '12345',
|
||||
ref: 'main',
|
||||
paths: ['skills'],
|
||||
token: '${GITLAB_SKILLS_TOKEN}',
|
||||
},
|
||||
],
|
||||
},
|
||||
bitbucket: {
|
||||
enabled: true,
|
||||
intervalMinutes: 45,
|
||||
runOnStartup: false,
|
||||
sources: [
|
||||
{
|
||||
id: 'bb-skills',
|
||||
workspace: 'myteam',
|
||||
repository: 'skills-repo',
|
||||
ref: 'main',
|
||||
paths: ['skills'],
|
||||
token: '${BITBUCKET_SKILLS_TOKEN}',
|
||||
},
|
||||
],
|
||||
},
|
||||
azuredevops: {
|
||||
enabled: true,
|
||||
intervalMinutes: 60,
|
||||
runOnStartup: false,
|
||||
sources: [
|
||||
{
|
||||
id: 'az-skills',
|
||||
organization: 'myorg',
|
||||
project: 'myproj',
|
||||
repository: 'skills-repo',
|
||||
ref: 'main',
|
||||
paths: ['skills'],
|
||||
token: '${AZUREDEVOPS_SKILLS_TOKEN}',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createDeps(overrides: Partial<GitHubSkillSyncDeps> = {}): GitHubSkillSyncDeps {
|
||||
const statuses: ISkillSyncStatus[] = [];
|
||||
return {
|
||||
getConfig: async () => createMultiProviderConfig(),
|
||||
getCredentialToken: jest.fn(async () => null),
|
||||
getCredentialSummary: jest.fn(async () => null),
|
||||
listCredentials: jest.fn(async () => []),
|
||||
listStatuses: jest.fn(async () => statuses),
|
||||
upsertStatus: jest.fn(async (input: SkillSyncStatusInput) => {
|
||||
const status = {
|
||||
provider: input.provider,
|
||||
sourceId: input.sourceId,
|
||||
tenantId: input.tenantId,
|
||||
status: input.status,
|
||||
credentialKey: input.credentialKey,
|
||||
ref: input.ref,
|
||||
paths: input.paths,
|
||||
startedAt: input.startedAt,
|
||||
finishedAt: input.finishedAt,
|
||||
lastSuccessAt: input.status === 'succeeded' ? input.finishedAt : undefined,
|
||||
lastFailureAt: input.status === 'failed' ? input.finishedAt : undefined,
|
||||
errorCode: input.errorCode,
|
||||
errorMessage: input.errorMessage,
|
||||
syncedSkillCount: input.syncedSkillCount ?? 0,
|
||||
syncedFileCount: input.syncedFileCount ?? 0,
|
||||
deletedSkillCount: input.deletedSkillCount ?? 0,
|
||||
deletedFileCount: input.deletedFileCount ?? 0,
|
||||
} as ISkillSyncStatus;
|
||||
statuses.push(status);
|
||||
return status;
|
||||
}),
|
||||
tryAcquireLock: jest.fn(async () => true),
|
||||
refreshLock: jest.fn(async () => true),
|
||||
releaseLock: jest.fn(async () => undefined),
|
||||
createSkill: jest.fn(async (input: CreateSkillInput): Promise<CreateSkillResult> => {
|
||||
return { skill: makeSkill(input), warnings: [] };
|
||||
}),
|
||||
updateSkill: jest.fn(),
|
||||
getSkillById: jest.fn(),
|
||||
findSkillBySourceIdentity: jest.fn(async () => null),
|
||||
listSkillsBySource: jest.fn(async () => []),
|
||||
listSkillFiles: jest.fn(async () => []),
|
||||
getSkillFileByPath: jest.fn(async () => null),
|
||||
upsertSkillFile: jest.fn(async () => ({
|
||||
_id: new Types.ObjectId(),
|
||||
skillId: new Types.ObjectId(),
|
||||
relativePath: 'file',
|
||||
file_id: 'fid',
|
||||
filename: 'file',
|
||||
filepath: '/uploads/file',
|
||||
source: 'local',
|
||||
mimeType: 'text/plain',
|
||||
bytes: 1,
|
||||
category: 'file',
|
||||
isExecutable: false,
|
||||
author: new Types.ObjectId(),
|
||||
})),
|
||||
deleteSkillFile: jest.fn(async () => ({ deleted: true })),
|
||||
deleteSkill: jest.fn(async () => ({ deleted: true })),
|
||||
saveBuffer: jest.fn(async () => ({ filepath: '/uploads/file', source: 'local' })),
|
||||
deleteFile: jest.fn(async () => undefined),
|
||||
grantPermission: jest.fn(async () => undefined),
|
||||
fetchFn: jest.fn(async () => ({ ok: false, status: 404 })) as unknown as typeof fetch,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('Multi-provider skill sync integration wiring', () => {
|
||||
const originalEnv = process.env;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env = {
|
||||
...originalEnv,
|
||||
GITLAB_SKILLS_TOKEN: 'glpat-test-token',
|
||||
BITBUCKET_SKILLS_TOKEN: 'bb-test-token',
|
||||
AZUREDEVOPS_SKILLS_TOKEN: 'az-test-token',
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
});
|
||||
|
||||
it('syncs a GitLab source when GitHub is disabled', async () => {
|
||||
const glFetch = gitlabFetch('---\nname: research\ndescription: From GitLab\n---\nGL body');
|
||||
const config = createMultiProviderConfig();
|
||||
// Only enable GitLab
|
||||
config.bitbucket!.enabled = false;
|
||||
config.azuredevops!.enabled = false;
|
||||
const deps = createDeps({
|
||||
fetchFn: glFetch,
|
||||
getConfig: async () => config,
|
||||
});
|
||||
const runner = createGitHubSkillSyncRunner(deps);
|
||||
|
||||
const result = await runner.runOnce();
|
||||
|
||||
expect(result.status).toBe('completed');
|
||||
expect(deps.createSkill).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: 'research',
|
||||
description: 'From GitLab',
|
||||
source: 'gitlab',
|
||||
sourceMetadata: expect.objectContaining({
|
||||
provider: 'gitlab',
|
||||
sourceId: 'gl-skills',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('syncs a Bitbucket source when GitHub is disabled', async () => {
|
||||
const bbFetch = bitbucketFetch('---\nname: writer\ndescription: From Bitbucket\n---\nBB body');
|
||||
const config = createMultiProviderConfig();
|
||||
// Only enable Bitbucket
|
||||
config.gitlab!.enabled = false;
|
||||
config.azuredevops!.enabled = false;
|
||||
const deps = createDeps({
|
||||
fetchFn: bbFetch,
|
||||
getConfig: async () => config,
|
||||
});
|
||||
const runner = createGitHubSkillSyncRunner(deps);
|
||||
|
||||
const result = await runner.runOnce();
|
||||
|
||||
expect(result.status).toBe('completed');
|
||||
expect(deps.createSkill).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: 'writer',
|
||||
source: 'bitbucket',
|
||||
sourceMetadata: expect.objectContaining({
|
||||
provider: 'bitbucket',
|
||||
sourceId: 'bb-skills',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('syncs an Azure DevOps source when GitHub is disabled', async () => {
|
||||
const azFetch = azureFetch('---\nname: coder\ndescription: From Azure\n---\nAZ body');
|
||||
const config = createMultiProviderConfig();
|
||||
// Only enable Azure DevOps
|
||||
config.gitlab!.enabled = false;
|
||||
config.bitbucket!.enabled = false;
|
||||
const deps = createDeps({
|
||||
fetchFn: azFetch,
|
||||
getConfig: async () => config,
|
||||
});
|
||||
const runner = createGitHubSkillSyncRunner(deps);
|
||||
|
||||
const result = await runner.runOnce();
|
||||
|
||||
expect(result.status).toBe('completed');
|
||||
expect(deps.createSkill).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: 'coder',
|
||||
source: 'azuredevops',
|
||||
sourceMetadata: expect.objectContaining({
|
||||
provider: 'azuredevops',
|
||||
sourceId: 'az-skills',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('reports failed status when token env var is missing', async () => {
|
||||
delete process.env.GITLAB_SKILLS_TOKEN;
|
||||
const config = createMultiProviderConfig();
|
||||
config.bitbucket!.enabled = false;
|
||||
config.azuredevops!.enabled = false;
|
||||
const deps = createDeps({ getConfig: async () => config });
|
||||
const runner = createGitHubSkillSyncRunner(deps);
|
||||
|
||||
const result = await runner.runOnce();
|
||||
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.sources).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
provider: 'gitlab',
|
||||
sourceId: 'gl-skills',
|
||||
status: 'failed',
|
||||
errorCode: 'MISSING_CREDENTIAL',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('skips disabled providers and syncs only enabled ones', async () => {
|
||||
const config = createMultiProviderConfig();
|
||||
config.gitlab!.enabled = false;
|
||||
config.bitbucket!.enabled = false;
|
||||
config.azuredevops!.enabled = false;
|
||||
const deps = createDeps({ getConfig: async () => config });
|
||||
const runner = createGitHubSkillSyncRunner(deps);
|
||||
|
||||
const result = await runner.runOnce();
|
||||
|
||||
expect(result.status).toBe('skipped');
|
||||
expect(result.sources).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
859
packages/api/src/skills/sync/genericSync.ts
Normal file
859
packages/api/src/skills/sync/genericSync.ts
Normal file
|
|
@ -0,0 +1,859 @@
|
|||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import { Types } from 'mongoose';
|
||||
import { logger, tenantStorage } from '@librechat/data-schemas';
|
||||
import {
|
||||
ResourceType,
|
||||
PrincipalType,
|
||||
AccessRoleIds,
|
||||
SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH,
|
||||
} from 'librechat-data-provider';
|
||||
import type {
|
||||
ISkill,
|
||||
ISkillFile,
|
||||
ISkillSyncStatus,
|
||||
SkillSyncProvider,
|
||||
SkillSyncStatusInput,
|
||||
UpsertSkillFileInput,
|
||||
} from '@librechat/data-schemas';
|
||||
import type { GitRepoAdapter, GitTreeEntry } from './adapter';
|
||||
import type { GitHubSkillSyncDeps } from './github';
|
||||
import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits';
|
||||
import { parseSkillMarkdown } from '../parse';
|
||||
|
||||
const SYSTEM_AUTHOR_ID = new Types.ObjectId('000000000000000000000000');
|
||||
|
||||
type AssertNotCancelled = () => void;
|
||||
|
||||
export type GenericSourceConfig = {
|
||||
id: string;
|
||||
ref: string;
|
||||
paths: string[];
|
||||
skillDiscoveryDepth?: number;
|
||||
credentialKey?: string;
|
||||
token?: string;
|
||||
tenantId?: string;
|
||||
};
|
||||
|
||||
type DiscoveredSkill = {
|
||||
rootPath: string;
|
||||
skillMd: GitTreeEntry;
|
||||
files: GitTreeEntry[];
|
||||
};
|
||||
|
||||
type SyncCounters = {
|
||||
syncedSkillCount: number;
|
||||
syncedFileCount: number;
|
||||
deletedSkillCount: number;
|
||||
deletedFileCount: number;
|
||||
};
|
||||
|
||||
type SaveBufferResult = {
|
||||
filepath: string;
|
||||
source: string;
|
||||
storageKey?: string;
|
||||
storageRegion?: string;
|
||||
};
|
||||
|
||||
type StoredSkillFileRef = {
|
||||
filepath: string;
|
||||
source: string;
|
||||
storageKey?: string;
|
||||
storageRegion?: string;
|
||||
author?: Types.ObjectId | string;
|
||||
tenantId?: string;
|
||||
};
|
||||
|
||||
// --- Pure helpers (duplicated from github.ts to avoid modifying it) ---
|
||||
|
||||
function normalizeRepoPath(value: string): string {
|
||||
const trimmed = value.trim().replace(/^\/+|\/+$/g, '');
|
||||
return trimmed === '.' ? '' : trimmed;
|
||||
}
|
||||
|
||||
function isSafeRelativePath(value: string): boolean {
|
||||
if (!value || value.startsWith('/') || value.startsWith('\\')) {
|
||||
return false;
|
||||
}
|
||||
if (!/^[a-zA-Z0-9._\-/]+$/.test(value)) {
|
||||
return false;
|
||||
}
|
||||
return value.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..');
|
||||
}
|
||||
|
||||
function toSkillName(value: string): string {
|
||||
const normalized = value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.replace(/-{2,}/g, '-');
|
||||
return normalized || 'synced-skill';
|
||||
}
|
||||
|
||||
function getFilename(relativePath: string): string {
|
||||
return path.posix.basename(relativePath);
|
||||
}
|
||||
|
||||
function guessMimeType(filename: string): string {
|
||||
const ext = path.extname(filename).toLowerCase();
|
||||
const mimeMap: Record<string, string> = {
|
||||
'.md': 'text/markdown',
|
||||
'.txt': 'text/plain',
|
||||
'.js': 'application/javascript',
|
||||
'.ts': 'text/typescript',
|
||||
'.json': 'application/json',
|
||||
'.yaml': 'text/yaml',
|
||||
'.yml': 'text/yaml',
|
||||
'.py': 'text/x-python',
|
||||
'.sh': 'application/x-sh',
|
||||
'.css': 'text/css',
|
||||
'.html': 'text/html',
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.webp': 'image/webp',
|
||||
'.pdf': 'application/pdf',
|
||||
};
|
||||
return mimeMap[ext] ?? 'application/octet-stream';
|
||||
}
|
||||
|
||||
function toCleanFrontmatter(
|
||||
frontmatter: Record<string, unknown> | undefined,
|
||||
): Record<string, unknown> {
|
||||
if (!frontmatter) {
|
||||
return {};
|
||||
}
|
||||
const clean = { ...frontmatter };
|
||||
delete clean.name;
|
||||
delete clean.description;
|
||||
if ('always-apply' in clean && typeof clean['always-apply'] !== 'boolean') {
|
||||
delete clean['always-apply'];
|
||||
}
|
||||
if ('alwaysApply' in clean && typeof clean.alwaysApply !== 'boolean') {
|
||||
delete clean.alwaysApply;
|
||||
}
|
||||
return clean;
|
||||
}
|
||||
|
||||
function getProviderAuthorName(provider: SkillSyncProvider): string {
|
||||
switch (provider) {
|
||||
case 'gitlab':
|
||||
return 'GitLab Sync';
|
||||
case 'bitbucket':
|
||||
return 'Bitbucket Sync';
|
||||
case 'azuredevops':
|
||||
return 'Azure DevOps Sync';
|
||||
default:
|
||||
return 'Skill Sync';
|
||||
}
|
||||
}
|
||||
|
||||
function makeUpstreamId(source: GenericSourceConfig, rootPath: string): string {
|
||||
return `${source.id}:${rootPath}`;
|
||||
}
|
||||
|
||||
function makeSourceAuthorId(
|
||||
provider: SkillSyncProvider,
|
||||
source: GenericSourceConfig,
|
||||
): Types.ObjectId {
|
||||
const seed = source.tenantId
|
||||
? `${provider}:${source.id}:${source.tenantId}`
|
||||
: `${provider}:${source.id}`;
|
||||
const digest = crypto.createHash('sha256').update(seed).digest('hex').slice(0, 24);
|
||||
return new Types.ObjectId(digest);
|
||||
}
|
||||
|
||||
function getLimitMegabytes(bytes: number): number {
|
||||
return Math.round(bytes / 1024 / 1024);
|
||||
}
|
||||
|
||||
function assertBlobSize(entry: GitTreeEntry, relativePath: string): number {
|
||||
if (typeof entry.size !== 'number' || !Number.isFinite(entry.size) || entry.size < 0) {
|
||||
// If size not available from tree listing, skip pre-check (will check buffer after fetch)
|
||||
return 0;
|
||||
}
|
||||
if (entry.size > DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes) {
|
||||
throw new GenericSyncError(
|
||||
'BLOB_TOO_LARGE',
|
||||
`File "${relativePath}" exceeds the ${getLimitMegabytes(DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes)}MB per-file limit`,
|
||||
);
|
||||
}
|
||||
return entry.size;
|
||||
}
|
||||
|
||||
function assertBufferSize(buffer: Buffer, relativePath: string): void {
|
||||
if (buffer.length > DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes) {
|
||||
throw new GenericSyncError(
|
||||
'BLOB_TOO_LARGE',
|
||||
`File "${relativePath}" exceeds the ${getLimitMegabytes(DEFAULT_SKILL_IMPORT_LIMITS.maxSingleFileBytes)}MB per-file limit`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertCumulativeSize(totalBytes: number): void {
|
||||
if (totalBytes > DEFAULT_SKILL_IMPORT_LIMITS.maxDecompressedBytes) {
|
||||
throw new GenericSyncError(
|
||||
'PACKAGE_TOO_LARGE',
|
||||
`Skill files exceed the ${getLimitMegabytes(DEFAULT_SKILL_IMPORT_LIMITS.maxDecompressedBytes)}MB cumulative limit`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertEntryCount(discovered: DiscoveredSkill): void {
|
||||
if (discovered.files.length + 1 > DEFAULT_SKILL_IMPORT_LIMITS.maxEntries) {
|
||||
throw new GenericSyncError(
|
||||
'TOO_MANY_FILES',
|
||||
`Skill "${discovered.rootPath}" exceeds the ${DEFAULT_SKILL_IMPORT_LIMITS.maxEntries} file limit`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class GenericSyncError extends Error {
|
||||
code: string;
|
||||
constructor(code: string, message: string) {
|
||||
super(message);
|
||||
this.name = 'GenericSyncError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizeError(error: unknown): { code: string; message: string } {
|
||||
if (error instanceof GenericSyncError) {
|
||||
return { code: error.code, message: error.message };
|
||||
}
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
code: 'SYNC_FAILED',
|
||||
message: error.message.replace(/Bearer\s+\S+/gi, 'Bearer [redacted]'),
|
||||
};
|
||||
}
|
||||
return { code: 'SYNC_FAILED', message: 'Unknown skill sync failure' };
|
||||
}
|
||||
|
||||
function getTokenEnvVarName(tokenReference: string | undefined): string | null {
|
||||
const match = tokenReference?.trim().match(/^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function resolveToken(
|
||||
deps: GitHubSkillSyncDeps,
|
||||
provider: SkillSyncProvider,
|
||||
source: GenericSourceConfig,
|
||||
): Promise<string | null> {
|
||||
if (deps.allowServerCredentials === false) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
const tokenEnvVar = getTokenEnvVarName(source.token);
|
||||
if (tokenEnvVar) {
|
||||
return Promise.resolve(process.env[tokenEnvVar]?.trim() || null);
|
||||
}
|
||||
if (!source.credentialKey) {
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
return deps.getCredentialToken(provider, source.credentialKey);
|
||||
}
|
||||
|
||||
function isSkillRootWithinDiscoveryDepth(
|
||||
rootPath: string,
|
||||
basePath: string,
|
||||
maxDepth: number,
|
||||
): boolean {
|
||||
if (rootPath === basePath) {
|
||||
return true;
|
||||
}
|
||||
if (basePath && !rootPath.startsWith(`${basePath}/`)) {
|
||||
return false;
|
||||
}
|
||||
const relative = basePath ? rootPath.slice(basePath.length).replace(/^\/+/, '') : rootPath;
|
||||
if (!relative) {
|
||||
return true;
|
||||
}
|
||||
return relative.split('/').length <= maxDepth;
|
||||
}
|
||||
|
||||
function filterTreeByPaths(tree: GitTreeEntry[], paths: string[]): GitTreeEntry[] {
|
||||
const normalizedPaths = paths.map(normalizeRepoPath);
|
||||
return tree.filter((entry) => {
|
||||
const entryPath = normalizeRepoPath(entry.path);
|
||||
return normalizedPaths.some(
|
||||
(basePath) =>
|
||||
basePath === '' || entryPath === basePath || entryPath.startsWith(`${basePath}/`),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function discoverSkills(tree: GitTreeEntry[], source: GenericSourceConfig): DiscoveredSkill[] {
|
||||
const basePaths = source.paths.map(normalizeRepoPath);
|
||||
const skillDiscoveryDepth = source.skillDiscoveryDepth ?? SKILL_SYNC_DEFAULT_DISCOVERY_DEPTH;
|
||||
const skillMdByRoot = new Map<string, GitTreeEntry>();
|
||||
|
||||
for (const entry of tree) {
|
||||
if (entry.type !== 'blob') {
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeRepoPath(entry.path);
|
||||
const basename = path.posix.basename(normalized);
|
||||
if (basename.toUpperCase() !== 'SKILL.MD') {
|
||||
continue;
|
||||
}
|
||||
const parent = normalizeRepoPath(path.posix.dirname(normalized));
|
||||
for (const basePath of basePaths) {
|
||||
if (isSkillRootWithinDiscoveryDepth(parent, basePath, skillDiscoveryDepth)) {
|
||||
skillMdByRoot.set(parent, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const skillRoots = [...skillMdByRoot.keys()];
|
||||
return [...skillMdByRoot.entries()].map(([rootPath, skillMd]) => {
|
||||
const prefix = rootPath ? `${rootPath}/` : '';
|
||||
const childSkillRoots = skillRoots.filter(
|
||||
(c) => c && c !== rootPath && (rootPath ? c.startsWith(`${rootPath}/`) : true),
|
||||
);
|
||||
const files = tree.filter((entry) => {
|
||||
if (entry.type !== 'blob') {
|
||||
return false;
|
||||
}
|
||||
const normalized = normalizeRepoPath(entry.path);
|
||||
if (!normalized.startsWith(prefix) || normalized === skillMd.path) {
|
||||
return false;
|
||||
}
|
||||
if (childSkillRoots.some((childRoot) => normalized.startsWith(`${childRoot}/`))) {
|
||||
return false;
|
||||
}
|
||||
const relativePath = prefix ? normalized.slice(prefix.length) : normalized;
|
||||
return isSafeRelativePath(relativePath) && relativePath.toUpperCase() !== 'SKILL.MD';
|
||||
});
|
||||
return { rootPath, skillMd, files };
|
||||
});
|
||||
}
|
||||
|
||||
function getDiscoveredRelativePath(discovered: DiscoveredSkill, entry: GitTreeEntry): string {
|
||||
const prefix = discovered.rootPath ? `${discovered.rootPath}/` : '';
|
||||
const normalized = normalizeRepoPath(entry.path);
|
||||
return prefix ? normalized.slice(prefix.length) : normalized;
|
||||
}
|
||||
|
||||
function makeStatusInput(params: {
|
||||
provider: SkillSyncProvider;
|
||||
source: GenericSourceConfig;
|
||||
status: SkillSyncStatusInput['status'];
|
||||
startedAt?: Date;
|
||||
finishedAt?: Date;
|
||||
errorCode?: string;
|
||||
errorMessage?: string;
|
||||
counts?: Partial<SyncCounters>;
|
||||
}): SkillSyncStatusInput {
|
||||
return {
|
||||
provider: params.provider,
|
||||
sourceId: params.source.id,
|
||||
tenantId: params.source.tenantId,
|
||||
status: params.status,
|
||||
credentialKey: params.source.credentialKey,
|
||||
ref: params.source.ref,
|
||||
paths: params.source.paths,
|
||||
startedAt: params.startedAt,
|
||||
finishedAt: params.finishedAt,
|
||||
errorCode: params.errorCode,
|
||||
errorMessage: params.errorMessage,
|
||||
syncedSkillCount: params.counts?.syncedSkillCount ?? 0,
|
||||
syncedFileCount: params.counts?.syncedFileCount ?? 0,
|
||||
deletedSkillCount: params.counts?.deletedSkillCount ?? 0,
|
||||
deletedFileCount: params.counts?.deletedFileCount ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function getSourceMetadataString(
|
||||
row: { sourceMetadata?: Record<string, unknown> },
|
||||
key: string,
|
||||
): string | undefined {
|
||||
const metadata = row.sourceMetadata;
|
||||
const value = metadata && typeof metadata === 'object' ? metadata[key] : undefined;
|
||||
return typeof value === 'string' ? value : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the blob reference to pass to adapter.fetchBlob().
|
||||
* Bitbucket requires "commitSha:filePath" format; others use the entry sha directly.
|
||||
*/
|
||||
function makeBlobRef(
|
||||
provider: SkillSyncProvider,
|
||||
entry: GitTreeEntry,
|
||||
commitSha: string,
|
||||
): string {
|
||||
if (provider === 'bitbucket') {
|
||||
return `${commitSha}:${entry.path}`;
|
||||
}
|
||||
return entry.sha;
|
||||
}
|
||||
|
||||
async function ensurePublicViewer(
|
||||
deps: GitHubSkillSyncDeps,
|
||||
skillId: Types.ObjectId,
|
||||
): Promise<void> {
|
||||
await deps.grantPermission({
|
||||
principalType: PrincipalType.PUBLIC,
|
||||
principalId: null,
|
||||
resourceType: ResourceType.SKILL,
|
||||
resourceId: skillId,
|
||||
accessRoleId: AccessRoleIds.SKILL_VIEWER,
|
||||
grantedBy: SYSTEM_AUTHOR_ID,
|
||||
});
|
||||
}
|
||||
|
||||
export async function syncGenericSource(params: {
|
||||
deps: GitHubSkillSyncDeps;
|
||||
adapter: GitRepoAdapter;
|
||||
provider: SkillSyncProvider;
|
||||
source: GenericSourceConfig;
|
||||
assertNotCancelled: AssertNotCancelled;
|
||||
}): Promise<ISkillSyncStatus> {
|
||||
const { deps, adapter, provider, source, assertNotCancelled } = params;
|
||||
const startedAt = new Date();
|
||||
await deps.upsertStatus(makeStatusInput({ provider, source, status: 'running', startedAt }));
|
||||
|
||||
try {
|
||||
assertNotCancelled();
|
||||
const token = await resolveToken(deps, provider, source);
|
||||
assertNotCancelled();
|
||||
if (!token) {
|
||||
throw new GenericSyncError(
|
||||
'MISSING_CREDENTIAL',
|
||||
`Missing ${provider} credential for source "${source.id}"`,
|
||||
);
|
||||
}
|
||||
|
||||
const { commitSha, treeSha } = await adapter.getTreeSha(source.ref);
|
||||
assertNotCancelled();
|
||||
|
||||
const fullTree = await adapter.listTree(treeSha, true);
|
||||
assertNotCancelled();
|
||||
|
||||
const tree = filterTreeByPaths(fullTree, source.paths);
|
||||
const discoveredSkills = discoverSkills(tree, source);
|
||||
const authorId = makeSourceAuthorId(provider, source);
|
||||
const authorName = getProviderAuthorName(provider);
|
||||
const syncedAt = new Date();
|
||||
const seenUpstreamIds = new Set<string>();
|
||||
const counts: SyncCounters = {
|
||||
syncedSkillCount: 0,
|
||||
syncedFileCount: 0,
|
||||
deletedSkillCount: 0,
|
||||
deletedFileCount: 0,
|
||||
};
|
||||
|
||||
for (const discovered of discoveredSkills) {
|
||||
assertNotCancelled();
|
||||
assertEntryCount(discovered);
|
||||
|
||||
const skillMdPath = discovered.rootPath
|
||||
? `${discovered.rootPath}/SKILL.md`
|
||||
: 'SKILL.md';
|
||||
const blobRef = makeBlobRef(provider, discovered.skillMd, commitSha);
|
||||
const skillMdBuffer = await adapter.fetchBlob(blobRef);
|
||||
assertNotCancelled();
|
||||
assertBufferSize(skillMdBuffer, skillMdPath);
|
||||
|
||||
const skillMdContent = skillMdBuffer.toString('utf-8');
|
||||
const parsed = parseSkillMarkdown(skillMdContent);
|
||||
if (parsed.parseError) {
|
||||
throw new GenericSyncError(
|
||||
'SKILL_PARSE_FAILED',
|
||||
`${skillMdPath} contains invalid YAML frontmatter: ${parsed.parseError}`,
|
||||
);
|
||||
}
|
||||
|
||||
const upstreamId = makeUpstreamId(source, discovered.rootPath);
|
||||
seenUpstreamIds.add(upstreamId);
|
||||
const fallbackName = toSkillName(path.posix.basename(discovered.rootPath) || source.id);
|
||||
|
||||
const sourceMetadata = {
|
||||
provider,
|
||||
sourceId: source.id,
|
||||
upstreamId,
|
||||
ref: source.ref,
|
||||
skillPath: discovered.rootPath,
|
||||
commitSha,
|
||||
skillBlobSha: discovered.skillMd.sha,
|
||||
syncedAt: syncedAt.toISOString(),
|
||||
syncStatus: 'synced',
|
||||
};
|
||||
|
||||
const update = {
|
||||
name: parsed.name || fallbackName,
|
||||
description: parsed.description || parsed.name || fallbackName,
|
||||
body: skillMdContent,
|
||||
frontmatter: toCleanFrontmatter(parsed.frontmatter),
|
||||
alwaysApply: parsed.alwaysApply,
|
||||
source: provider,
|
||||
sourceMetadata,
|
||||
};
|
||||
|
||||
const sourceTenantId = source.tenantId ?? undefined;
|
||||
const existing = await deps.findSkillBySourceIdentity({
|
||||
source: provider,
|
||||
upstreamId,
|
||||
tenantId: sourceTenantId,
|
||||
});
|
||||
|
||||
let skill: ISkill & { _id: Types.ObjectId };
|
||||
if (existing && (existing.tenantId ?? undefined) === sourceTenantId) {
|
||||
const result = await deps.updateSkill({
|
||||
id: existing._id.toString(),
|
||||
expectedVersion: existing.version,
|
||||
update,
|
||||
});
|
||||
if (result.status === 'updated') {
|
||||
skill = result.skill;
|
||||
} else {
|
||||
throw new GenericSyncError('SKILL_CONFLICT', `Skill "${existing.name}" conflict during sync`);
|
||||
}
|
||||
} else {
|
||||
const created = await deps.createSkill({
|
||||
...update,
|
||||
name: update.name ?? fallbackName,
|
||||
description: update.description ?? fallbackName,
|
||||
author: authorId,
|
||||
authorName,
|
||||
source: provider,
|
||||
tenantId: source.tenantId,
|
||||
});
|
||||
skill = created.skill;
|
||||
}
|
||||
|
||||
await ensurePublicViewer(deps, skill._id);
|
||||
|
||||
// Sync files
|
||||
const remotePaths = new Set<string>();
|
||||
let totalFileBytes = 0;
|
||||
for (const entry of discovered.files) {
|
||||
assertNotCancelled();
|
||||
const relativePath = getDiscoveredRelativePath(discovered, entry);
|
||||
if (!isSafeRelativePath(relativePath) || relativePath.toUpperCase() === 'SKILL.MD') {
|
||||
continue;
|
||||
}
|
||||
totalFileBytes += assertBlobSize(entry, relativePath);
|
||||
assertCumulativeSize(totalFileBytes);
|
||||
remotePaths.add(relativePath);
|
||||
|
||||
const existingFile = await deps.getSkillFileByPath(skill._id, relativePath);
|
||||
if (existingFile && getSourceMetadataString(existingFile, 'blobSha') === entry.sha) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fileBlobRef = makeBlobRef(provider, entry, commitSha);
|
||||
const buffer = await adapter.fetchBlob(fileBlobRef);
|
||||
assertNotCancelled();
|
||||
assertBufferSize(buffer, relativePath);
|
||||
|
||||
const fileId = crypto.randomUUID();
|
||||
const filename = getFilename(relativePath);
|
||||
const mimeType = guessMimeType(filename);
|
||||
const saved = await deps.saveBuffer({
|
||||
userId: skill.author.toString(),
|
||||
buffer,
|
||||
fileName: `${fileId}__${filename}`,
|
||||
basePath: 'uploads',
|
||||
isImage: mimeType.startsWith('image/'),
|
||||
tenantId: skill.tenantId,
|
||||
});
|
||||
|
||||
await deps.upsertSkillFile({
|
||||
skillId: skill._id,
|
||||
relativePath,
|
||||
file_id: fileId,
|
||||
filename,
|
||||
filepath: saved.filepath,
|
||||
storageKey: saved.storageKey,
|
||||
storageRegion: saved.storageRegion,
|
||||
source: saved.source,
|
||||
sourceMetadata: {
|
||||
provider,
|
||||
sourceId: source.id,
|
||||
upstreamId,
|
||||
commitSha,
|
||||
blobSha: entry.sha,
|
||||
path: entry.path,
|
||||
},
|
||||
mimeType,
|
||||
bytes: buffer.length,
|
||||
isExecutable: false,
|
||||
author: skill.author,
|
||||
tenantId: skill.tenantId,
|
||||
});
|
||||
counts.syncedFileCount++;
|
||||
}
|
||||
|
||||
// Delete files no longer in remote
|
||||
const existingFiles = await deps.listSkillFiles(skill._id);
|
||||
for (const file of existingFiles) {
|
||||
if (remotePaths.has(file.relativePath)) {
|
||||
continue;
|
||||
}
|
||||
const result = await deps.deleteSkillFile(skill._id, file.relativePath);
|
||||
if (result.deleted) {
|
||||
counts.deletedFileCount++;
|
||||
if (deps.deleteFile) {
|
||||
await deps.deleteFile({
|
||||
filepath: file.filepath,
|
||||
source: file.source,
|
||||
storageKey: file.storageKey,
|
||||
storageRegion: file.storageRegion,
|
||||
user: file.author,
|
||||
tenantId: file.tenantId,
|
||||
}).catch((e) => logger.error(`[${provider}Sync] Failed to clean up file:`, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
counts.syncedSkillCount++;
|
||||
}
|
||||
|
||||
// Delete stale skills no longer in remote
|
||||
const currentSyncedSkills = await deps.listSkillsBySource({
|
||||
source: provider,
|
||||
sourceId: source.id,
|
||||
});
|
||||
const sourceTenantId = source.tenantId ?? undefined;
|
||||
for (const skill of currentSyncedSkills) {
|
||||
assertNotCancelled();
|
||||
if ((skill.tenantId ?? undefined) !== sourceTenantId) {
|
||||
continue;
|
||||
}
|
||||
const upstreamId = getSourceMetadataString(skill, 'upstreamId') ?? '';
|
||||
if (seenUpstreamIds.has(upstreamId)) {
|
||||
continue;
|
||||
}
|
||||
// Delete skill and its files
|
||||
const files = await deps.listSkillFiles(skill._id);
|
||||
for (const file of files) {
|
||||
if (deps.deleteFile) {
|
||||
await deps.deleteFile({
|
||||
filepath: file.filepath,
|
||||
source: file.source,
|
||||
storageKey: file.storageKey,
|
||||
storageRegion: file.storageRegion,
|
||||
user: file.author,
|
||||
tenantId: file.tenantId,
|
||||
}).catch((e) => logger.error(`[${provider}Sync] Failed to clean up stale file:`, e));
|
||||
}
|
||||
}
|
||||
await deps.deleteSkill(skill._id.toString());
|
||||
counts.deletedSkillCount++;
|
||||
counts.deletedFileCount += files.length;
|
||||
}
|
||||
|
||||
return deps.upsertStatus(
|
||||
makeStatusInput({
|
||||
provider,
|
||||
source,
|
||||
status: 'succeeded',
|
||||
startedAt,
|
||||
finishedAt: new Date(),
|
||||
counts,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
const sanitized = sanitizeError(error);
|
||||
logger.error(`[${provider}Sync] Source "${source.id}" failed: ${sanitized.message}`);
|
||||
return deps.upsertStatus(
|
||||
makeStatusInput({
|
||||
provider,
|
||||
source,
|
||||
status: 'failed',
|
||||
startedAt,
|
||||
finishedAt: new Date(),
|
||||
errorCode: sanitized.code,
|
||||
errorMessage: sanitized.message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function syncGenericSourceInTenantContext(params: {
|
||||
deps: GitHubSkillSyncDeps;
|
||||
adapter: GitRepoAdapter;
|
||||
provider: SkillSyncProvider;
|
||||
source: GenericSourceConfig;
|
||||
assertNotCancelled: AssertNotCancelled;
|
||||
}): Promise<ISkillSyncStatus> {
|
||||
if (!params.source.tenantId) {
|
||||
return syncGenericSource(params);
|
||||
}
|
||||
return tenantStorage.run({ tenantId: params.source.tenantId }, async () =>
|
||||
syncGenericSource(params),
|
||||
);
|
||||
}
|
||||
|
||||
export { syncGenericSourceInTenantContext };
|
||||
|
||||
/**
|
||||
* Called from the main runner's runOnce() to sync all non-GitHub providers.
|
||||
* Reads gitlab/bitbucket/azuredevops from the config and syncs each enabled source.
|
||||
*/
|
||||
export async function syncNonGitHubProviders(params: {
|
||||
deps: GitHubSkillSyncDeps;
|
||||
fetchFn: typeof fetch;
|
||||
assertNotCancelled: AssertNotCancelled;
|
||||
}): Promise<ISkillSyncStatus[]> {
|
||||
const { deps, fetchFn, assertNotCancelled } = params;
|
||||
const config = await deps.getConfig();
|
||||
if (!config) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const results: ISkillSyncStatus[] = [];
|
||||
|
||||
// GitLab
|
||||
if (config.gitlab?.enabled && config.gitlab.sources.length > 0) {
|
||||
for (const source of config.gitlab.sources) {
|
||||
assertNotCancelled();
|
||||
const token = await resolveToken(deps, 'gitlab', source);
|
||||
if (!token) {
|
||||
results.push(
|
||||
await deps.upsertStatus(
|
||||
makeStatusInput({
|
||||
provider: 'gitlab',
|
||||
source,
|
||||
status: 'failed',
|
||||
startedAt: new Date(),
|
||||
finishedAt: new Date(),
|
||||
errorCode: 'MISSING_CREDENTIAL',
|
||||
errorMessage: `Missing gitlab credential for source "${source.id}"`,
|
||||
}),
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const { createRepoAdapter } = await import('./adapterFactory');
|
||||
const adapter = createRepoAdapter({
|
||||
provider: 'gitlab',
|
||||
token,
|
||||
projectId: source.projectId,
|
||||
baseUrl: source.baseUrl,
|
||||
fetchFn,
|
||||
});
|
||||
results.push(
|
||||
await syncGenericSourceInTenantContext({
|
||||
deps,
|
||||
adapter,
|
||||
provider: 'gitlab',
|
||||
source: {
|
||||
id: source.id,
|
||||
ref: source.ref,
|
||||
paths: source.paths,
|
||||
skillDiscoveryDepth: source.skillDiscoveryDepth,
|
||||
credentialKey: source.credentialKey,
|
||||
token: source.token,
|
||||
tenantId: source.tenantId,
|
||||
},
|
||||
assertNotCancelled,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Bitbucket
|
||||
if (config.bitbucket?.enabled && config.bitbucket.sources.length > 0) {
|
||||
for (const source of config.bitbucket.sources) {
|
||||
assertNotCancelled();
|
||||
const token = await resolveToken(deps, 'bitbucket', source);
|
||||
if (!token) {
|
||||
results.push(
|
||||
await deps.upsertStatus(
|
||||
makeStatusInput({
|
||||
provider: 'bitbucket',
|
||||
source,
|
||||
status: 'failed',
|
||||
startedAt: new Date(),
|
||||
finishedAt: new Date(),
|
||||
errorCode: 'MISSING_CREDENTIAL',
|
||||
errorMessage: `Missing bitbucket credential for source "${source.id}"`,
|
||||
}),
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const { createRepoAdapter } = await import('./adapterFactory');
|
||||
const adapter = createRepoAdapter({
|
||||
provider: 'bitbucket',
|
||||
token,
|
||||
workspace: source.workspace,
|
||||
repository: source.repository,
|
||||
fetchFn,
|
||||
});
|
||||
results.push(
|
||||
await syncGenericSourceInTenantContext({
|
||||
deps,
|
||||
adapter,
|
||||
provider: 'bitbucket',
|
||||
source: {
|
||||
id: source.id,
|
||||
ref: source.ref,
|
||||
paths: source.paths,
|
||||
skillDiscoveryDepth: source.skillDiscoveryDepth,
|
||||
credentialKey: source.credentialKey,
|
||||
token: source.token,
|
||||
tenantId: source.tenantId,
|
||||
},
|
||||
assertNotCancelled,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Azure DevOps
|
||||
if (config.azuredevops?.enabled && config.azuredevops.sources.length > 0) {
|
||||
for (const source of config.azuredevops.sources) {
|
||||
assertNotCancelled();
|
||||
const token = await resolveToken(deps, 'azuredevops', source);
|
||||
if (!token) {
|
||||
results.push(
|
||||
await deps.upsertStatus(
|
||||
makeStatusInput({
|
||||
provider: 'azuredevops',
|
||||
source,
|
||||
status: 'failed',
|
||||
startedAt: new Date(),
|
||||
finishedAt: new Date(),
|
||||
errorCode: 'MISSING_CREDENTIAL',
|
||||
errorMessage: `Missing azuredevops credential for source "${source.id}"`,
|
||||
}),
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const { createRepoAdapter } = await import('./adapterFactory');
|
||||
const adapter = createRepoAdapter({
|
||||
provider: 'azuredevops',
|
||||
token,
|
||||
organization: source.organization,
|
||||
project: source.project,
|
||||
repository: source.repository,
|
||||
baseUrl: source.baseUrl,
|
||||
fetchFn,
|
||||
});
|
||||
results.push(
|
||||
await syncGenericSourceInTenantContext({
|
||||
deps,
|
||||
adapter,
|
||||
provider: 'azuredevops',
|
||||
source: {
|
||||
id: source.id,
|
||||
ref: source.ref,
|
||||
paths: source.paths,
|
||||
skillDiscoveryDepth: source.skillDiscoveryDepth,
|
||||
credentialKey: source.credentialKey,
|
||||
token: source.token,
|
||||
tenantId: source.tenantId,
|
||||
},
|
||||
assertNotCancelled,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
|
@ -24,6 +24,7 @@ import type {
|
|||
import type { SkillSyncConfig, SkillSyncGitHubSourceConfig } from 'librechat-data-provider';
|
||||
import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits';
|
||||
import { parseSkillMarkdown } from '../parse';
|
||||
import { syncNonGitHubProviders } from './genericSync';
|
||||
|
||||
const GITHUB_API_BASE = 'https://api.github.com';
|
||||
const SYSTEM_AUTHOR_ID = new Types.ObjectId('000000000000000000000000');
|
||||
|
|
@ -171,12 +172,12 @@ export type GitHubSkillSyncDeps = {
|
|||
}) => Promise<UpdateSkillResult>;
|
||||
getSkillById: (id: string | Types.ObjectId) => Promise<(ISkill & { _id: Types.ObjectId }) | null>;
|
||||
findSkillBySourceIdentity: (params: {
|
||||
source: 'github' | 'notion';
|
||||
source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
upstreamId: string;
|
||||
tenantId?: string;
|
||||
}) => Promise<(ISkill & { _id: Types.ObjectId }) | null>;
|
||||
listSkillsBySource: (params: {
|
||||
source: 'github' | 'notion';
|
||||
source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
sourceId: string;
|
||||
}) => Promise<Array<ISkill & { _id: Types.ObjectId }>>;
|
||||
listSkillFiles: (
|
||||
|
|
@ -1790,8 +1791,16 @@ export function createGitHubSkillSyncRunner(deps: GitHubSkillSyncDeps): GitHubSk
|
|||
|
||||
async function runOnce(): Promise<GitHubSkillSyncRunResult> {
|
||||
const github = getGithubConfig(await deps.getConfig());
|
||||
if (!github.enabled || github.sources.length === 0) {
|
||||
return { status: 'skipped', message: 'GitHub skill sync is disabled', sources: [] };
|
||||
const githubEnabled = github.enabled && github.sources.length > 0;
|
||||
if (!githubEnabled) {
|
||||
// Even if GitHub is disabled, process other providers
|
||||
const assertNotCancelled = () => {};
|
||||
const genericSources = await syncNonGitHubProviders({ deps, fetchFn, assertNotCancelled });
|
||||
if (genericSources.length === 0) {
|
||||
return { status: 'skipped', message: 'Skill sync is disabled', sources: [] };
|
||||
}
|
||||
const failed = genericSources.some((s) => s.status === 'failed');
|
||||
return { status: failed ? 'failed' : 'completed', sources: genericSources };
|
||||
}
|
||||
const allowServerCredentials = deps.allowServerCredentials !== false;
|
||||
if (!allowServerCredentials) {
|
||||
|
|
@ -1856,6 +1865,13 @@ export function createGitHubSkillSyncRunner(deps: GitHubSkillSyncDeps): GitHubSk
|
|||
await syncSourceInTenantContext({ deps, source, fetchFn, assertNotCancelled }),
|
||||
);
|
||||
}
|
||||
|
||||
// Sync non-GitHub providers
|
||||
if (!lockLost) {
|
||||
const genericSources = await syncNonGitHubProviders({ deps, fetchFn, assertNotCancelled });
|
||||
sources.push(...genericSources);
|
||||
}
|
||||
|
||||
const failed = sources.some((source) => source.status === 'failed');
|
||||
return {
|
||||
status: failed || lockLost ? 'failed' : 'completed',
|
||||
|
|
|
|||
|
|
@ -77,6 +77,25 @@ function hasGitHubConfig(config: SkillSyncConfig | undefined): config is SkillSy
|
|||
return Boolean(config?.github);
|
||||
}
|
||||
|
||||
function hasAnyEnabledProvider(config: SkillSyncConfig | undefined): boolean {
|
||||
if (!config) {
|
||||
return false;
|
||||
}
|
||||
if (config.github?.enabled && config.github.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config.gitlab?.enabled && config.gitlab.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config.bitbucket?.enabled && config.bitbucket.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config.azuredevops?.enabled && config.azuredevops.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isSameSkillSyncConfig(
|
||||
left: SkillSyncConfig | undefined,
|
||||
right: SkillSyncConfig | undefined,
|
||||
|
|
@ -153,13 +172,18 @@ function getRequestSkillSyncConfig(
|
|||
logger: SkillSyncTriggerLogger,
|
||||
): SkillSyncConfig | undefined {
|
||||
const resolved = parseSkillSyncConfig(appConfig?.skillSync, logger);
|
||||
if (
|
||||
!hasGitHubConfig(resolved) ||
|
||||
!resolved.github.enabled ||
|
||||
resolved.github.sources.length === 0
|
||||
) {
|
||||
if (!hasAnyEnabledProvider(resolved)) {
|
||||
return undefined;
|
||||
}
|
||||
if (!hasGitHubConfig(resolved)) {
|
||||
// Non-GitHub providers are configured but no GitHub — pass through the
|
||||
// full config so the runner can process them via syncNonGitHubProviders.
|
||||
const base = parseSkillSyncConfig(appConfig?.config?.skillSync, logger);
|
||||
if (isSameSkillSyncConfig(resolved, base)) {
|
||||
return undefined;
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
const base = parseSkillSyncConfig(appConfig?.config?.skillSync, logger);
|
||||
if (isSameSkillSyncConfig(resolved, base)) {
|
||||
|
|
@ -268,11 +292,13 @@ export function createSkillSyncTriggerOrchestrator(
|
|||
|
||||
async function maybeRunForRequest(request: SkillSyncRequestLike): Promise<boolean> {
|
||||
const config = getRequestSkillSyncConfig(request.config, request.user, deps.logger);
|
||||
if (!hasGitHubConfig(config)) {
|
||||
if (!config) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const syncKey = getRequestSyncKey(config, request.user);
|
||||
const syncKey = hasGitHubConfig(config)
|
||||
? getRequestSyncKey(config, request.user)
|
||||
: `nongithub:${JSON.stringify(request.user?.tenantId ?? '')}`;
|
||||
if (inFlight.has(syncKey)) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -282,9 +308,15 @@ export function createSkillSyncTriggerOrchestrator(
|
|||
loadAppConfig: async () => request.config,
|
||||
allowServerCredentials: Boolean(request.skillSyncAllowServerCredentials),
|
||||
});
|
||||
const status = await requestRunner.getStatus();
|
||||
if (!shouldRunRequestSync(status, { minIntervalMs, staleRunningMs })) {
|
||||
return false;
|
||||
|
||||
// For configs with GitHub sources, use the status-based interval check.
|
||||
// For non-GitHub-only configs, run unconditionally (interval gating happens
|
||||
// inside the generic sync via the per-source status timestamps).
|
||||
if (hasGitHubConfig(config)) {
|
||||
const status = await requestRunner.getStatus();
|
||||
if (!shouldRunRequestSync(status, { minIntervalMs, staleRunningMs })) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
inFlight.add(syncKey);
|
||||
|
|
|
|||
|
|
@ -28,6 +28,79 @@ function getSources(config: SkillSyncConfig | undefined) {
|
|||
return Array.isArray(sources) ? sources : [];
|
||||
}
|
||||
|
||||
function hasAnySources(config: SkillSyncConfig | undefined): boolean {
|
||||
if (getSources(config).length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config?.gitlab?.enabled && config.gitlab.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config?.bitbucket?.enabled && config.bitbucket.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config?.azuredevops?.enabled && config.azuredevops.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasAnyEnabled(config: SkillSyncConfig | undefined): boolean {
|
||||
if (config?.github?.enabled && getSources(config).length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config?.gitlab?.enabled && config.gitlab.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config?.bitbucket?.enabled && config.bitbucket.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config?.azuredevops?.enabled && config.azuredevops.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function getMinIntervalMinutes(config: SkillSyncConfig | undefined): number {
|
||||
const intervals: number[] = [];
|
||||
if (config?.github?.enabled) {
|
||||
intervals.push(config.github.intervalMinutes ?? 60);
|
||||
}
|
||||
if (config?.gitlab?.enabled) {
|
||||
intervals.push(config.gitlab.intervalMinutes ?? 60);
|
||||
}
|
||||
if (config?.bitbucket?.enabled) {
|
||||
intervals.push(config.bitbucket.intervalMinutes ?? 60);
|
||||
}
|
||||
if (config?.azuredevops?.enabled) {
|
||||
intervals.push(config.azuredevops.intervalMinutes ?? 60);
|
||||
}
|
||||
return intervals.length > 0 ? Math.min(...intervals) : 60;
|
||||
}
|
||||
|
||||
function shouldRunOnStartup(config: SkillSyncConfig | undefined): boolean {
|
||||
if (config?.github?.enabled && config.github.runOnStartup && getSources(config).length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (config?.gitlab?.enabled && config.gitlab.runOnStartup && config.gitlab.sources.length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
config?.bitbucket?.enabled &&
|
||||
config.bitbucket.runOnStartup &&
|
||||
config.bitbucket.sources.length > 0
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
config?.azuredevops?.enabled &&
|
||||
config.azuredevops.runOnStartup &&
|
||||
config.azuredevops.sources.length > 0
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function startGitHubSkillSyncScheduler(params: {
|
||||
getConfig: () => MaybePromise<SkillSyncConfig | undefined>;
|
||||
runner: GitHubSkillSyncRunner;
|
||||
|
|
@ -48,21 +121,20 @@ export function startGitHubSkillSyncScheduler(params: {
|
|||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
const delayMs = normalizeIntervalMinutes(config?.github?.intervalMinutes) * 60 * 1000;
|
||||
const delayMs = normalizeIntervalMinutes(getMinIntervalMinutes(config)) * 60 * 1000;
|
||||
timer = setTimeout(tick, delayMs);
|
||||
timer.unref?.();
|
||||
};
|
||||
|
||||
const runIfEnabled = async () => {
|
||||
const config = await getConfig();
|
||||
const github = config?.github;
|
||||
if (!github?.enabled || getSources(config).length === 0) {
|
||||
if (!hasAnyEnabled(config)) {
|
||||
return config;
|
||||
}
|
||||
try {
|
||||
await params.runner.runOnce();
|
||||
} catch (error) {
|
||||
logger.error('[GitHubSkillSync] Scheduled run failed:', error);
|
||||
logger.error('[SkillSync] Scheduled run failed:', error);
|
||||
}
|
||||
return getConfig();
|
||||
};
|
||||
|
|
@ -87,9 +159,9 @@ export function startGitHubSkillSyncScheduler(params: {
|
|||
|
||||
void (async () => {
|
||||
const config = await getConfig();
|
||||
if (config?.github?.enabled && config.github.runOnStartup && getSources(config).length > 0) {
|
||||
if (shouldRunOnStartup(config)) {
|
||||
void params.runner.runOnce().catch((error) => {
|
||||
logger.error('[GitHubSkillSync] Scheduled startup run failed:', error);
|
||||
logger.error('[SkillSync] Scheduled startup run failed:', error);
|
||||
});
|
||||
}
|
||||
scheduleNext(config);
|
||||
|
|
|
|||
|
|
@ -480,7 +480,7 @@ export type CreateSkillInput = {
|
|||
category?: string;
|
||||
author: Types.ObjectId;
|
||||
authorName: string;
|
||||
source?: 'inline' | 'github' | 'notion';
|
||||
source?: 'inline' | 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
sourceMetadata?: Record<string, unknown>;
|
||||
/**
|
||||
* When `true`, the skill is auto-primed into every turn. Callers pass this
|
||||
|
|
@ -499,7 +499,7 @@ export type UpdateSkillInput = {
|
|||
frontmatter?: Record<string, unknown>;
|
||||
category?: string;
|
||||
alwaysApply?: boolean;
|
||||
source?: 'inline' | 'github' | 'notion';
|
||||
source?: 'inline' | 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
sourceMetadata?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
|
|
@ -936,12 +936,12 @@ export function createSkillMethods(
|
|||
deleteSkill: (id: string) => Promise<{ deleted: boolean }>;
|
||||
deleteUserSkills: (userId: Types.ObjectId | string) => Promise<number>;
|
||||
findSkillBySourceIdentity: (params: {
|
||||
source: 'github' | 'notion';
|
||||
source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
upstreamId: string;
|
||||
tenantId?: string;
|
||||
}) => Promise<(ISkill & { _id: Types.ObjectId }) | null>;
|
||||
listSkillsBySource: (params: {
|
||||
source: 'github' | 'notion';
|
||||
source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
sourceId: string;
|
||||
}) => Promise<Array<ISkill & { _id: Types.ObjectId }>>;
|
||||
listSkillFiles: (
|
||||
|
|
@ -1592,7 +1592,7 @@ export function createSkillMethods(
|
|||
}
|
||||
|
||||
async function findSkillBySourceIdentity(params: {
|
||||
source: 'github' | 'notion';
|
||||
source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
upstreamId: string;
|
||||
tenantId?: string;
|
||||
}): Promise<(ISkill & { _id: Types.ObjectId }) | null> {
|
||||
|
|
@ -1609,7 +1609,7 @@ export function createSkillMethods(
|
|||
}
|
||||
|
||||
async function listSkillsBySource(params: {
|
||||
source: 'github' | 'notion';
|
||||
source: 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
sourceId: string;
|
||||
}): Promise<Array<ISkill & { _id: Types.ObjectId }>> {
|
||||
const Skill = mongoose.models.Skill as Model<ISkillDocument>;
|
||||
|
|
|
|||
|
|
@ -72,9 +72,12 @@ export interface ISkill {
|
|||
* Provenance of this skill's canonical definition.
|
||||
* - `inline` — authored inside LibreChat.
|
||||
* - `github` — mirrored from a configured GitHub skill sync source.
|
||||
* - `gitlab` — mirrored from a configured GitLab skill sync source.
|
||||
* - `bitbucket` — mirrored from a configured Bitbucket skill sync source.
|
||||
* - `azuredevops` — mirrored from a configured Azure DevOps skill sync source.
|
||||
* - `notion` — reserved for future external sync integrations.
|
||||
*/
|
||||
source: 'inline' | 'github' | 'notion';
|
||||
source: 'inline' | 'github' | 'gitlab' | 'bitbucket' | 'azuredevops' | 'notion';
|
||||
/**
|
||||
* Provenance payload keyed by `source`, including upstream identifiers
|
||||
* such as GitHub source id, path, and commit/blob SHAs.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue