diff --git a/packages/backend/src/gitlab.test.ts b/packages/backend/src/gitlab.test.ts index 4d9190e08..454c1d9db 100644 --- a/packages/backend/src/gitlab.test.ts +++ b/packages/backend/src/gitlab.test.ts @@ -1,5 +1,52 @@ -import { expect, test } from 'vitest'; -import { shouldExcludeProject } from './gitlab'; +import { expect, test, vi, describe, beforeEach } from 'vitest'; +import { shouldExcludeProject, parseQuery, getGitLabReposFromConfig } from './gitlab'; +import { ProjectSchema } from '@gitbeaker/rest'; + +// Mock dependencies +const mockGitlabAll = vi.fn(); + +vi.mock('@gitbeaker/rest', () => { + return { + Gitlab: vi.fn().mockImplementation(() => ({ + Projects: { + all: mockGitlabAll, + } + })), + }; +}); + +vi.mock('@sourcebot/shared', async () => ({ + getTokenFromConfig: vi.fn(), + createLogger: () => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + warning: vi.fn(), + }), + env: { + GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: 10, + } +})); + +vi.mock('./connectionUtils', () => ({ + processPromiseResults: (results: any[]) => { + const validItems = results + .filter((r) => r.status === 'fulfilled' && r.value.type === 'valid') + .flatMap((r) => r.value.data); + return { validItems, warnings: [] }; + }, + throwIfAnyFailed: vi.fn(), +})); + +vi.mock('./utils', () => ({ + measure: async (fn: () => any) => { + const data = await fn(); + return { durationMs: 0, data }; + }, + fetchWithRetry: async (fn: () => any) => fn(), +})); +import { shouldExcludeProject, parseQuery } from './gitlab'; import { ProjectSchema } from '@gitbeaker/rest'; @@ -68,3 +115,63 @@ test('shouldExcludeProject returns false when exclude.userOwnedProjects is true exclude: { userOwnedProjects: true }, })).toBe(false); }); + +test('parseQuery correctly parses query strings', () => { + expect(parseQuery('projects?include_subgroups=true&archived=false&id=123')).toEqual({ + include_subgroups: true, + archived: false, + id: 123 + }); + expect(parseQuery('projects?search=foo')).toEqual({ + search: 'foo' + }); + expect(parseQuery('groups/1/projects?simple=true')).toEqual({ + simple: true + }); +}); + +describe('getGitLabReposFromConfig', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test('fetches projects using projectQuery and deduplicates results', async () => { + const mockProjects1 = [ + { id: 1, path_with_namespace: 'group1/project1', name: 'project1' }, + { id: 2, path_with_namespace: 'group1/project2', name: 'project2' }, + ]; + const mockProjects2 = [ + { id: 2, path_with_namespace: 'group1/project2', name: 'project2' }, // Duplicate + { id: 3, path_with_namespace: 'group2/project3', name: 'project3' }, + ]; + + mockGitlabAll.mockResolvedValueOnce(mockProjects1); + mockGitlabAll.mockResolvedValueOnce(mockProjects2); + + const config = { + type: 'gitlab' as const, + projectQuery: [ + 'groups/group1/projects?include_subgroups=true', + 'projects?topic=devops' + ] + }; + + const result = await getGitLabReposFromConfig(config); + + // Verify API calls + expect(mockGitlabAll).toHaveBeenCalledTimes(2); + expect(mockGitlabAll).toHaveBeenCalledWith(expect.objectContaining({ + perPage: 100, + include_subgroups: true + })); + expect(mockGitlabAll).toHaveBeenCalledWith(expect.objectContaining({ + perPage: 100, + topic: 'devops' + })); + + // Verify deduplication + expect(result.repos).toHaveLength(3); + const projectIds = result.repos.map((p: any) => p.id).sort(); + expect(projectIds).toEqual([1, 2, 3]); + }); +}); diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index 44685424a..2446358ac 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -41,8 +41,8 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = const token = config.token ? await getTokenFromConfig(config.token) : hostname === GITLAB_CLOUD_HOSTNAME ? - env.FALLBACK_GITLAB_CLOUD_TOKEN : - undefined; + env.FALLBACK_GITLAB_CLOUD_TOKEN : + undefined; const api = await createGitLabFromPersonalAccessToken({ token, @@ -189,6 +189,46 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = } })); + const { validItems: validRepos, warnings } = processPromiseResults(results); + allRepos = allRepos.concat(validRepos); + allWarnings = allWarnings.concat(warnings); + } + + if (config.projectQuery) { + const results = await Promise.allSettled(config.projectQuery.map(async (query) => { + try { + logger.debug(`Fetching projects for query ${query}...`); + const { durationMs, data } = await measure(async () => { + const fetchFn = () => api.Projects.all({ + perPage: 100, + ...parseQuery(query), + } as any); + return fetchWithRetry(fetchFn, `query ${query}`, logger); + }); + logger.debug(`Found ${data.length} projects for query ${query} in ${durationMs}ms.`); + return { + type: 'valid' as const, + data + }; + } catch (e: any) { + Sentry.captureException(e); + logger.error(`Failed to fetch projects for query ${query}.`, e); + + const status = e?.cause?.response?.status; + if (status !== undefined) { + const warning = `GitLab API returned ${status}` + logger.warning(warning); + return { + type: 'warning' as const, + warning + } + } + + logger.error("No API response status returned"); + throw e; + } + })); + throwIfAnyFailed(results); const { validItems: validRepos, warnings } = processPromiseResults(results); allRepos = allRepos.concat(validRepos); @@ -196,6 +236,11 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = } let repos = allRepos + .filter((project, index, self) => + index === self.findIndex((t) => ( + t.id === project.id + )) + ) .filter((project) => { const isExcluded = shouldExcludeProject({ project, @@ -207,7 +252,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = return !isExcluded; }); - + logger.debug(`Found ${repos.length} total repositories.`); return { @@ -300,6 +345,23 @@ export const getProjectMembers = async (projectId: string, api: InstanceType { + const params = new URLSearchParams(query.split('?')[1]); + const result: Record = {}; + for (const [key, value] of params.entries()) { + if (value === 'true') { + result[key] = true; + } else if (value === 'false') { + result[key] = false; + } else if (!isNaN(Number(value))) { + result[key] = Number(value); + } else { + result[key] = value; + } + } + return result; +} + export const getProjectsForAuthenticatedUser = async (visibility: 'private' | 'internal' | 'public' | 'all' = 'all', api: InstanceType) => { try { const fetchFn = () => api.Projects.all({ diff --git a/schemas/v3/gitlab.json b/schemas/v3/gitlab.json index 9d3b1ca9f..addcbc54e 100644 --- a/schemas/v3/gitlab.json +++ b/schemas/v3/gitlab.json @@ -65,6 +65,19 @@ ], "description": "List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/" }, + "projectQuery": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + [ + "groups/group1/projects?include_subgroups=true", + "projects?topic=devops" + ] + ], + "description": "List of GitLab API query paths to fetch projects from. Each string should be a path relative to the GitLab API root (e.g. `groups/my-group/projects`)." + }, "topics": { "type": "array", "items": {