From 8ef285c39c113a97ee85a2ce291abb31d16644ee Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 14:31:09 +0100 Subject: [PATCH 01/10] feat: implement DAL for new repositories tables --- .../src/repositories/index.ts | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 services/libs/data-access-layer/src/repositories/index.ts diff --git a/services/libs/data-access-layer/src/repositories/index.ts b/services/libs/data-access-layer/src/repositories/index.ts new file mode 100644 index 0000000000..b5535114a0 --- /dev/null +++ b/services/libs/data-access-layer/src/repositories/index.ts @@ -0,0 +1,230 @@ +import { QueryExecutor } from '../queryExecutor' + +/** + * Repository entity from the public.repositories table + */ +export interface IRepository { + id: string + url: string + segmentId: string + gitIntegrationId: string + sourceIntegrationId: string + insightsProjectId: string + archived: boolean + forkedFrom: string | null + excluded: boolean + createdAt: string + updatedAt: string + deletedAt: string | null + lastArchivedCheckAt: string | null +} + +export interface ICreateRepository { + id: string + url: string + segmentId: string + gitIntegrationId: string + sourceIntegrationId: string + insightsProjectId: string + archived?: boolean + forkedFrom?: string | null + excluded?: boolean +} + +export async function createRepository( + qx: QueryExecutor, + data: ICreateRepository, +): Promise { + // TODO: Implement + throw new Error('Not implemented') +} + +/** + * Bulk inserts repositories into public.repositories and git.repositoryProcessing + * @param qx - Query executor (should be transactional) + * @param repositories - Array of repositories to insert + */ +export async function insertRepositories( + qx: QueryExecutor, + repositories: ICreateRepository[], +): Promise { + if (repositories.length === 0) { + return + } + + const values = repositories.map((repo) => ({ + id: repo.id, + url: repo.url, + segmentId: repo.segmentId, + gitIntegrationId: repo.gitIntegrationId, + sourceIntegrationId: repo.sourceIntegrationId, + insightsProjectId: repo.insightsProjectId, + archived: repo.archived ?? false, + forkedFrom: repo.forkedFrom ?? null, + excluded: repo.excluded ?? false, + })) + + await qx.result( + ` + INSERT INTO public.repositories ( + id, + url, + "segmentId", + "gitIntegrationId", + "sourceIntegrationId", + "insightsProjectId", + archived, + "forkedFrom", + excluded, + "createdAt", + "updatedAt" + ) + SELECT + v.id::uuid, + v.url, + v."segmentId"::uuid, + v."gitIntegrationId"::uuid, + v."sourceIntegrationId"::uuid, + v."insightsProjectId"::uuid, + v.archived::boolean, + v."forkedFrom", + v.excluded::boolean, + NOW(), + NOW() + FROM jsonb_to_recordset($(values)::jsonb) AS v( + id text, + url text, + "segmentId" text, + "gitIntegrationId" text, + "sourceIntegrationId" text, + "insightsProjectId" text, + archived boolean, + "forkedFrom" text, + excluded boolean + ) + `, + { values: JSON.stringify(values) }, + ) + + // Insert into git.repositoryProcessing to sync into git integration worker + const repositoryIds = repositories.map((repo) => ({ repositoryId: repo.id })) + await qx.result( + ` + INSERT INTO git."repositoryProcessing" ( + "repositoryId", + "createdAt", + "updatedAt" + ) + SELECT + v."repositoryId"::uuid, + NOW(), + NOW() + FROM jsonb_to_recordset($(repositoryIds)::jsonb) AS v( + "repositoryId" text + ) + `, + { repositoryIds: JSON.stringify(repositoryIds) }, + ) +} + +/** + * Get repositories by source integration ID + * @param qx - Query executor + * @param sourceIntegrationId - The source integration ID + * @returns Array of repositories for the given integration (excluding soft-deleted) + */ +export async function getRepositoriesBySourceIntegrationId( + qx: QueryExecutor, + sourceIntegrationId: string, +): Promise { + return qx.select( + ` + SELECT + id, + url, + "segmentId", + "gitIntegrationId", + "sourceIntegrationId", + "insightsProjectId", + archived, + "forkedFrom", + excluded, + "createdAt", + "updatedAt", + "deletedAt", + "lastArchivedCheckAt" + FROM public.repositories + WHERE "sourceIntegrationId" = $(sourceIntegrationId) + AND "deletedAt" IS NULL + `, + { sourceIntegrationId }, + ) +} + +/** + * Get git repository IDs by URLs from git.repositories table + * @param qx - Query executor + * @param urls - Array of repository URLs + * @returns Map of URL to repository ID + */ +export async function getGitRepositoryIdsByUrl( + qx: QueryExecutor, + urls: string[], +): Promise> { + if (urls.length === 0) { + return new Map() + } + + const results = await qx.select( + ` + SELECT id, url + FROM git.repositories + WHERE url IN ($(urls:csv)) + `, + { urls }, + ) + + return new Map(results.map((row: { id: string; url: string }) => [row.url, row.id])) +} + +/** + * Get repositories by their URLs + * @param qx - Query executor + * @param repoUrls - Array of repository URLs to search for + * @param includeSoftDeleted - Whether to include soft-deleted repositories (default: false) + * @returns Array of repositories matching the given URLs + */ +export async function getRepositoriesByUrl( + qx: QueryExecutor, + repoUrls: string[], + includeSoftDeleted = false, +): Promise { + if (repoUrls.length === 0) { + return [] + } + + const deletedFilter = includeSoftDeleted ? '' : 'AND "deletedAt" IS NULL' + + return qx.select( + ` + SELECT + id, + url, + "segmentId", + "gitIntegrationId", + "sourceIntegrationId", + "insightsProjectId", + archived, + "forkedFrom", + excluded, + "createdAt", + "updatedAt", + "deletedAt", + "lastArchivedCheckAt" + FROM public.repositories + WHERE url IN ($(repoUrls:csv)) + ${deletedFilter} + `, + { repoUrls }, + ) +} From 650a8f2f01c086fdd90b7450f4a9fdd86fe8c38f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 14:32:05 +0100 Subject: [PATCH 02/10] feat: implement repos insertions and enable it for github-nango --- backend/src/services/integrationService.ts | 191 ++++++++++++++++++++- 1 file changed, 190 insertions(+), 1 deletion(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 6d74c2cb90..fe55ae68b1 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -6,7 +6,7 @@ import lodash from 'lodash' import moment from 'moment' import { Transaction } from 'sequelize' -import { EDITION, Error400, Error404, Error542, encryptData } from '@crowd/common' +import { EDITION, Error400, Error404, Error542, encryptData, generateUUIDv4 } from '@crowd/common' import { CommonIntegrationService, getGithubInstallationToken } from '@crowd/common_services' import { syncRepositoriesToGitV2 } from '@crowd/data-access-layer' import { @@ -16,6 +16,14 @@ import { upsertSegmentRepositories, } from '@crowd/data-access-layer/src/collections' import { findRepositoriesForSegment } from '@crowd/data-access-layer/src/integrations' +import { + getRepositoriesByUrl, + getRepositoriesBySourceIntegrationId, + getGitRepositoryIdsByUrl, + insertRepositories, + IRepository, + ICreateRepository, +} from '@crowd/data-access-layer/src/repositories' import { getGithubMappedRepos, getGitlabMappedRepos } from '@crowd/data-access-layer/src/segments' import { NangoIntegration, @@ -916,6 +924,7 @@ export default class IntegrationService { // create github mapping - this also creates git integration await txService.mapGithubRepos(integration.id, mapping, false) + } else { // update existing integration integration = await txService.findById(integrationId) @@ -945,6 +954,9 @@ export default class IntegrationService { ) } + // sync to public.repositories + await txService.mapUnifiedRepositories(PlatformType.GITHUB_NANGO, integration.id, mapping) + if (!existingTransaction) { await SequelizeRepository.commitTransaction(transaction) } @@ -2966,4 +2978,181 @@ export default class IntegrationService { ) return integration } + + private validateRepoIntegrationMapping( + existingRepos: IRepository[], + sourceIntegrationId: string, + ): void { + const integrationMismatches = existingRepos.filter( + (repo) => repo.deletedAt === null && repo.sourceIntegrationId !== sourceIntegrationId + ) + + if (integrationMismatches.length > 0) { + const mismatchDetails = integrationMismatches.map( + (repo) => `${repo.url} belongs to integration ${repo.sourceIntegrationId}` + ).join(', ') + throw new Error400( + this.options.language, + `Cannot remap repositories from different integration: ${mismatchDetails}` + ) + } + } + + /** + * Builds repository payloads for insertion into public.repositories + */ + private async buildRepositoryPayloads( + qx: any, + urls: string[], + mapping: { [url: string]: string }, + sourcePlatform: PlatformType, + sourceIntegrationId: string, + txOptions: IRepositoryOptions, + ): Promise { + if (urls.length === 0) { + return [] + } + + const segmentIds = [...new Set(urls.map((url) => mapping[url]))] + + const isGitHubPlatform = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO].includes(sourcePlatform) + + const [gitRepoIdMap, sourceIntegration] = await Promise.all([ + // TODO: after migration, generate UUIDs instead of fetching from git.repositories + getGitRepositoryIdsByUrl(qx, urls), + isGitHubPlatform ? IntegrationRepository.findById(sourceIntegrationId, txOptions) : null, + ]) + + const collectionService = new CollectionService(txOptions) + const insightsProjectMap = new Map() + const gitIntegrationMap = new Map() + + for (const segmentId of segmentIds) { + const [insightsProject] = await collectionService.findInsightsProjectsBySegmentId(segmentId) + if (!insightsProject) { + throw new Error400(this.options.language, `Insights project not found for segment ${segmentId}`) + } + insightsProjectMap.set(segmentId, insightsProject.id) + + if (sourcePlatform === PlatformType.GIT) { + gitIntegrationMap.set(segmentId, sourceIntegrationId) + continue + } + + try { + const segmentOptions: IRepositoryOptions = { + ...txOptions, + currentSegments: [{ ...this.options.currentSegments[0], id: segmentId }], + } + const gitIntegration = await IntegrationRepository.findByPlatform( + PlatformType.GIT, + segmentOptions, + ) + gitIntegrationMap.set(segmentId, gitIntegration.id) + } catch { + throw new Error400(this.options.language, `Git integration not found for segment ${segmentId}`) + } + } + + // Build forkedFrom map from integration settings (for GITHUB repositories) + const forkedFromMap = new Map() + if (sourceIntegration?.settings?.orgs) { + const allRepos = sourceIntegration.settings.orgs.flatMap((org: any) => org.repos || []) + for (const repo of allRepos) { + if (repo.url && repo.forkedFrom) { + forkedFromMap.set(repo.url, repo.forkedFrom) + } + } + } + + // Build payloads + const payloads: ICreateRepository[] = [] + for (const url of urls) { + const segmentId = mapping[url] + let id = gitRepoIdMap.get(url) + const insightsProjectId = insightsProjectMap.get(segmentId) + const gitIntegrationId = gitIntegrationMap.get(segmentId) + + if (!id) { + // TODO: after migration, this will be the default behavior + this.options.log.warn(`No git.repositories ID found for URL ${url}, generating new UUID...`) + id = generateUUIDv4() + } + + payloads.push({ + id, + url, + segmentId, + gitIntegrationId, + sourceIntegrationId, + insightsProjectId, + forkedFrom: forkedFromMap.get(url) ?? null, + }) + } + + return payloads + } + + async mapUnifiedRepositories(sourcePlatform: PlatformType, sourceIntegrationId: string, mapping: { [url: string]: string }){ + const transaction = await SequelizeRepository.createTransaction(this.options) + const txOptions = { + ...this.options, + transaction, + } + + try { + const qx = SequelizeRepository.getQueryExecutor(txOptions) + const mappedUrls = Object.keys(mapping) + const mappedUrlSet = new Set(mappedUrls) + + const [existingMappedRepos, activeIntegrationRepos] = await Promise.all([ + getRepositoriesByUrl(qx, mappedUrls, true), + getRepositoriesBySourceIntegrationId(qx, sourceIntegrationId), + ]) + + // Block repos that belong to a different integration + this.validateRepoIntegrationMapping(existingMappedRepos, sourceIntegrationId) + + const existingUrlSet = new Set(existingMappedRepos.map((repo) => repo.url)) + const toInsertUrls = mappedUrls.filter((url) => !existingUrlSet.has(url)) + // Repos to restore: soft-deleted OR segment changed (both need re-onboarding) + const toRestoreRepos = existingMappedRepos.filter( + (repo) => repo.deletedAt !== null || repo.segmentId !== mapping[repo.url] + ) + const toSoftDeleteRepos = activeIntegrationRepos.filter((repo) => !mappedUrlSet.has(repo.url)) + + this.options.log.info( + `Repository mapping: ${toInsertUrls.length} to insert, ${toRestoreRepos.length} to restore, ${toSoftDeleteRepos.length} to soft-delete` + ) + + if (toInsertUrls.length > 0) { + this.options.log.info(`Inserting ${toInsertUrls.length} new repos into public.repositories...`) + const payloads = await this.buildRepositoryPayloads( + qx, + toInsertUrls, + mapping, + sourcePlatform, + sourceIntegrationId, + txOptions, + ) + if (payloads.length > 0) { + await insertRepositories(qx, payloads) + this.options.log.info(`Inserted ${payloads.length} repos into public.repositories`) + } + } + + // TODO: restore repos & re-onboard git integration + // TODO: implement soft-deletion + + await SequelizeRepository.commitTransaction(transaction) + } catch (err) { + this.options.log.error(err, 'Error while mapping unified repositories!') + try { + await SequelizeRepository.rollbackTransaction(transaction) + } catch (rErr) { + this.options.log.error(rErr, 'Error while rolling back transaction!') + } + throw err + } + } } From 332ecf0d444b3538ff28230561c83385c465a17f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 16:42:54 +0100 Subject: [PATCH 03/10] feat: enable unified mapping for rest of code platforms --- backend/src/services/integrationService.ts | 60 +++++++++++++++++----- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index fe55ae68b1..22118e9ee1 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -270,6 +270,7 @@ export default class IntegrationService { remotes: repositories.map((url) => ({ url, forkedFrom: null })), }, txOptions, + platform, ) } @@ -459,6 +460,7 @@ export default class IntegrationService { remotes: remainingRemotes.map((url: string) => ({ url, forkedFrom: null })), }, segmentOptions, + integration.platform, ) } } @@ -1065,6 +1067,7 @@ export default class IntegrationService { }), }, segmentOptions, + PlatformType.GITHUB, ) } else { this.options.log.info(`Updating Git integration for segment ${segmentId}!`) @@ -1076,6 +1079,7 @@ export default class IntegrationService { }), }, segmentOptions, + PlatformType.GITHUB, ) } } @@ -1357,6 +1361,7 @@ export default class IntegrationService { * @param integrationData.remotes - Repository objects with url and optional forkedFrom (parent repo URL). * If forkedFrom is null, existing DB value is preserved. * @param options - Optional repository options + * @param sourcePlatform - If provided, mapUnifiedRepositories is skipped (caller handles it) * @returns Integration object or null if no remotes */ async gitConnectOrUpdate( @@ -1364,6 +1369,7 @@ export default class IntegrationService { remotes: Array<{ url: string; forkedFrom?: string | null }> }, options?: IRepositoryOptions, + sourcePlatform?: PlatformType, ) { const stripGit = (url: string) => { if (url.endsWith('.git')) { @@ -1404,6 +1410,7 @@ export default class IntegrationService { ) // upsert repositories to git.repositories in order to be processed by git-integration V2 + const currentSegmentId = (options || this.options).currentSegments[0].id const qx = SequelizeRepository.getQueryExecutor({ ...(options || this.options), transaction, @@ -1412,9 +1419,22 @@ export default class IntegrationService { qx, remotes, integration.id, - (options || this.options).currentSegments[0].id, + currentSegmentId, ) + // sync to public.repositories (only for direct GIT connections, other platforms handle it themselves) + if (!sourcePlatform) { + const mapping = remotes.reduce((acc, remote) => { + acc[remote.url] = currentSegmentId + return acc + }, {} as Record) + + // Use service with transaction context so mapUnifiedRepositories joins this transaction + const txOptions = { ...(options || this.options), transaction } + const txService = new IntegrationService(txOptions) + await txService.mapUnifiedRepositories(PlatformType.GIT, integration.id, mapping) + } + // Only commit if we created the transaction ourselves if (!existingTransaction) { await SequelizeRepository.commitTransaction(transaction) @@ -1722,14 +1742,21 @@ export default class IntegrationService { transaction, ) - if (integrationData.remote.enableGit) { - const stripGit = (url: string) => { - if (url.endsWith('.git')) { - return url.slice(0, -4) - } - return url + const stripGit = (url: string) => { + if (url.endsWith('.git')) { + return url.slice(0, -4) } + return url + } + // Build full repository URLs from orgURL and repo names + const currentSegmentId = this.options.currentSegments[0].id + const remotes = integrationData.remote.repoNames.map((repoName) => { + const fullUrl = stripGit(`${integrationData.remote.orgURL}/${repoName}`) + return { url: fullUrl, forkedFrom: null } + }) + + if (integrationData.remote.enableGit) { const segmentOptions: IRepositoryOptions = { ...this.options, transaction, @@ -1740,20 +1767,25 @@ export default class IntegrationService { ], } - // Build full repository URLs from orgURL and repo names - const remotes = integrationData.remote.repoNames.map((repoName) => { - const fullUrl = stripGit(`${integrationData.remote.orgURL}/${repoName}`) - return { url: fullUrl, forkedFrom: null } - }) - await this.gitConnectOrUpdate( { remotes, }, segmentOptions, + PlatformType.GERRIT, ) } + // sync to public.repositories + const mapping = remotes.reduce((acc, remote) => { + acc[remote.url] = currentSegmentId + return acc + }, {} as Record) + + const txOptions = { ...this.options, transaction } + const txService = new IntegrationService(txOptions) + await txService.mapUnifiedRepositories(PlatformType.GERRIT, integration.id, mapping) + await startNangoSync(NangoIntegration.GERRIT, connectionId) await SequelizeRepository.commitTransaction(transaction) @@ -2829,6 +2861,7 @@ export default class IntegrationService { }), }, { ...segmentOptions, transaction }, + PlatformType.GITLAB, ) } else { await this.gitConnectOrUpdate( @@ -2839,6 +2872,7 @@ export default class IntegrationService { }), }, { ...segmentOptions, transaction }, + PlatformType.GITLAB, ) } } From bde95c5049ca16591be4ff29b4504f759232dc36 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 17:45:12 +0100 Subject: [PATCH 04/10] feat: implement restore & delete --- backend/src/services/integrationService.ts | 27 ++++- .../src/repositories/index.ts | 108 ++++++++++++++++-- 2 files changed, 125 insertions(+), 10 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 22118e9ee1..4309db7f80 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -23,6 +23,8 @@ import { insertRepositories, IRepository, ICreateRepository, + softDeleteRepositories, + restoreRepositories, } from '@crowd/data-access-layer/src/repositories' import { getGithubMappedRepos, getGitlabMappedRepos } from '@crowd/data-access-layer/src/segments' import { @@ -3175,8 +3177,29 @@ export default class IntegrationService { } } - // TODO: restore repos & re-onboard git integration - // TODO: implement soft-deletion + if (toRestoreRepos.length > 0) { + this.options.log.info(`Restoring ${toRestoreRepos.length} repos in public.repositories...`) + const toRestoreUrls = toRestoreRepos.map((repo) => repo.url) + const restorePayloads = await this.buildRepositoryPayloads( + qx, + toRestoreUrls, + mapping, + sourcePlatform, + sourceIntegrationId, + txOptions, + ) + if (restorePayloads.length > 0) { + await restoreRepositories(qx, restorePayloads) + this.options.log.info(`Restored ${restorePayloads.length} repos in public.repositories`) + } + } + + if (toSoftDeleteRepos.length > 0) { + this.options.log.info(`Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`) + //TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts + await softDeleteRepositories(qx, toSoftDeleteRepos.map((repo) => repo.url)) + this.options.log.info(`Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`) + } await SequelizeRepository.commitTransaction(transaction) } catch (err) { diff --git a/services/libs/data-access-layer/src/repositories/index.ts b/services/libs/data-access-layer/src/repositories/index.ts index b5535114a0..02bca09f2d 100644 --- a/services/libs/data-access-layer/src/repositories/index.ts +++ b/services/libs/data-access-layer/src/repositories/index.ts @@ -31,14 +31,6 @@ export interface ICreateRepository { excluded?: boolean } -export async function createRepository( - qx: QueryExecutor, - data: ICreateRepository, -): Promise { - // TODO: Implement - throw new Error('Not implemented') -} - /** * Bulk inserts repositories into public.repositories and git.repositoryProcessing * @param qx - Query executor (should be transactional) @@ -228,3 +220,103 @@ export async function getRepositoriesByUrl( { repoUrls }, ) } + +/** + * Soft deletes repositories by setting deletedAt = NOW() + * @param qx - Query executor + * @param urls - Array of repository URLs to soft delete + * @returns Number of rows affected + */ +export async function softDeleteRepositories(qx: QueryExecutor, urls: string[]): Promise { + if (urls.length === 0) { + return 0 + } + + return qx.result( + ` + UPDATE public.repositories + SET "deletedAt" = NOW(), "updatedAt" = NOW() + WHERE url IN ($(urls:csv)) + AND "deletedAt" IS NULL + `, + { urls }, + ) +} + +/** + * Restores soft-deleted and/or updated repositories and resets their processing state + * Updates fields in public.repositories and resets git.repositoryProcessing for re-onboarding + * @param qx - Query executor + * @param repositories - Array of repositories with url (required) and optional fields to update + */ +export async function restoreRepositories( + qx: QueryExecutor, + repositories: Partial[], +): Promise { + if (repositories.length === 0) { + return + } + + const urls = repositories.map((repo) => repo.url).filter(Boolean) as string[] + if (urls.length === 0) { + return + } + + const values = repositories.map((repo) => ({ + url: repo.url, + segmentId: repo.segmentId ?? null, + gitIntegrationId: repo.gitIntegrationId ?? null, + sourceIntegrationId: repo.sourceIntegrationId ?? null, + insightsProjectId: repo.insightsProjectId ?? null, + archived: repo.archived ?? null, + forkedFrom: repo.forkedFrom ?? null, + excluded: repo.excluded ?? null, + })) + + await qx.result( + ` + UPDATE public.repositories r + SET + "segmentId" = COALESCE(v."segmentId"::uuid, r."segmentId"), + "gitIntegrationId" = COALESCE(v."gitIntegrationId"::uuid, r."gitIntegrationId"), + "sourceIntegrationId" = COALESCE(v."sourceIntegrationId"::uuid, r."sourceIntegrationId"), + "insightsProjectId" = COALESCE(v."insightsProjectId"::uuid, r."insightsProjectId"), + archived = COALESCE(v.archived::boolean, r.archived), + "forkedFrom" = COALESCE(v."forkedFrom", r."forkedFrom"), + excluded = COALESCE(v.excluded::boolean, r.excluded), + "deletedAt" = NULL, + "updatedAt" = NOW() + FROM jsonb_to_recordset($(values)::jsonb) AS v( + url text, + "segmentId" text, + "gitIntegrationId" text, + "sourceIntegrationId" text, + "insightsProjectId" text, + archived boolean, + "forkedFrom" text, + excluded boolean + ) + WHERE r.url = v.url + `, + { values: JSON.stringify(values) }, + ) + + // Reset git.repositoryProcessing for git re-onboarding + await qx.result( + ` + UPDATE git."repositoryProcessing" rp + SET + "lastProcessedAt" = NULL, + "lastProcessedCommit" = NULL, + "lastMaintainerRunAt" = NULL, + branch = NULL, + "lockedAt" = NULL, + state = 'pending', + "updatedAt" = NOW() + FROM public.repositories r + WHERE rp."repositoryId" = r.id + AND r.url IN ($(urls:csv)) + `, + { urls }, + ) +} From 55b29357f11830bea862a0ec43c901970d4d06ce Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 17:50:20 +0100 Subject: [PATCH 05/10] fix: lint --- backend/src/services/integrationService.ts | 29 +++++++++++----------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 4309db7f80..7f306443d5 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -3072,21 +3072,20 @@ export default class IntegrationService { if (sourcePlatform === PlatformType.GIT) { gitIntegrationMap.set(segmentId, sourceIntegrationId) - continue - } - - try { - const segmentOptions: IRepositoryOptions = { - ...txOptions, - currentSegments: [{ ...this.options.currentSegments[0], id: segmentId }], + } else { + try { + const segmentOptions: IRepositoryOptions = { + ...txOptions, + currentSegments: [{ ...this.options.currentSegments[0], id: segmentId }], + } + const gitIntegration = await IntegrationRepository.findByPlatform( + PlatformType.GIT, + segmentOptions, + ) + gitIntegrationMap.set(segmentId, gitIntegration.id) + } catch { + throw new Error400(this.options.language, `Git integration not found for segment ${segmentId}`) } - const gitIntegration = await IntegrationRepository.findByPlatform( - PlatformType.GIT, - segmentOptions, - ) - gitIntegrationMap.set(segmentId, gitIntegration.id) - } catch { - throw new Error400(this.options.language, `Git integration not found for segment ${segmentId}`) } } @@ -3196,7 +3195,7 @@ export default class IntegrationService { if (toSoftDeleteRepos.length > 0) { this.options.log.info(`Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`) - //TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts + // TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts await softDeleteRepositories(qx, toSoftDeleteRepos.map((repo) => repo.url)) this.options.log.info(`Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`) } From 53939dc4f83ddeb47f65bed815171c6afc381708 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Tue, 30 Dec 2025 17:54:03 +0100 Subject: [PATCH 06/10] fix: formatting --- backend/src/services/integrationService.ts | 91 +++++++++++++--------- 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 7f306443d5..f7fb1b56d6 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -17,14 +17,14 @@ import { } from '@crowd/data-access-layer/src/collections' import { findRepositoriesForSegment } from '@crowd/data-access-layer/src/integrations' import { - getRepositoriesByUrl, - getRepositoriesBySourceIntegrationId, + ICreateRepository, + IRepository, getGitRepositoryIdsByUrl, + getRepositoriesBySourceIntegrationId, + getRepositoriesByUrl, insertRepositories, - IRepository, - ICreateRepository, - softDeleteRepositories, restoreRepositories, + softDeleteRepositories, } from '@crowd/data-access-layer/src/repositories' import { getGithubMappedRepos, getGitlabMappedRepos } from '@crowd/data-access-layer/src/segments' import { @@ -928,7 +928,6 @@ export default class IntegrationService { // create github mapping - this also creates git integration await txService.mapGithubRepos(integration.id, mapping, false) - } else { // update existing integration integration = await txService.findById(integrationId) @@ -1417,19 +1416,17 @@ export default class IntegrationService { ...(options || this.options), transaction, }) - await syncRepositoriesToGitV2( - qx, - remotes, - integration.id, - currentSegmentId, - ) + await syncRepositoriesToGitV2(qx, remotes, integration.id, currentSegmentId) // sync to public.repositories (only for direct GIT connections, other platforms handle it themselves) if (!sourcePlatform) { - const mapping = remotes.reduce((acc, remote) => { - acc[remote.url] = currentSegmentId - return acc - }, {} as Record) + const mapping = remotes.reduce( + (acc, remote) => { + acc[remote.url] = currentSegmentId + return acc + }, + {} as Record, + ) // Use service with transaction context so mapUnifiedRepositories joins this transaction const txOptions = { ...(options || this.options), transaction } @@ -1779,10 +1776,13 @@ export default class IntegrationService { } // sync to public.repositories - const mapping = remotes.reduce((acc, remote) => { - acc[remote.url] = currentSegmentId - return acc - }, {} as Record) + const mapping = remotes.reduce( + (acc, remote) => { + acc[remote.url] = currentSegmentId + return acc + }, + {} as Record, + ) const txOptions = { ...this.options, transaction } const txService = new IntegrationService(txOptions) @@ -3020,16 +3020,16 @@ export default class IntegrationService { sourceIntegrationId: string, ): void { const integrationMismatches = existingRepos.filter( - (repo) => repo.deletedAt === null && repo.sourceIntegrationId !== sourceIntegrationId + (repo) => repo.deletedAt === null && repo.sourceIntegrationId !== sourceIntegrationId, ) if (integrationMismatches.length > 0) { - const mismatchDetails = integrationMismatches.map( - (repo) => `${repo.url} belongs to integration ${repo.sourceIntegrationId}` - ).join(', ') + const mismatchDetails = integrationMismatches + .map((repo) => `${repo.url} belongs to integration ${repo.sourceIntegrationId}`) + .join(', ') throw new Error400( this.options.language, - `Cannot remap repositories from different integration: ${mismatchDetails}` + `Cannot remap repositories from different integration: ${mismatchDetails}`, ) } } @@ -3051,7 +3051,9 @@ export default class IntegrationService { const segmentIds = [...new Set(urls.map((url) => mapping[url]))] - const isGitHubPlatform = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO].includes(sourcePlatform) + const isGitHubPlatform = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO].includes( + sourcePlatform, + ) const [gitRepoIdMap, sourceIntegration] = await Promise.all([ // TODO: after migration, generate UUIDs instead of fetching from git.repositories @@ -3066,7 +3068,10 @@ export default class IntegrationService { for (const segmentId of segmentIds) { const [insightsProject] = await collectionService.findInsightsProjectsBySegmentId(segmentId) if (!insightsProject) { - throw new Error400(this.options.language, `Insights project not found for segment ${segmentId}`) + throw new Error400( + this.options.language, + `Insights project not found for segment ${segmentId}`, + ) } insightsProjectMap.set(segmentId, insightsProject.id) @@ -3084,7 +3089,10 @@ export default class IntegrationService { ) gitIntegrationMap.set(segmentId, gitIntegration.id) } catch { - throw new Error400(this.options.language, `Git integration not found for segment ${segmentId}`) + throw new Error400( + this.options.language, + `Git integration not found for segment ${segmentId}`, + ) } } } @@ -3128,7 +3136,11 @@ export default class IntegrationService { return payloads } - async mapUnifiedRepositories(sourcePlatform: PlatformType, sourceIntegrationId: string, mapping: { [url: string]: string }){ + async mapUnifiedRepositories( + sourcePlatform: PlatformType, + sourceIntegrationId: string, + mapping: { [url: string]: string }, + ) { const transaction = await SequelizeRepository.createTransaction(this.options) const txOptions = { ...this.options, @@ -3152,16 +3164,18 @@ export default class IntegrationService { const toInsertUrls = mappedUrls.filter((url) => !existingUrlSet.has(url)) // Repos to restore: soft-deleted OR segment changed (both need re-onboarding) const toRestoreRepos = existingMappedRepos.filter( - (repo) => repo.deletedAt !== null || repo.segmentId !== mapping[repo.url] + (repo) => repo.deletedAt !== null || repo.segmentId !== mapping[repo.url], ) const toSoftDeleteRepos = activeIntegrationRepos.filter((repo) => !mappedUrlSet.has(repo.url)) this.options.log.info( - `Repository mapping: ${toInsertUrls.length} to insert, ${toRestoreRepos.length} to restore, ${toSoftDeleteRepos.length} to soft-delete` + `Repository mapping: ${toInsertUrls.length} to insert, ${toRestoreRepos.length} to restore, ${toSoftDeleteRepos.length} to soft-delete`, ) if (toInsertUrls.length > 0) { - this.options.log.info(`Inserting ${toInsertUrls.length} new repos into public.repositories...`) + this.options.log.info( + `Inserting ${toInsertUrls.length} new repos into public.repositories...`, + ) const payloads = await this.buildRepositoryPayloads( qx, toInsertUrls, @@ -3194,10 +3208,17 @@ export default class IntegrationService { } if (toSoftDeleteRepos.length > 0) { - this.options.log.info(`Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`) + this.options.log.info( + `Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`, + ) // TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts - await softDeleteRepositories(qx, toSoftDeleteRepos.map((repo) => repo.url)) - this.options.log.info(`Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`) + await softDeleteRepositories( + qx, + toSoftDeleteRepos.map((repo) => repo.url), + ) + this.options.log.info( + `Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`, + ) } await SequelizeRepository.commitTransaction(transaction) From addd22df5cb79c9e86eceef864d61a9802668857 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 31 Dec 2025 11:39:50 +0100 Subject: [PATCH 07/10] feat: enable unified mapping for gitlab --- backend/src/services/integrationService.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index f7fb1b56d6..aa9dab52cd 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -2878,6 +2878,10 @@ export default class IntegrationService { ) } } + + // sync to public.repositories + const txService = new IntegrationService(txOptions) + await txService.mapUnifiedRepositories(PlatformType.GITLAB, integrationId, mapping) } const integration = await IntegrationRepository.update( From 8cf77e6c33fbe81e56b31dd987425028939b96e3 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 31 Dec 2025 16:03:49 +0100 Subject: [PATCH 08/10] feat: validate repo ownership in deletions --- backend/src/services/integrationService.ts | 57 ++++++++++++++++++- .../src/repositories/index.ts | 11 +++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index aa9dab52cd..b7e4c9a60c 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -16,6 +16,7 @@ import { upsertSegmentRepositories, } from '@crowd/data-access-layer/src/collections' import { findRepositoriesForSegment } from '@crowd/data-access-layer/src/integrations' +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { ICreateRepository, IRepository, @@ -518,12 +519,23 @@ export default class IntegrationService { // Soft delete git.repositories for git integration if (integration.platform === PlatformType.GIT) { + await this.validateGitIntegrationDeletion(integration.id, { + ...this.options, + transaction, + }) + await GitReposRepository.delete(integration.id, { ...this.options, transaction, }) } + // Soft delete from public.repositories for code integrations + if (IntegrationService.isCodePlatform(integration.platform)) { + const txService = new IntegrationService({ ...this.options, transaction }) + await txService.mapUnifiedRepositories(integration.platform, integration.id, {}) + } + await IntegrationRepository.destroy(id, { ...this.options, transaction, @@ -3038,6 +3050,47 @@ export default class IntegrationService { } } + private validateReposOwnership(repos: IRepository[], sourceIntegrationId: string): void { + const ownershipMismatches = repos.filter( + (repo) => repo.sourceIntegrationId !== sourceIntegrationId, + ) + + if (ownershipMismatches.length > 0) { + const mismatchUrls = ownershipMismatches.map((repo) => repo.url).join(', ') + throw new Error400( + this.options.language, + `These repos are managed by another integration: ${mismatchUrls}`, + ) + } + } + + private async validateGitIntegrationDeletion( + gitIntegrationId: string, + options: IRepositoryOptions, + ): Promise { + const qx = SequelizeRepository.getQueryExecutor(options) + + // Find repos linked to this GIT integration but owned by a different integration + const ownedByOthers = await qx.select( + ` + SELECT url + FROM public.repositories + WHERE "gitIntegrationId" = $(gitIntegrationId) + AND "sourceIntegrationId" != $(gitIntegrationId) + AND "deletedAt" IS NULL + `, + { gitIntegrationId }, + ) + + if (ownedByOthers.length > 0) { + const mismatchUrls = ownedByOthers.map((repo: { url: string }) => repo.url).join(', ') + throw new Error400( + this.options.language, + `Cannot delete GIT integration: these repos are managed by another integration: ${mismatchUrls}`, + ) + } + } + /** * Builds repository payloads for insertion into public.repositories */ @@ -3212,13 +3265,15 @@ export default class IntegrationService { } if (toSoftDeleteRepos.length > 0) { + this.validateReposOwnership(toSoftDeleteRepos, sourceIntegrationId) + this.options.log.info( `Soft-deleting ${toSoftDeleteRepos.length} repos from public.repositories...`, ) - // TODO: post migration, add sourceIntegrationId to the delete condition to avoid cross-integration conflicts await softDeleteRepositories( qx, toSoftDeleteRepos.map((repo) => repo.url), + sourceIntegrationId, ) this.options.log.info( `Soft-deleted ${toSoftDeleteRepos.length} repos from public.repositories`, diff --git a/services/libs/data-access-layer/src/repositories/index.ts b/services/libs/data-access-layer/src/repositories/index.ts index 02bca09f2d..3607884eea 100644 --- a/services/libs/data-access-layer/src/repositories/index.ts +++ b/services/libs/data-access-layer/src/repositories/index.ts @@ -223,11 +223,17 @@ export async function getRepositoriesByUrl( /** * Soft deletes repositories by setting deletedAt = NOW() + * Only deletes repos matching both the URLs and sourceIntegrationId * @param qx - Query executor * @param urls - Array of repository URLs to soft delete + * @param sourceIntegrationId - Only delete repos belonging to this integration * @returns Number of rows affected */ -export async function softDeleteRepositories(qx: QueryExecutor, urls: string[]): Promise { +export async function softDeleteRepositories( + qx: QueryExecutor, + urls: string[], + sourceIntegrationId: string, +): Promise { if (urls.length === 0) { return 0 } @@ -237,9 +243,10 @@ export async function softDeleteRepositories(qx: QueryExecutor, urls: string[]): UPDATE public.repositories SET "deletedAt" = NOW(), "updatedAt" = NOW() WHERE url IN ($(urls:csv)) + AND "sourceIntegrationId" = $(sourceIntegrationId) AND "deletedAt" IS NULL `, - { urls }, + { urls, sourceIntegrationId }, ) } From b71251d1d28db1c6613c47c4347ad386967a7a9f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 31 Dec 2025 16:10:14 +0100 Subject: [PATCH 09/10] fix: gerrit bug of overriding existing remotes in git integration --- backend/src/services/integrationService.ts | 36 +++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index b7e4c9a60c..a192d3882f 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -1778,13 +1778,35 @@ export default class IntegrationService { ], } - await this.gitConnectOrUpdate( - { - remotes, - }, - segmentOptions, - PlatformType.GERRIT, - ) + // Check if git integration already exists and merge remotes + let isGitIntegrationConfigured = false + try { + await IntegrationRepository.findByPlatform(PlatformType.GIT, segmentOptions) + isGitIntegrationConfigured = true + } catch (err) { + isGitIntegrationConfigured = false + } + + if (isGitIntegrationConfigured) { + const gitInfo = await this.gitGetRemotes(segmentOptions) + const gitRemotes = gitInfo[currentSegmentId]?.remotes || [] + const allUrls = Array.from(new Set([...gitRemotes, ...remotes.map((r) => r.url)])) + await this.gitConnectOrUpdate( + { + remotes: allUrls.map((url) => ({ url, forkedFrom: null })), + }, + segmentOptions, + PlatformType.GERRIT, + ) + } else { + await this.gitConnectOrUpdate( + { + remotes, + }, + segmentOptions, + PlatformType.GERRIT, + ) + } } // sync to public.repositories From 9df6d8235ab8636551aeb03990147987c1692bac Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 31 Dec 2025 17:23:44 +0100 Subject: [PATCH 10/10] fix: always enable git integration for gerrit to ensure code platform consistency --- .../gerrit/components/gerrit-settings-drawer.vue | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frontend/src/config/integrations/gerrit/components/gerrit-settings-drawer.vue b/frontend/src/config/integrations/gerrit/components/gerrit-settings-drawer.vue index a71d89e099..51aea09060 100644 --- a/frontend/src/config/integrations/gerrit/components/gerrit-settings-drawer.vue +++ b/frontend/src/config/integrations/gerrit/components/gerrit-settings-drawer.vue @@ -71,9 +71,9 @@ Enable All Projects - + @@ -136,7 +136,7 @@ const form = reactive({ // user: '', // pass: '', enableAllRepos: false, - enableGit: false, + enableGit: true, repoNames: [], }); @@ -161,7 +161,6 @@ onMounted(() => { // form.pass = props.integration?.settings.remote.pass; form.repoNames = props.integration?.settings.remote.repoNames; form.enableAllRepos = props.integration?.settings.remote.enableAllRepos; - form.enableGit = props.integration?.settings.remote.enableGit; } formSnapshot(); });