From 9a4f3d4bc577af8924636238d710de4a47243e28 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sat, 21 Dec 2024 16:48:41 +0100 Subject: [PATCH 01/13] feat: add deleteAllDocuments method to SearchService for bulk document deletion --- api/src/search/service.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 4ac9436e..056da51b 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -43,6 +43,12 @@ export class SearchService { }); }; + public deleteAllDocuments = async (index: SearchType): Promise => { + this.logger.info({ message: `Deleting all documents in ${index}` }); + await this.meilisearch.index(index).deleteAllDocuments(); + this.logger.info({ message: `Deleted all documents in ${index}` }); + }; + public ensureIndexes = async (): Promise => { await this.meilisearch.createIndex("project"); this.logger.info({ message: "project index created" }); From 0d14b8c2f4d7dcb6859e69d9d70f930e81239ac0 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sat, 21 Dec 2024 17:01:28 +0100 Subject: [PATCH 02/13] feat: enhance DigestCron to integrate search service for indexing and deletion of project, contribution, and contributor items --- api/src/digest/cron.ts | 103 +++++++++++++++++++++++++++++------------ 1 file changed, 74 insertions(+), 29 deletions(-) diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 458b70d8..d901a355 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -1,12 +1,15 @@ import { captureException, cron } from "@sentry/node"; -import { CronJob } from "cron"; + import { ContributionRepository } from "src/contribution/repository"; import { ContributorRepository } from "src/contributor/repository"; +import { CronJob } from "cron"; import { DataService } from "src/data/service"; import { GithubService } from "src/github/service"; import { LoggerService } from "src/logger/service"; import { ProjectRepository } from "src/project/repository"; import { RepositoryRepository } from "src/repository/repository"; +import { SearchItem } from "src/search/types"; +import { SearchService } from "src/search/service"; import { Service } from "typedi"; @Service() @@ -22,6 +25,7 @@ export class DigestCron { private readonly repositoriesRepository: RepositoryRepository, private readonly contributionsRepository: ContributionRepository, private readonly contributorsRepository: ContributorRepository, + private readonly searchService: SearchService, ) { const SentryCronJob = cron.instrumentCron(CronJob, "DigestCron"); new SentryCronJob( @@ -66,6 +70,10 @@ export class DigestCron { const projectsFromDataFolder = await this.dataService.listProjects(); + const searchProjectItems: SearchItem[] = []; + const searchContributorItems: SearchItem[] = []; + const searchContributionItems: SearchItem[] = []; + for (const project of projectsFromDataFolder) { const [{ id: projectId }] = await this.projectsRepository.upsert({ ...project, @@ -84,14 +92,20 @@ export class DigestCron { }); const provider = "github"; - const [{ id: repositoryId }] = await this.repositoriesRepository.upsert({ - provider, - name: repoInfo.name, - owner: repoInfo.owner.login, - runId, - projectId, - stars: repoInfo.stargazers_count, + const [{ id: repositoryId }] = + await this.repositoriesRepository.upsert({ + provider, + name: repoInfo.name, + owner: repoInfo.owner.login, + runId, + projectId, + stars: repoInfo.stargazers_count, + id: `${provider}-${repoInfo.id}`, + }); + searchProjectItems.push({ id: `${provider}-${repoInfo.id}`, + title: repoInfo.name, + type: "project", }); addedRepositoryCount++; @@ -101,17 +115,25 @@ export class DigestCron { }); for (const issue of issues) { - const githubUser = await this.githubService.getUser({ username: issue.user.login }); + const githubUser = await this.githubService.getUser({ + username: issue.user.login, + }); if (githubUser.type !== "User") continue; - const [{ id: contributorId }] = await this.contributorsRepository.upsert({ - name: githubUser.name || githubUser.login, - username: githubUser.login, - url: githubUser.html_url, - avatarUrl: githubUser.avatar_url, - runId, + const [{ id: contributorId }] = + await this.contributorsRepository.upsert({ + name: githubUser.name || githubUser.login, + username: githubUser.login, + url: githubUser.html_url, + avatarUrl: githubUser.avatar_url, + runId, + id: `${provider}-${githubUser.login}`, + }); + searchContributorItems.push({ id: `${provider}-${githubUser.login}`, + title: githubUser.name || githubUser.login, + type: "contributor", }); await this.contributorsRepository.upsertRelationWithRepository({ @@ -128,17 +150,26 @@ export class DigestCron { updatedAt: issue.updated_at, activityCount: issue.comments, runId, - url: type === "PULL_REQUEST" ? issue.pull_request.html_url : issue.html_url, + url: + type === "PULL_REQUEST" + ? issue.pull_request.html_url + : issue.html_url, repositoryId, contributorId, id: `${provider}-${issue.id}`, }); + searchContributionItems.push({ + id: `${provider}-${issue.id}`, + title: issue.title, + type: "contribution", + }); } - const repoContributors = await this.githubService.listRepositoryContributors({ - owner: repository.owner, - repository: repository.name, - }); + const repoContributors = + await this.githubService.listRepositoryContributors({ + owner: repository.owner, + repository: repository.name, + }); const repoContributorsFiltered = repoContributors.filter( (contributor) => contributor.type === "User", @@ -148,14 +179,15 @@ export class DigestCron { const contributor = await this.githubService.getUser({ username: repoContributor.login, }); - const [{ id: contributorId }] = await this.contributorsRepository.upsert({ - name: contributor.name || contributor.login, - username: contributor.login, - url: contributor.html_url, - avatarUrl: contributor.avatar_url, - runId, - id: `${provider}-${contributor.login}`, - }); + const [{ id: contributorId }] = + await this.contributorsRepository.upsert({ + name: contributor.name || contributor.login, + username: contributor.login, + url: contributor.html_url, + avatarUrl: contributor.avatar_url, + runId, + id: `${provider}-${contributor.login}`, + }); await this.contributorsRepository.upsertRelationWithRepository({ contributorId, @@ -179,11 +211,24 @@ export class DigestCron { } try { - await this.contributorsRepository.deleteAllRelationWithRepositoryButWithRunId(runId); + await this.contributorsRepository.deleteAllRelationWithRepositoryButWithRunId( + runId, + ); await this.contributionsRepository.deleteAllButWithRunId(runId); await this.contributorsRepository.deleteAllButWithRunId(runId); await this.repositoriesRepository.deleteAllButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); + await this.searchService.deleteAllDocuments("project"); + await this.searchService.deleteAllDocuments("contribution"); + await this.searchService.deleteAllDocuments("contributor"); + } catch (error) { + captureException(error, { tags: { type: "CRON" } }); + } + + try { + await this.searchService.index("project", searchProjectItems); + await this.searchService.index("contribution", searchContributionItems); + await this.searchService.index("contributor", searchContributorItems); } catch (error) { captureException(error, { tags: { type: "CRON" } }); } From 595e2fc8232f8f771490fc8849c6454cdbfe2d5e Mon Sep 17 00:00:00 2001 From: omdxp Date: Sat, 21 Dec 2024 17:41:01 +0100 Subject: [PATCH 03/13] feat: update DigestCron to modify project ID format for MeiliSearch compatibility --- api/src/digest/cron.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index d901a355..889fc905 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -80,6 +80,11 @@ export class DigestCron { runId, id: project.slug, }); + searchProjectItems.push({ + id: project.slug.replace(/[.]/g, "-"), // MeiliSearch doesn't allow dots in ids + title: project.name, + type: "project", + }); let addedRepositoryCount = 0; try { @@ -102,11 +107,6 @@ export class DigestCron { stars: repoInfo.stargazers_count, id: `${provider}-${repoInfo.id}`, }); - searchProjectItems.push({ - id: `${provider}-${repoInfo.id}`, - title: repoInfo.name, - type: "project", - }); addedRepositoryCount++; const issues = await this.githubService.listRepositoryIssues({ From 204a0289d5c4fd3e123c4b64b8d521a77d27bc17 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sat, 21 Dec 2024 18:32:30 +0100 Subject: [PATCH 04/13] feat: refactor DigestCron to use upsert method for search indexing and remove bulk deletion --- api/src/digest/cron.ts | 22 +++------------------- api/src/search/service.ts | 16 +++++----------- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 889fc905..a1cafa4e 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -8,7 +8,6 @@ import { GithubService } from "src/github/service"; import { LoggerService } from "src/logger/service"; import { ProjectRepository } from "src/project/repository"; import { RepositoryRepository } from "src/repository/repository"; -import { SearchItem } from "src/search/types"; import { SearchService } from "src/search/service"; import { Service } from "typedi"; @@ -70,17 +69,13 @@ export class DigestCron { const projectsFromDataFolder = await this.dataService.listProjects(); - const searchProjectItems: SearchItem[] = []; - const searchContributorItems: SearchItem[] = []; - const searchContributionItems: SearchItem[] = []; - for (const project of projectsFromDataFolder) { const [{ id: projectId }] = await this.projectsRepository.upsert({ ...project, runId, id: project.slug, }); - searchProjectItems.push({ + this.searchService.upsert("project", { id: project.slug.replace(/[.]/g, "-"), // MeiliSearch doesn't allow dots in ids title: project.name, type: "project", @@ -130,7 +125,7 @@ export class DigestCron { runId, id: `${provider}-${githubUser.login}`, }); - searchContributorItems.push({ + this.searchService.upsert("contributor", { id: `${provider}-${githubUser.login}`, title: githubUser.name || githubUser.login, type: "contributor", @@ -158,7 +153,7 @@ export class DigestCron { contributorId, id: `${provider}-${issue.id}`, }); - searchContributionItems.push({ + this.searchService.upsert("contribution", { id: `${provider}-${issue.id}`, title: issue.title, type: "contribution", @@ -218,17 +213,6 @@ export class DigestCron { await this.contributorsRepository.deleteAllButWithRunId(runId); await this.repositoriesRepository.deleteAllButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); - await this.searchService.deleteAllDocuments("project"); - await this.searchService.deleteAllDocuments("contribution"); - await this.searchService.deleteAllDocuments("contributor"); - } catch (error) { - captureException(error, { tags: { type: "CRON" } }); - } - - try { - await this.searchService.index("project", searchProjectItems); - await this.searchService.index("contribution", searchContributionItems); - await this.searchService.index("contributor", searchContributorItems); } catch (error) { captureException(error, { tags: { type: "CRON" } }); } diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 056da51b..6dc39625 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -30,25 +30,19 @@ export class SearchService { return []; }; - public index = async ( + public upsert = async ( index: SearchType, - data: SearchItem[], + data: SearchItem, ): Promise => { this.logger.info({ - message: `Indexing ${data.length} items in ${index}`, + message: `Upserting ${data.title} in ${index}`, }); - await this.meilisearch.index(index).addDocuments(data); + await this.meilisearch.index(index).updateDocuments([data]); this.logger.info({ - message: `Indexed ${data.length} items in ${index}`, + message: `Upserted ${data.title} in ${index}`, }); }; - public deleteAllDocuments = async (index: SearchType): Promise => { - this.logger.info({ message: `Deleting all documents in ${index}` }); - await this.meilisearch.index(index).deleteAllDocuments(); - this.logger.info({ message: `Deleted all documents in ${index}` }); - }; - public ensureIndexes = async (): Promise => { await this.meilisearch.createIndex("project"); this.logger.info({ message: "project index created" }); From e4bb276047272f273f85fe3101e71823469d0791 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sat, 21 Dec 2024 19:55:20 +0100 Subject: [PATCH 05/13] feat: enhance DigestCron and SearchService to support bulk upsert and deletion of search items with runId --- api/src/digest/cron.ts | 27 +++++++++++++++++----- api/src/search/service.ts | 47 +++++++++++++++++++++++++++++++++------ api/src/search/types.ts | 1 + 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index a1cafa4e..596654b5 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -8,6 +8,7 @@ import { GithubService } from "src/github/service"; import { LoggerService } from "src/logger/service"; import { ProjectRepository } from "src/project/repository"; import { RepositoryRepository } from "src/repository/repository"; +import { SearchItem } from "src/search/types"; import { SearchService } from "src/search/service"; import { Service } from "typedi"; @@ -69,16 +70,22 @@ export class DigestCron { const projectsFromDataFolder = await this.dataService.listProjects(); + const projectSearchItems: SearchItem[] = []; + const contributionSearchItems: SearchItem[] = []; + const contributorSearchItems: SearchItem[] = []; + for (const project of projectsFromDataFolder) { const [{ id: projectId }] = await this.projectsRepository.upsert({ ...project, runId, id: project.slug, }); - this.searchService.upsert("project", { - id: project.slug.replace(/[.]/g, "-"), // MeiliSearch doesn't allow dots in ids + const sanitizedSlug = project.slug.replace(/[.]/g, "-"); // MeiliSearch doesn't allow dots in ids + projectSearchItems.push({ + id: `${runId}--${sanitizedSlug}`, title: project.name, type: "project", + runId, }); let addedRepositoryCount = 0; @@ -125,10 +132,11 @@ export class DigestCron { runId, id: `${provider}-${githubUser.login}`, }); - this.searchService.upsert("contributor", { - id: `${provider}-${githubUser.login}`, + contributorSearchItems.push({ + id: `${runId}--${provider}-${githubUser.login}`, title: githubUser.name || githubUser.login, type: "contributor", + runId, }); await this.contributorsRepository.upsertRelationWithRepository({ @@ -153,10 +161,11 @@ export class DigestCron { contributorId, id: `${provider}-${issue.id}`, }); - this.searchService.upsert("contribution", { - id: `${provider}-${issue.id}`, + contributionSearchItems.push({ + id: `${runId}--${provider}-${issue.id}`, title: issue.title, type: "contribution", + runId, }); } @@ -213,6 +222,12 @@ export class DigestCron { await this.contributorsRepository.deleteAllButWithRunId(runId); await this.repositoriesRepository.deleteAllButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); + await this.searchService.upsert("project", projectSearchItems); + await this.searchService.upsert("contribution", contributionSearchItems); + await this.searchService.upsert("contributor", contributorSearchItems); + await this.searchService.deleteAllButWithRunId("project", runId); + await this.searchService.deleteAllButWithRunId("contribution", runId); + await this.searchService.deleteAllButWithRunId("contributor", runId); } catch (error) { captureException(error, { tags: { type: "CRON" } }); } diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 6dc39625..b549d6f9 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -32,25 +32,58 @@ export class SearchService { public upsert = async ( index: SearchType, - data: SearchItem, + data: SearchItem[], ): Promise => { this.logger.info({ - message: `Upserting ${data.title} in ${index}`, + message: `Upserting ${data.length} items to ${index}`, + }); + await this.meilisearch.index(index).updateDocuments(data); + this.logger.info({ message: `Upserted ${data.length} items to ${index}` }); + }; + + public deleteAllButWithRunId = async ( + index: SearchType, + runId: string, + ): Promise => { + this.logger.info({ + message: `Deleting all ${index} but with runId ${runId}`, + }); + await this.meilisearch.index(index).deleteDocuments({ + filter: `NOT runId=${runId}`, }); - await this.meilisearch.index(index).updateDocuments([data]); this.logger.info({ - message: `Upserted ${data.title} in ${index}`, + message: `Deleted all ${index} but with runId ${runId}`, }); }; public ensureIndexes = async (): Promise => { - await this.meilisearch.createIndex("project"); + await this.meilisearch.createIndex("project", { + primaryKey: "id", + }); this.logger.info({ message: "project index created" }); - await this.meilisearch.createIndex("contribution"); + await this.meilisearch.createIndex("contribution", { + primaryKey: "id", + }); this.logger.info({ message: "contribution index created" }); - await this.meilisearch.createIndex("contributor"); + await this.meilisearch.createIndex("contributor", { + primaryKey: "id", + }); this.logger.info({ message: "contributor index created" }); + + await this.updateFilterableAttributes(); }; + + private async updateFilterableAttributes(): Promise { + await this.meilisearch + .index("project") + .updateFilterableAttributes(["runId"]); + await this.meilisearch + .index("contribution") + .updateFilterableAttributes(["runId"]); + await this.meilisearch + .index("contributor") + .updateFilterableAttributes(["runId"]); + } } diff --git a/api/src/search/types.ts b/api/src/search/types.ts index 4c34e538..7a6bab69 100644 --- a/api/src/search/types.ts +++ b/api/src/search/types.ts @@ -8,6 +8,7 @@ export interface SearchItem { id: string; title: string; type: SearchType; + runId: string; } export type SearchType = "project" | "contribution" | "contributor"; From a131b1248f5bc3cd2bea39bbd9408adaa1e9ce54 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sat, 21 Dec 2024 22:34:01 +0100 Subject: [PATCH 06/13] feat: refactor DigestCron to streamline upsert operations and remove unused search item arrays --- api/src/digest/cron.ts | 64 +++++++++++------------------- api/src/search/service.ts | 11 ++--- api/src/search/types.ts | 1 - packages/models/src/_base/index.ts | 5 +++ 4 files changed, 35 insertions(+), 46 deletions(-) diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 596654b5..b857bfb8 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -8,7 +8,6 @@ import { GithubService } from "src/github/service"; import { LoggerService } from "src/logger/service"; import { ProjectRepository } from "src/project/repository"; import { RepositoryRepository } from "src/repository/repository"; -import { SearchItem } from "src/search/types"; import { SearchService } from "src/search/service"; import { Service } from "typedi"; @@ -70,23 +69,15 @@ export class DigestCron { const projectsFromDataFolder = await this.dataService.listProjects(); - const projectSearchItems: SearchItem[] = []; - const contributionSearchItems: SearchItem[] = []; - const contributorSearchItems: SearchItem[] = []; - for (const project of projectsFromDataFolder) { - const [{ id: projectId }] = await this.projectsRepository.upsert({ + const projectEntity = { ...project, runId, - id: project.slug, - }); - const sanitizedSlug = project.slug.replace(/[.]/g, "-"); // MeiliSearch doesn't allow dots in ids - projectSearchItems.push({ - id: `${runId}--${sanitizedSlug}`, - title: project.name, - type: "project", - runId, - }); + id: project.slug.replace(/[.]/g, "-"), // MeiliSearch doesn't allow dots in ids, + }; + const [{ id: projectId }] = + await this.projectsRepository.upsert(projectEntity); + await this.searchService.upsert("project", projectEntity); let addedRepositoryCount = 0; try { @@ -123,21 +114,18 @@ export class DigestCron { if (githubUser.type !== "User") continue; - const [{ id: contributorId }] = - await this.contributorsRepository.upsert({ - name: githubUser.name || githubUser.login, - username: githubUser.login, - url: githubUser.html_url, - avatarUrl: githubUser.avatar_url, - runId, - id: `${provider}-${githubUser.login}`, - }); - contributorSearchItems.push({ - id: `${runId}--${provider}-${githubUser.login}`, - title: githubUser.name || githubUser.login, - type: "contributor", + const contributorEntity = { + name: githubUser.name || githubUser.login, + username: githubUser.login, + url: githubUser.html_url, + avatarUrl: githubUser.avatar_url, runId, - }); + id: `${provider}-${githubUser.login}`, + }; + + const [{ id: contributorId }] = + await this.contributorsRepository.upsert(contributorEntity); + await this.searchService.upsert("contributor", contributorEntity); await this.contributorsRepository.upsertRelationWithRepository({ contributorId, @@ -147,7 +135,7 @@ export class DigestCron { }); const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE"; - await this.contributionsRepository.upsert({ + const contributionEntity = { title: issue.title, type, updatedAt: issue.updated_at, @@ -160,13 +148,12 @@ export class DigestCron { repositoryId, contributorId, id: `${provider}-${issue.id}`, - }); - contributionSearchItems.push({ - id: `${runId}--${provider}-${issue.id}`, - title: issue.title, - type: "contribution", - runId, - }); + } as const; + await this.contributionsRepository.upsert(contributionEntity); + await this.searchService.upsert( + "contribution", + contributionEntity, + ); } const repoContributors = @@ -222,9 +209,6 @@ export class DigestCron { await this.contributorsRepository.deleteAllButWithRunId(runId); await this.repositoriesRepository.deleteAllButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); - await this.searchService.upsert("project", projectSearchItems); - await this.searchService.upsert("contribution", contributionSearchItems); - await this.searchService.upsert("contributor", contributorSearchItems); await this.searchService.deleteAllButWithRunId("project", runId); await this.searchService.deleteAllButWithRunId("contribution", runId); await this.searchService.deleteAllButWithRunId("contributor", runId); diff --git a/api/src/search/service.ts b/api/src/search/service.ts index b549d6f9..a648719d 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -1,5 +1,6 @@ import { SearchItem, SearchType } from "./types"; +import { BaseSearchItem } from "@dzcode.io/models/dist/_base"; import { ConfigService } from "src/config/service"; import { LoggerService } from "src/logger/service"; import { MeiliSearch } from "meilisearch"; @@ -30,15 +31,15 @@ export class SearchService { return []; }; - public upsert = async ( + public upsert = async ( index: SearchType, - data: SearchItem[], + data: T, ): Promise => { this.logger.info({ - message: `Upserting ${data.length} items to ${index}`, + message: `Upserting "${data.id}" item to ${index}`, }); - await this.meilisearch.index(index).updateDocuments(data); - this.logger.info({ message: `Upserted ${data.length} items to ${index}` }); + await this.meilisearch.index(index).updateDocuments([data]); + this.logger.info({ message: `Upserted "${data.id}" item to ${index}` }); }; public deleteAllButWithRunId = async ( diff --git a/api/src/search/types.ts b/api/src/search/types.ts index 7a6bab69..59e28a90 100644 --- a/api/src/search/types.ts +++ b/api/src/search/types.ts @@ -7,7 +7,6 @@ export interface GetSearchResponse extends GeneralResponse { export interface SearchItem { id: string; title: string; - type: SearchType; runId: string; } diff --git a/packages/models/src/_base/index.ts b/packages/models/src/_base/index.ts index e4c2511d..63e5fd99 100644 --- a/packages/models/src/_base/index.ts +++ b/packages/models/src/_base/index.ts @@ -2,3 +2,8 @@ export type BaseEntity = { id: string; runId: string; }; + +export type BaseSearchItem = { + id: string; + runId: string; +}; From 42778c6d61bc5f5fab81b248c857db216de8c532 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sat, 21 Dec 2024 23:04:32 +0100 Subject: [PATCH 07/13] chore: introduce setupIndexes search method that ensure indexes and add update filterable attributes --- api/src/app/index.ts | 2 +- api/src/search/service.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/src/app/index.ts b/api/src/app/index.ts index 5af6505e..c4b26022 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -41,7 +41,7 @@ useContainer(Container); // eslint-disable-line react-hooks/rules-of-hooks // Initialize Search Service const searchService = Container.get(SearchService); - await searchService.ensureIndexes(); + await searchService.setupIndexes(); // Add crons to DI container const CronServices = [DigestCron]; diff --git a/api/src/search/service.ts b/api/src/search/service.ts index a648719d..3948d9ef 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -57,7 +57,12 @@ export class SearchService { }); }; - public ensureIndexes = async (): Promise => { + public setupIndexes = async (): Promise => { + await this.ensureIndexes(); + await this.updateFilterableAttributes(); + }; + + private ensureIndexes = async (): Promise => { await this.meilisearch.createIndex("project", { primaryKey: "id", }); @@ -72,8 +77,6 @@ export class SearchService { primaryKey: "id", }); this.logger.info({ message: "contributor index created" }); - - await this.updateFilterableAttributes(); }; private async updateFilterableAttributes(): Promise { From 229e3411fb534a47b17b2375d7730e04c5affcb9 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sat, 21 Dec 2024 23:24:20 +0100 Subject: [PATCH 08/13] feat: update GetSearchResponse to use MultiSearchResponse and simplify SearchItem type chore: introduce setupIndexes search method that ensure indexes and add update filterable attributes --- api/src/search/service.ts | 8 +++++--- api/src/search/types.ts | 14 ++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 3948d9ef..1f31b91f 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -1,4 +1,4 @@ -import { SearchItem, SearchType } from "./types"; +import { SearchResults, SearchType } from "./types"; import { BaseSearchItem } from "@dzcode.io/models/dist/_base"; import { ConfigService } from "src/config/service"; @@ -26,9 +26,11 @@ export class SearchService { }); } - public search = async (query: string): Promise => { + public search = async (query: string): Promise => { this.logger.info({ message: `Searching for ${query}` }); - return []; + return { + results: [], + }; }; public upsert = async ( diff --git a/api/src/search/types.ts b/api/src/search/types.ts index 59e28a90..33c8ce19 100644 --- a/api/src/search/types.ts +++ b/api/src/search/types.ts @@ -1,13 +1,15 @@ +import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; +import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; import { GeneralResponse } from "src/app/types"; +import { MultiSearchResponse } from "meilisearch"; +import { ProjectEntity } from "@dzcode.io/models/dist/project"; export interface GetSearchResponse extends GeneralResponse { - searchResults: Array; + searchResults: SearchResults; } -export interface SearchItem { - id: string; - title: string; - runId: string; -} +export type SearchResults = MultiSearchResponse; + +type SearchItem = ProjectEntity | ContributionEntity | ContributorEntity; export type SearchType = "project" | "contribution" | "contributor"; From 8e5201b4582a424e12eaffe142fd4f6c36b5c349 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sun, 22 Dec 2024 15:44:37 +0100 Subject: [PATCH 09/13] refactor: rename setupIndexes to setupSearch for clarity and streamline search service initialization --- api/src/app/index.ts | 2 +- api/src/search/service.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/app/index.ts b/api/src/app/index.ts index c4b26022..dc142be4 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -41,7 +41,7 @@ useContainer(Container); // eslint-disable-line react-hooks/rules-of-hooks // Initialize Search Service const searchService = Container.get(SearchService); - await searchService.setupIndexes(); + await searchService.setupSearch(); // Add crons to DI container const CronServices = [DigestCron]; diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 1f31b91f..0ec83784 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -59,12 +59,12 @@ export class SearchService { }); }; - public setupIndexes = async (): Promise => { - await this.ensureIndexes(); + public setupSearch = async (): Promise => { + await this.setupIndexes(); await this.updateFilterableAttributes(); }; - private ensureIndexes = async (): Promise => { + private setupIndexes = async (): Promise => { await this.meilisearch.createIndex("project", { primaryKey: "id", }); From 278351b2e906bffa3fe689c1b64261b9564aa3ea Mon Sep 17 00:00:00 2001 From: omdxp Date: Sun, 22 Dec 2024 16:00:12 +0100 Subject: [PATCH 10/13] refactor: update entity types in DigestCron and SearchService for consistency and clarity --- api/src/digest/cron.ts | 39 ++++++++++++++++++------------ api/src/search/service.ts | 4 +-- packages/models/src/_base/index.ts | 7 +----- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index b857bfb8..43bebc24 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -1,12 +1,15 @@ import { captureException, cron } from "@sentry/node"; import { ContributionRepository } from "src/contribution/repository"; +import { ContributionRow } from "src/contribution/table"; import { ContributorRepository } from "src/contributor/repository"; +import { ContributorRow } from "src/contributor/table"; import { CronJob } from "cron"; import { DataService } from "src/data/service"; import { GithubService } from "src/github/service"; import { LoggerService } from "src/logger/service"; import { ProjectRepository } from "src/project/repository"; +import { ProjectRow } from "src/project/table"; import { RepositoryRepository } from "src/repository/repository"; import { SearchService } from "src/search/service"; import { Service } from "typedi"; @@ -70,10 +73,10 @@ export class DigestCron { const projectsFromDataFolder = await this.dataService.listProjects(); for (const project of projectsFromDataFolder) { - const projectEntity = { - ...project, + const projectEntity: ProjectRow = { runId, - id: project.slug.replace(/[.]/g, "-"), // MeiliSearch doesn't allow dots in ids, + id: project.slug.replace(/[.]/g, "-"), // NOTE-OB: MeiliSearch doesn't allow dots in ids + name: project.name, }; const [{ id: projectId }] = await this.projectsRepository.upsert(projectEntity); @@ -114,7 +117,7 @@ export class DigestCron { if (githubUser.type !== "User") continue; - const contributorEntity = { + const contributorEntity: ContributorRow = { name: githubUser.name || githubUser.login, username: githubUser.login, url: githubUser.html_url, @@ -135,7 +138,7 @@ export class DigestCron { }); const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE"; - const contributionEntity = { + const contributionEntity: ContributionRow = { title: issue.title, type, updatedAt: issue.updated_at, @@ -170,15 +173,17 @@ export class DigestCron { const contributor = await this.githubService.getUser({ username: repoContributor.login, }); + const contributorEntity: ContributorRow = { + name: contributor.name || contributor.login, + username: contributor.login, + url: contributor.html_url, + avatarUrl: contributor.avatar_url, + runId, + id: `${provider}-${contributor.login}`, + }; const [{ id: contributorId }] = - await this.contributorsRepository.upsert({ - name: contributor.name || contributor.login, - username: contributor.login, - url: contributor.html_url, - avatarUrl: contributor.avatar_url, - runId, - id: `${provider}-${contributor.login}`, - }); + await this.contributorsRepository.upsert(contributorEntity); + await this.searchService.upsert("contributor", contributorEntity); await this.contributorsRepository.upsertRelationWithRepository({ contributorId, @@ -209,9 +214,11 @@ export class DigestCron { await this.contributorsRepository.deleteAllButWithRunId(runId); await this.repositoriesRepository.deleteAllButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); - await this.searchService.deleteAllButWithRunId("project", runId); - await this.searchService.deleteAllButWithRunId("contribution", runId); - await this.searchService.deleteAllButWithRunId("contributor", runId); + await Promise.all([ + this.searchService.deleteAllButWithRunId("project", runId), + this.searchService.deleteAllButWithRunId("contribution", runId), + this.searchService.deleteAllButWithRunId("contributor", runId), + ]); } catch (error) { captureException(error, { tags: { type: "CRON" } }); } diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 0ec83784..087ec03f 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -1,6 +1,6 @@ import { SearchResults, SearchType } from "./types"; -import { BaseSearchItem } from "@dzcode.io/models/dist/_base"; +import { BaseEntity } from "@dzcode.io/models/dist/_base"; import { ConfigService } from "src/config/service"; import { LoggerService } from "src/logger/service"; import { MeiliSearch } from "meilisearch"; @@ -33,7 +33,7 @@ export class SearchService { }; }; - public upsert = async ( + public upsert = async ( index: SearchType, data: T, ): Promise => { diff --git a/packages/models/src/_base/index.ts b/packages/models/src/_base/index.ts index 63e5fd99..64a2cfaf 100644 --- a/packages/models/src/_base/index.ts +++ b/packages/models/src/_base/index.ts @@ -1,9 +1,4 @@ export type BaseEntity = { id: string; - runId: string; -}; - -export type BaseSearchItem = { - id: string; - runId: string; + runId?: string; }; From b49c098e9a7a09b98c6a203085490dabcce2a0dd Mon Sep 17 00:00:00 2001 From: omdxp Date: Sun, 22 Dec 2024 16:07:55 +0100 Subject: [PATCH 11/13] refactor: remove redundant type assertion in contributionEntity creation --- api/src/digest/cron.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 43bebc24..24bb3ff7 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -151,7 +151,7 @@ export class DigestCron { repositoryId, contributorId, id: `${provider}-${issue.id}`, - } as const; + }; await this.contributionsRepository.upsert(contributionEntity); await this.searchService.upsert( "contribution", From 62a8ecbc465f103e27a3b4a1c36aaff7d1bb5c66 Mon Sep 17 00:00:00 2001 From: omdxp Date: Sun, 22 Dec 2024 16:55:50 +0100 Subject: [PATCH 12/13] refactor: streamline index setup by introducing upsertIndex method for improved index management --- api/src/search/service.ts | 29 +++++++++++++++-------------- api/src/search/types.ts | 6 +++--- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 087ec03f..2206d641 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -65,22 +65,23 @@ export class SearchService { }; private setupIndexes = async (): Promise => { - await this.meilisearch.createIndex("project", { - primaryKey: "id", - }); - this.logger.info({ message: "project index created" }); - - await this.meilisearch.createIndex("contribution", { - primaryKey: "id", - }); - this.logger.info({ message: "contribution index created" }); - - await this.meilisearch.createIndex("contributor", { - primaryKey: "id", - }); - this.logger.info({ message: "contributor index created" }); + await this.upsertIndex("project"); + await this.upsertIndex("contribution"); + await this.upsertIndex("contributor"); }; + private async upsertIndex(index: SearchType): Promise { + try { + await this.meilisearch.getIndex(index); + this.logger.info({ message: `${index} index already exists` }); + } catch { + await this.meilisearch.createIndex(index, { + primaryKey: "id", + }); + this.logger.info({ message: `${index} index created` }); + } + } + private async updateFilterableAttributes(): Promise { await this.meilisearch .index("project") diff --git a/api/src/search/types.ts b/api/src/search/types.ts index 33c8ce19..46f846fd 100644 --- a/api/src/search/types.ts +++ b/api/src/search/types.ts @@ -8,8 +8,8 @@ export interface GetSearchResponse extends GeneralResponse { searchResults: SearchResults; } -export type SearchResults = MultiSearchResponse; - -type SearchItem = ProjectEntity | ContributionEntity | ContributorEntity; +export type SearchResults = MultiSearchResponse< + ProjectEntity | ContributionEntity | ContributorEntity +>; export type SearchType = "project" | "contribution" | "contributor"; From 63e1ec77b2b72f487e2d98039668d4597c7f37fb Mon Sep 17 00:00:00 2001 From: omdxp Date: Sun, 22 Dec 2024 17:16:28 +0100 Subject: [PATCH 13/13] refactor: remove default value for runId in contributors and projects tables for consistency --- api/src/contributor/table.ts | 2 +- api/src/project/table.ts | 2 +- packages/models/src/_base/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/contributor/table.ts b/api/src/contributor/table.ts index 35b81336..7854fe82 100644 --- a/api/src/contributor/table.ts +++ b/api/src/contributor/table.ts @@ -8,7 +8,7 @@ export const contributorsTable = pgTable("contributors", { recordImportedAt: text("record_imported_at") .notNull() .default(sql`CURRENT_TIMESTAMP`), - runId: text("run_id").notNull().default("initial-run-id"), + runId: text("run_id").notNull(), name: text("name").notNull(), username: text("username").notNull(), url: text("url").notNull().unique(), diff --git a/api/src/project/table.ts b/api/src/project/table.ts index 01aad9c1..4847c1c7 100644 --- a/api/src/project/table.ts +++ b/api/src/project/table.ts @@ -8,7 +8,7 @@ export const projectsTable = pgTable("projects", { .notNull() .default(sql`CURRENT_TIMESTAMP`), name: text("name").notNull(), - runId: text("run_id").notNull().default("initial-run-id"), + runId: text("run_id").notNull(), }); projectsTable.$inferSelect satisfies ProjectEntity; diff --git a/packages/models/src/_base/index.ts b/packages/models/src/_base/index.ts index 64a2cfaf..e4c2511d 100644 --- a/packages/models/src/_base/index.ts +++ b/packages/models/src/_base/index.ts @@ -1,4 +1,4 @@ export type BaseEntity = { id: string; - runId?: string; + runId: string; };