From 28b3e560d4dff87620373c5aec92348595cd5956 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Wed, 1 Jan 2025 13:55:49 +0100 Subject: [PATCH 1/5] added AI service --- api/package.json | 1 + api/src/ai/service.ts | 66 ++++++++++++++++++++++++++++++++++++++++ api/src/config/types.ts | 3 ++ api/src/fetch/service.ts | 23 +++++++++++--- api/src/fetch/types.ts | 1 + package-lock.json | 53 ++++++++++++++++++++++++++++++-- 6 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 api/src/ai/service.ts diff --git a/api/package.json b/api/package.json index 79c21d636..fd862625f 100644 --- a/api/package.json +++ b/api/package.json @@ -15,6 +15,7 @@ "@types/make-fetch-happen": "^10.0.4", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "class-validator-jsonschema": "^5.0.1", "cors": "^2.8.5", "cron": "^3.1.7", "dotenv": "^16.4.5", diff --git a/api/src/ai/service.ts b/api/src/ai/service.ts new file mode 100644 index 000000000..5ac6ce97f --- /dev/null +++ b/api/src/ai/service.ts @@ -0,0 +1,66 @@ +import { ConfigService } from "src/config/service"; +import { LoggerService } from "src/logger/service"; +import { Service } from "typedi"; +import { targetConstructorToSchema } from "class-validator-jsonschema"; +import { FetchService } from "src/fetch/service"; +import { ClassConstructor, plainToClass } from "class-transformer"; +import { validateSync } from "class-validator"; + +type AIChat = { role: "user" | "system"; content: string }; + +type OpenAIResponse = { + choices: Array<{ message: AIChat }>; +}; + +@Service() +export class AIService { + constructor( + private readonly configService: ConfigService, + private readonly logger: LoggerService, + private readonly fetchService: FetchService, + ) {} + + public query = async ( + payload: AIChat[], + ResponseDto: ClassConstructor, + ): Promise => { + const schema = targetConstructorToSchema(ResponseDto); + + const payloadWithValidationPrompt: AIChat[] = [ + { + role: "system", + content: `system response must strictly follow the schema:\n${JSON.stringify(schema)}`, + }, + ...payload, + ]; + + const { OPENAI_KEY } = this.configService.env(); + + // todo: cache response + const res = await this.fetchService.post( + "https://api.openai.com/v1/chat/completions", + { + headers: { Authorization: `Bearer ${OPENAI_KEY}` }, + body: { + model: "gpt-4o", + messages: payloadWithValidationPrompt, + }, + }, + ); + + const chatResponseUnchecked = JSON.parse(res.choices[0].message.content) as T; + + const output = plainToClass(ResponseDto, chatResponseUnchecked); + const errors = validateSync(output); + + if (errors.length > 0) + throw new Error( + `⚠️ Errors in AI response in the following keys:${errors.reduce( + (pV, cV) => (pV += "\n" + cV.property + " : " + JSON.stringify(cV.constraints)), + "", + )}`, + ); + + return output; + }; +} diff --git a/api/src/config/types.ts b/api/src/config/types.ts index 09ccc1caa..dd5f22086 100644 --- a/api/src/config/types.ts +++ b/api/src/config/types.ts @@ -40,4 +40,7 @@ export class EnvRecord { } MEILISEARCH_MASTER_KEY = "default"; + + @IsString() + OPENAI_KEY!: string; } diff --git a/api/src/fetch/service.ts b/api/src/fetch/service.ts index d48684405..e757cdfe0 100644 --- a/api/src/fetch/service.ts +++ b/api/src/fetch/service.ts @@ -1,4 +1,4 @@ -import { defaults } from "make-fetch-happen"; +import { defaults, FetchOptions } from "make-fetch-happen"; import { ConfigService } from "src/config/service"; import { LoggerService } from "src/logger/service"; import { Service } from "typedi"; @@ -18,6 +18,21 @@ export class FetchService { }); } + public post = async ( + url: string, + { headers = {}, body }: FetchConfig = {}, + ): Promise> => { + const response = await this.fetch(url, { + headers: { + "Content-Type": "application/json", + ...headers, + }, + method: "POST", + body: body ? JSON.stringify(body) : undefined, + }); + return response; + }; + public get = async ( url: string, { params = {}, headers = {} }: FetchConfig = {}, @@ -25,14 +40,14 @@ export class FetchService { const _url = new URL(url); Object.keys(params).forEach((key) => _url.searchParams.append(key, String(params[key]))); - const response = await this.fetch(_url.toString(), { headers }); + const response = await this.fetch(_url.toString(), { headers, method: "GET" }); return response; }; private makeFetchHappenInstance; - private async fetch(url: string, { headers }: Omit = {}) { + private async fetch(url: string, options: FetchOptions) { this.logger.info({ message: `Fetching ${url}` }); - const response = await this.makeFetchHappenInstance(url, { headers }); + const response = await this.makeFetchHappenInstance(url, options); const jsonResponse = (await response.json()) as T; return jsonResponse; } diff --git a/api/src/fetch/types.ts b/api/src/fetch/types.ts index b426fb2e3..07bd7b558 100644 --- a/api/src/fetch/types.ts +++ b/api/src/fetch/types.ts @@ -1,4 +1,5 @@ export interface FetchConfig { params?: Record; headers?: Record; + body?: Record; } diff --git a/package-lock.json b/package-lock.json index a6b5ed534..712720ea7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -72,6 +72,7 @@ "@types/make-fetch-happen": "^10.0.4", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "class-validator-jsonschema": "^5.0.1", "cors": "^2.8.5", "cron": "^3.1.7", "dotenv": "^16.4.5", @@ -8600,6 +8601,29 @@ "validator": "^13.9.0" } }, + "node_modules/class-validator-jsonschema": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/class-validator-jsonschema/-/class-validator-jsonschema-5.0.1.tgz", + "integrity": "sha512-9uTdo5jSnJUj7f0dS8YZDqM0Fv1Uky0BWefswnNa2F4nRcKPCiEb5z3nDUaXyEzcERCrizE+0AGDSao1uSNX9g==", + "license": "MIT", + "dependencies": { + "lodash.groupby": "^4.6.0", + "lodash.merge": "^4.6.2", + "openapi3-ts": "^3.0.0", + "reflect-metadata": "^0.1.13", + "tslib": "^2.4.1" + }, + "peerDependencies": { + "class-transformer": "^0.4.0 || ^0.5.0", + "class-validator": "^0.14.0" + } + }, + "node_modules/class-validator-jsonschema/node_modules/reflect-metadata": { + "version": "0.1.14", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz", + "integrity": "sha512-ZhYeb6nRaXCfhnndflDK8qI6ZQ/YcWZCISRAWICW9XYqMUwjZM9Z0DveWX/ABN01oxSHwVxKQmxeYZSsm0jh5A==", + "license": "Apache-2.0" + }, "node_modules/clean-regexp": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/clean-regexp/-/clean-regexp-1.0.0.tgz", @@ -17905,6 +17929,12 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "license": "MIT" }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, "node_modules/lodash.ismatch": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", @@ -17923,7 +17953,6 @@ "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true, "license": "MIT" }, "node_modules/lodash.once": { @@ -20555,6 +20584,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openapi3-ts": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/openapi3-ts/-/openapi3-ts-3.2.0.tgz", + "integrity": "sha512-/ykNWRV5Qs0Nwq7Pc0nJ78fgILvOT/60OxEmB3v7yQ8a8Bwcm43D4diaYazG/KBn6czA+52XYy931WFLMCUeSg==", + "license": "MIT", + "dependencies": { + "yaml": "^2.2.1" + } + }, + "node_modules/openapi3-ts/node_modules/yaml": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", + "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -26268,7 +26318,6 @@ "version": "2.8.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz", "integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==", - "dev": true, "license": "0BSD" }, "node_modules/tsscmp": { From 3468a4567da06b564a16d37b77e4d088a7cec003 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Wed, 1 Jan 2025 13:56:53 +0100 Subject: [PATCH 2/5] integrate AI service for translating project and contributor names, and issue titles to Arabic --- api/src/digest/cron.ts | 91 +++++++++++++++++++++++++++++++++++------- api/src/digest/dto.ts | 17 ++++++++ 2 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 api/src/digest/dto.ts diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 9b584a10c..4941e1204 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -14,6 +14,8 @@ import { RepositoryRepository } from "src/repository/repository"; import { SearchService } from "src/search/service"; import { Service } from "typedi"; import { TagRepository } from "src/tag/repository"; +import { AIService } from "src/ai/service"; +import { AIResponseTranslateNameDto, AIResponseTranslateTitleDto } from "./dto"; @Service() export class DigestCron { @@ -29,7 +31,8 @@ export class DigestCron { private readonly contributionsRepository: ContributionRepository, private readonly contributorsRepository: ContributorRepository, private readonly searchService: SearchService, - private readonly tagsRepository: TagRepository, + private readonly tagRepository: TagRepository, + private readonly aiService: AIService, ) { const SentryCronJob = cron.instrumentCron(CronJob, "DigestCron"); new SentryCronJob( @@ -81,10 +84,29 @@ export class DigestCron { // or uncomment to skip the cron // if (Math.random()) return; + const projectTitleSystemPrompt = `user will give you an open-source project name, and you will translate it to Arabic.`; + const contributorNameSystemPrompt = `user will give you an open-source contributor name, and you will translate it to Arabic. +if the name contain both english and arabic only keep the parts related to the language.`; + const issueTitleSystemPrompt = `user will give you an open-source issue/PR title, and you will translate it to Arabic.`; + for (const project of projectsFromDataFolder) { - // todo: call AIService - const name_en = project.name; - const name_ar = `ar ${name_en}`; + let name_en = project.name; + let name_ar = name_en; + + try { + const aiRes = await this.aiService.query( + [ + { role: "system", content: projectTitleSystemPrompt }, + { role: "user", content: name_en }, + ], + AIResponseTranslateNameDto, + ); + + name_en = aiRes.name_en; + name_ar = aiRes.name_ar; + } catch (error) { + captureException(error, { tags: { type: "CRON" } }); + } const projectEntity: ProjectRow = { runId, @@ -94,7 +116,7 @@ export class DigestCron { }; const [{ id: projectId }] = await this.projectsRepository.upsert(projectEntity); for (const tagId of project.tags || []) { - await this.tagsRepository.upsert({ id: tagId, runId }); + await this.tagRepository.upsert({ id: tagId, runId }); await this.projectsRepository.upsertRelationWithTag({ projectId, tagId, runId }); } await this.searchService.upsert("project", projectEntity); @@ -133,9 +155,22 @@ export class DigestCron { if (githubUser.type !== "User") continue; - // todo: call AIService - const name_en = githubUser.name || githubUser.login; - const name_ar = `ar ${name_en}`; + let name_en = githubUser.name || githubUser.login; + let name_ar = name_en; + try { + const aiRes = await this.aiService.query( + [ + { role: "system", content: contributorNameSystemPrompt }, + { role: "user", content: name_en }, + ], + AIResponseTranslateNameDto, + ); + + name_en = aiRes.name_en; + name_ar = aiRes.name_ar; + } catch (error) { + captureException(error, { tags: { type: "CRON" } }); + } const contributorEntity: ContributorRow = { name_en, @@ -160,9 +195,22 @@ export class DigestCron { const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE"; - // todo: call AIService - const title_en = issue.title; - const title_ar = `ar ${title_en}`; + let title_en = issue.title; + let title_ar = `ar ${title_en}`; + try { + const aiRes = await this.aiService.query( + [ + { role: "system", content: issueTitleSystemPrompt }, + { role: "user", content: title_en }, + ], + AIResponseTranslateTitleDto, + ); + + title_en = aiRes.title_en; + title_ar = aiRes.title_ar; + } catch (error) { + captureException(error, { tags: { type: "CRON" } }); + } const contributionEntity: ContributionRow = { title_en, @@ -194,9 +242,22 @@ export class DigestCron { username: repoContributor.login, }); - // todo: call AIService - const name_en = contributor.name || contributor.login; - const name_ar = `ar ${name_en}`; + let name_en = contributor.name || contributor.login; + let name_ar = `ar ${name_en}`; + try { + const aiRes = await this.aiService.query( + [ + { role: "system", content: contributorNameSystemPrompt }, + { role: "user", content: name_en }, + ], + AIResponseTranslateNameDto, + ); + + name_en = aiRes.name_en; + name_ar = aiRes.name_ar; + } catch (error) { + captureException(error, { tags: { type: "CRON" } }); + } const contributorEntity: ContributorRow = { name_en, @@ -244,7 +305,7 @@ export class DigestCron { await this.projectsRepository.deleteAllRelationWithTagButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); - await this.tagsRepository.deleteAllButWithRunId(runId); + await this.tagRepository.deleteAllButWithRunId(runId); await Promise.all([ this.searchService.deleteAllButWithRunId("project", runId), diff --git a/api/src/digest/dto.ts b/api/src/digest/dto.ts new file mode 100644 index 000000000..46b9ac831 --- /dev/null +++ b/api/src/digest/dto.ts @@ -0,0 +1,17 @@ +import { IsString } from "class-validator"; + +export class AIResponseTranslateNameDto { + @IsString() + name_en!: string; + + @IsString() + name_ar!: string; +} + +export class AIResponseTranslateTitleDto { + @IsString() + title_en!: string; + + @IsString() + title_ar!: string; +} From 5ef93260e4b894fd7b4fda67f5e8f3dd5594f832 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Wed, 1 Jan 2025 13:57:05 +0100 Subject: [PATCH 3/5] remove unnecessary dir attributes from contribution and project cards --- web/src/components/contribution-card.tsx | 1 - web/src/components/project-card.tsx | 1 - web/src/pages/contribute/contribution/index.tsx | 5 +---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/web/src/components/contribution-card.tsx b/web/src/components/contribution-card.tsx index 03fc4b145..152e3cc3c 100644 --- a/web/src/components/contribution-card.tsx +++ b/web/src/components/contribution-card.tsx @@ -20,7 +20,6 @@ export function ContributionCard({ return ( {/* TODO-ZM: more tailored design for /contribute/:slug page instead of copy-pasting components from /contribute */} -
+

From b573417e40095bf29a03df80f849725ed7f1b9f8 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Wed, 1 Jan 2025 14:20:15 +0100 Subject: [PATCH 4/5] update README to clarify GitHub token setup and add placeholder for OpenAI key --- api/README.md | 3 ++- api/src/config/types.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/README.md b/api/README.md index 1608b93dc..4e0d1fc90 100644 --- a/api/README.md +++ b/api/README.md @@ -24,8 +24,9 @@ NODE_ENV=development Keep in mind that you have limited calls to the GitHub API (60 calls per hour). The [FetchService](./api/src/fetch/service.ts) does a great job of caching these calls so it doesn't unnecessarily consume the GitHub API quota. If you wish to extend the limit from 60 to 5000, simply create a [GitHub Personal Access Token](https://github.com/settings/tokens) (make sure it has `Access public repositories` checked), and set it in `./api/.env` like this: ```.env -GITHUB_TOKEN=Paste_Your_Token_Here +GITHUB_TOKEN=Paste_your_token_here NODE_ENV=development +OPENAI_KEY=Pase_your_key_here ``` **Note:** If the README is still unclear, please create a PR with your suggested changes/additions. diff --git a/api/src/config/types.ts b/api/src/config/types.ts index dd5f22086..69faa53c5 100644 --- a/api/src/config/types.ts +++ b/api/src/config/types.ts @@ -42,5 +42,5 @@ export class EnvRecord { MEILISEARCH_MASTER_KEY = "default"; @IsString() - OPENAI_KEY!: string; + OPENAI_KEY = "no-key"; } From 5e07302712b02a44f03bf2b5cfff12af6e8739f4 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 2 Jan 2025 10:21:11 +0100 Subject: [PATCH 5/5] add TODO comment to handle deletion of old containers in docker-compose --- api/oracle-cloud/deploy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/oracle-cloud/deploy.ts b/api/oracle-cloud/deploy.ts index 087d4dc06..de5024fd8 100644 --- a/api/oracle-cloud/deploy.ts +++ b/api/oracle-cloud/deploy.ts @@ -52,6 +52,7 @@ const appPath = "~/app"; const sshPrefix = "ssh -o StrictHostKeyChecking=no " + (sshKeyPath ? `-i ${sshKeyPath} ` : "") + sshServer + " "; +// todo-ZM: let docker-compose handle deletion of old containers // Check for existing containers logs = execSync(sshPrefix + '"sudo docker ps -aq"');