From fd99b016cc54978b7c68a71db778c9abd188f1e1 Mon Sep 17 00:00:00 2001 From: omdxp Date: Fri, 20 Dec 2024 13:39:45 +0100 Subject: [PATCH 1/8] Enhance MeiliSearch integration by adding index creation --- api/package.json | 2 +- api/src/app/index.ts | 4 ++++ api/src/search/service.ts | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index 06686b19..d5a192e2 100644 --- a/api/package.json +++ b/api/package.json @@ -77,7 +77,7 @@ "lint:prettier": "prettier --config ../packages/tooling/.prettierrc --ignore-path ../packages/tooling/.prettierignore --log-level warn", "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error", "lint:tsc": "tspc --noEmit", - "start": "wait-port postgres:5432 && delay 2 && node dist/app/index.js", + "start": "wait-port postgres:5432 && wait-port meilisearch:7700 && delay 2 && node dist/app/index.js", "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && npm-run-all --parallel start:nodemon db:server", "start:nodemon": "wait-port localhost:5432 && delay 2 && nodemon dist/app/index.js", "test": "npm run build && npm run test:alone", diff --git a/api/src/app/index.ts b/api/src/app/index.ts index b6b0a15b..873677fd 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -25,6 +25,7 @@ import { PostgresService } from "src/postgres/service"; import { ProjectController } from "src/project/controller"; import { RobotsController } from "./middlewares/robots"; import { SearchController } from "src/search/controller"; +import { SearchService } from "src/search/service"; import { SecurityMiddleware } from "./middlewares/security"; import { fsConfig } from "@dzcode.io/utils/dist/config"; @@ -38,6 +39,9 @@ useContainer(Container); // eslint-disable-line react-hooks/rules-of-hooks const { NODE_ENV, PORT } = Container.get(ConfigService).env(); + // Initialize Search Service + Container.get(SearchService); + // Add crons to DI container const CronServices = [DigestCron]; CronServices.forEach((service) => Container.get(service)); diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 332fd4d2..801ab3c4 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -11,6 +11,7 @@ export class SearchService { private readonly configService: ConfigService, private readonly logger: LoggerService, ) { + this.logger.info({ message: "Initializing MeiliSearch client" }); const { MEILISEARCH_URL, MEILISEARCH_MASTER_KEY } = this.configService.env(); @@ -18,6 +19,43 @@ export class SearchService { host: MEILISEARCH_URL, apiKey: MEILISEARCH_MASTER_KEY, }); + + this.logger.info({ + message: `MeiliSearch client initialized with url ${MEILISEARCH_URL}`, + }); + + this.meilisearch + .createIndex("project") + .then(() => { + this.logger.info({ message: "project index created" }); + }) + .catch((error) => { + this.logger.error({ + message: `failed to create project index: ${error.message}`, + }); + }); + + this.meilisearch + .createIndex("contribution") + .then(() => { + this.logger.info({ message: "contribution index created" }); + }) + .catch((error) => { + this.logger.error({ + message: `failed to create contribution index: ${error.message}`, + }); + }); + + this.meilisearch + .createIndex("contributor") + .then(() => { + this.logger.info({ message: "contributor index created" }); + }) + .catch((error) => { + this.logger.error({ + message: `failed to create contributor index: ${error.message}`, + }); + }); } public search = async (query: string): Promise => { From 470d79b4eca255dcfb5888e5fd79feb10eae3c65 Mon Sep 17 00:00:00 2001 From: omdxp Date: Fri, 20 Dec 2024 13:46:27 +0100 Subject: [PATCH 2/8] Add indexing functionality to SearchService --- api/src/search/service.ts | 23 ++++++++++++++++++++++- api/src/search/types.ts | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 801ab3c4..dc709ad1 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -1,7 +1,8 @@ +import { SearchItem, SearchType } from "./types"; + import { ConfigService } from "src/config/service"; import { LoggerService } from "src/logger/service"; import { MeiliSearch } from "meilisearch"; -import { SearchItem } from "./types"; import { Service } from "typedi"; @Service() @@ -62,4 +63,24 @@ export class SearchService { this.logger.info({ message: `Searching for ${query}` }); return []; }; + + public index = async ( + index: SearchType, + data: SearchItem[], + ): Promise => { + this.logger.info({ message: `Indexing ${data.length} items in ${index}` }); + this.meilisearch + .index(index) + .addDocuments(data) + .then(() => { + this.logger.info({ + message: `Indexed ${data.length} items in ${index}`, + }); + }) + .catch((error) => { + this.logger.error({ + message: `failed to index ${data.length} items in ${index}: ${error.message}`, + }); + }); + }; } diff --git a/api/src/search/types.ts b/api/src/search/types.ts index 0643ebab..4c34e538 100644 --- a/api/src/search/types.ts +++ b/api/src/search/types.ts @@ -10,4 +10,4 @@ export interface SearchItem { type: SearchType; } -type SearchType = "project" | "contribution" | "contributor"; +export type SearchType = "project" | "contribution" | "contributor"; From c471dbdd804d7587fb4a379c48bfdb7ba26d4a00 Mon Sep 17 00:00:00 2001 From: omdxp Date: Fri, 20 Dec 2024 13:54:24 +0100 Subject: [PATCH 3/8] Update README to include search server URL --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1ca2c49b..d49b845c 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ npm run dev:all - For api server go to - For web server go to +- For search server go to **Note** From 3ab5a0748d19beeeef012626e6c55034cd4f768b Mon Sep 17 00:00:00 2001 From: omdxp Date: Fri, 20 Dec 2024 14:27:25 +0100 Subject: [PATCH 4/8] Refactor index creation in SearchService to use ensureIndexes method --- api/src/search/service.ts | 51 ++++++++++++++------------------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/api/src/search/service.ts b/api/src/search/service.ts index dc709ad1..1fa43fa0 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -20,43 +20,11 @@ export class SearchService { host: MEILISEARCH_URL, apiKey: MEILISEARCH_MASTER_KEY, }); - this.logger.info({ message: `MeiliSearch client initialized with url ${MEILISEARCH_URL}`, }); - this.meilisearch - .createIndex("project") - .then(() => { - this.logger.info({ message: "project index created" }); - }) - .catch((error) => { - this.logger.error({ - message: `failed to create project index: ${error.message}`, - }); - }); - - this.meilisearch - .createIndex("contribution") - .then(() => { - this.logger.info({ message: "contribution index created" }); - }) - .catch((error) => { - this.logger.error({ - message: `failed to create contribution index: ${error.message}`, - }); - }); - - this.meilisearch - .createIndex("contributor") - .then(() => { - this.logger.info({ message: "contributor index created" }); - }) - .catch((error) => { - this.logger.error({ - message: `failed to create contributor index: ${error.message}`, - }); - }); + this.ensureIndexes(); } public search = async (query: string): Promise => { @@ -83,4 +51,21 @@ export class SearchService { }); }); }; + + private ensureIndexes = async (): Promise => { + try { + await this.meilisearch.createIndex("project"); + this.logger.info({ message: "project index created" }); + + await this.meilisearch.createIndex("contribution"); + this.logger.info({ message: "contribution index created" }); + + await this.meilisearch.createIndex("contributor"); + this.logger.info({ message: "contributor index created" }); + } catch { + this.logger.error({ + message: `failed to create indexes`, + }); + } + }; } From 5099e73958f7541436ee4188956fd58fcda8298a Mon Sep 17 00:00:00 2001 From: omdxp Date: Fri, 20 Dec 2024 14:31:52 +0100 Subject: [PATCH 5/8] Refactor indexing logic in SearchService to use async/await for better error handling --- api/src/search/service.ts | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 1fa43fa0..01f91b34 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -36,20 +36,19 @@ export class SearchService { index: SearchType, data: SearchItem[], ): Promise => { - this.logger.info({ message: `Indexing ${data.length} items in ${index}` }); - this.meilisearch - .index(index) - .addDocuments(data) - .then(() => { - this.logger.info({ - message: `Indexed ${data.length} items in ${index}`, - }); - }) - .catch((error) => { - this.logger.error({ - message: `failed to index ${data.length} items in ${index}: ${error.message}`, - }); + try { + this.logger.info({ + message: `Indexing ${data.length} items in ${index}`, + }); + await this.meilisearch.index(index).addDocuments(data); + this.logger.info({ + message: `Indexed ${data.length} items in ${index}`, + }); + } catch { + this.logger.error({ + message: `failed to index ${data.length} items in ${index}`, }); + } }; private ensureIndexes = async (): Promise => { From 9392542cf21429bc6fba7023c80637ae0824a576 Mon Sep 17 00:00:00 2001 From: omdxp Date: Fri, 20 Dec 2024 15:49:12 +0100 Subject: [PATCH 6/8] Refactor SearchService to ensure indexes during initialization and improve error handling in indexing --- api/src/app/index.ts | 3 ++- api/src/search/service.ts | 42 +++++++++++++-------------------------- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/api/src/app/index.ts b/api/src/app/index.ts index 873677fd..5af6505e 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -40,7 +40,8 @@ useContainer(Container); // eslint-disable-line react-hooks/rules-of-hooks const { NODE_ENV, PORT } = Container.get(ConfigService).env(); // Initialize Search Service - Container.get(SearchService); + const searchService = Container.get(SearchService); + await searchService.ensureIndexes(); // Add crons to DI container const CronServices = [DigestCron]; diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 01f91b34..4ac9436e 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -23,8 +23,6 @@ export class SearchService { this.logger.info({ message: `MeiliSearch client initialized with url ${MEILISEARCH_URL}`, }); - - this.ensureIndexes(); } public search = async (query: string): Promise => { @@ -36,35 +34,23 @@ export class SearchService { index: SearchType, data: SearchItem[], ): Promise => { - try { - this.logger.info({ - message: `Indexing ${data.length} items in ${index}`, - }); - await this.meilisearch.index(index).addDocuments(data); - this.logger.info({ - message: `Indexed ${data.length} items in ${index}`, - }); - } catch { - this.logger.error({ - message: `failed to index ${data.length} items in ${index}`, - }); - } + this.logger.info({ + message: `Indexing ${data.length} items in ${index}`, + }); + await this.meilisearch.index(index).addDocuments(data); + this.logger.info({ + message: `Indexed ${data.length} items in ${index}`, + }); }; - private ensureIndexes = async (): Promise => { - try { - await this.meilisearch.createIndex("project"); - this.logger.info({ message: "project index created" }); + public ensureIndexes = async (): Promise => { + await this.meilisearch.createIndex("project"); + this.logger.info({ message: "project index created" }); - await this.meilisearch.createIndex("contribution"); - this.logger.info({ message: "contribution index created" }); + await this.meilisearch.createIndex("contribution"); + this.logger.info({ message: "contribution index created" }); - await this.meilisearch.createIndex("contributor"); - this.logger.info({ message: "contributor index created" }); - } catch { - this.logger.error({ - message: `failed to create indexes`, - }); - } + await this.meilisearch.createIndex("contributor"); + this.logger.info({ message: "contributor index created" }); }; } From 843700d1bb3252bea9248383ce727055b17ec48b Mon Sep 17 00:00:00 2001 From: omdxp Date: Fri, 20 Dec 2024 16:46:19 +0100 Subject: [PATCH 7/8] Add MeiliSearch service to Cloud Docker Compose and update .gitignore --- .gitignore | 1 + api/oracle-cloud/docker-compose.yml | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 3d4574c5..39446496 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ coverage api/oracle-cloud/build api/fetch_cache api/postgres_db +api/meilisearch_db api/nodemon.json # web diff --git a/api/oracle-cloud/docker-compose.yml b/api/oracle-cloud/docker-compose.yml index 9dbc5577..5a81e761 100644 --- a/api/oracle-cloud/docker-compose.yml +++ b/api/oracle-cloud/docker-compose.yml @@ -27,3 +27,12 @@ services: environment: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: db + meilisearch: + image: getmeili/meilisearch:latest + ports: + - "7700:7700" + volumes: + - /home/ubuntu/app-data/api/meilisearch_db:/data.ms + environment: + MEILI_NO_ANALYTICS: true + MEILI_MASTER_KEY: "default" From 512aa88776b6e8c739e664ab905bb80e9b3be06a Mon Sep 17 00:00:00 2001 From: omdxp Date: Fri, 20 Dec 2024 16:47:42 +0100 Subject: [PATCH 8/8] Update MeiliSearch volume paths in Docker Compose files --- api/docker-compose.yml | 2 +- api/oracle-cloud/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/docker-compose.yml b/api/docker-compose.yml index 5608a938..f14e426a 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "7700:7700" volumes: - - ./meilisearch_db:/data.ms + - ./meilisearch_db:/meili_data environment: MEILI_NO_ANALYTICS: true MEILI_MASTER_KEY: "default" diff --git a/api/oracle-cloud/docker-compose.yml b/api/oracle-cloud/docker-compose.yml index 5a81e761..a285e7d0 100644 --- a/api/oracle-cloud/docker-compose.yml +++ b/api/oracle-cloud/docker-compose.yml @@ -32,7 +32,7 @@ services: ports: - "7700:7700" volumes: - - /home/ubuntu/app-data/api/meilisearch_db:/data.ms + - /home/ubuntu/app-data/api/meilisearch_db:/meili_data environment: MEILI_NO_ANALYTICS: true MEILI_MASTER_KEY: "default"