perf: hf caching + openai alignment

Author: andhreljaKern

controller.py

@@ -3,6 +3,7 @@
 from fastapi import status
 from spacy.tokens import DocBin, Doc
 from spacy.vocab import Vocab
+from functools import lru_cache
 
 import json
 import torch
@@ -636,6 +637,11 @@ def __setup_tmp_embedder(project_id: str, embedder_id: str) -> Transformer:
     embedder_path = util.INFERENCE_DIR / project_id / embedder_id / "embedder.json"
     if not embedder_path.exists():
         raise Exception(f"Embedder {embedder_id} not found")
+    return __load_embedder_by_path(embedder_path)
+
+
+@lru_cache(maxsize=32)
+def __load_embedder_by_path(embedder_path: str) -> Transformer:
     with open(embedder_path, "r") as f:
         embedder = json.load(f)
         Embedder = eval(embedder["cls"])

src/embedders/classification/contextual.py

@@ -44,7 +44,7 @@ def load(embedder: dict) -> "HuggingFaceSentenceEmbedder":
     def to_json(self) -> dict:
         return {
             "cls": "HuggingFaceSentenceEmbedder",
-            "model_name": self.model.model_card_data.base_model,
+            "config_string": self.model.model_card_data.base_model,
             "batch_size": self.batch_size,
         }
 
@@ -121,19 +121,13 @@ def __init__(
                 and api_version is not None
                 and api_base is not None
             ), "If you want to use Azure, you need to provide api_type, api_version and api_base."
-
-    @property
-    def openai_client(self):
-        if self.use_azure:
-            return AzureOpenAI(
+            self.openai_client = AzureOpenAI(
                 api_key=self.openai_api_key,
                 azure_endpoint=self.api_base,
                 api_version=self.api_version,
             )
-        return OpenAI(
-            api_key=self.openai_api_key,
-            base_url=self.api_base,
-        )
+        else:
+            self.openai_client = OpenAI(api_key=self.openai_api_key)
 
     def _encode(
         self, documents: List[Union[str, Doc]], fit_model: bool

src/embedders/util.py

@@ -5,9 +5,7 @@
 import json
 import pickle
 
-INFERENCE_DIR = Path(
-    os.getenv("INFERENCE_DIR", "/Users/andhrelja/Projects/dev-setup/inference")
-)
+INFERENCE_DIR = Path(os.getenv("INFERENCE_DIR", "/inference"))
 
 
 def batch(documents: List[Any], batch_size: int) -> Generator[List[Any], None, None]: