add langchain test

Author: hassiebp

tests/test_langchain.py

@@ -36,7 +36,7 @@
 from langfuse.callback import CallbackHandler
 from langfuse.client import Langfuse
 from tests.api_wrapper import LangfuseAPI
-from tests.utils import create_uuid, get_api
+from tests.utils import create_uuid, encode_file_to_base64, get_api
 
 
 def test_callback_init():
@@ -2210,3 +2210,39 @@ def _generate_random_dict(n: int, key_length: int = 8) -> Dict[str, Any]:
     print(f"Full execution took {duration_full}ms")
 
     assert duration_full > 1000, "Full execution should take longer than 1 second"
+
+
+def test_multimodal():
+    api = get_api()
+    handler = CallbackHandler()
+    model = ChatOpenAI(model="gpt-4o-mini")
+
+    image_data = encode_file_to_base64("static/puton.jpg")
+
+    message = HumanMessage(
+        content=[
+            {"type": "text", "text": "What's in this image?"},
+            {
+                "type": "image_url",
+                "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
+            },
+        ],
+    )
+
+    response = model.invoke([message], config={"callbacks": [handler]})
+
+    print(response.content)
+
+    handler.flush()
+
+    trace = api.trace.get(handler.get_trace_id())
+
+    assert len(trace.observations) == 1
+    assert trace.observations[0].type == "GENERATION"
+
+    print(trace.observations[0].input)
+
+    assert (
+        "@@@langfuseMedia:type=image/jpeg|id="
+        in trace.observations[0].input[0]["content"][1]["image_url"]["url"]
+    )

tests/test_openai.py

@@ -1,4 +1,3 @@
-import base64
 import os
 
 import pytest
@@ -14,7 +13,7 @@
     _is_openai_v1,
     openai,
 )
-from tests.utils import create_uuid, get_api
+from tests.utils import create_uuid, encode_file_to_base64, get_api
 
 chat_func = (
     openai.chat.completions.create if _is_openai_v1() else openai.ChatCompletion.create
@@ -1482,7 +1481,7 @@ def test_base_64_image_input():
     content_path = "static/puton.jpg"
     content_type = "image/jpeg"
 
-    base64_image = encode_file(content_path)
+    base64_image = encode_file_to_base64(content_path)
 
     client.chat.completions.create(
         name=generation_name,
@@ -1532,7 +1531,7 @@ def test_audio_input_and_output():
     generation_name = "test_audio_input_and_output" + create_uuid()[:8]
 
     content_path = "static/joke_prompt.wav"
-    base64_string = encode_file(content_path)
+    base64_string = encode_file_to_base64(content_path)
 
     client.chat.completions.create(
         name=generation_name,
@@ -1580,8 +1579,3 @@ def test_audio_input_and_output():
         "@@@langfuseMedia:type=audio/wav|id="
         in generation.data[0].output["content"][1]["output_audio"]["data"]
     )
-
-
-def encode_file(image_path):
-    with open(image_path, "rb") as file:
-        return base64.b64encode(file.read()).decode("utf-8")

tests/utils.py

@@ -1,3 +1,4 @@
+import base64
 import os
 import typing
 from uuid import uuid4
@@ -7,18 +8,17 @@
 except ImportError:
     import pydantic  # type: ignore
 
-from langfuse.api.client import FernLangfuse
-
 from llama_index.core import (
     Settings,
-    VectorStoreIndex,
     SimpleDirectoryReader,
-    load_index_from_storage,
     StorageContext,
+    VectorStoreIndex,
+    load_index_from_storage,
 )
-
 from llama_index.core.callbacks import CallbackManager
 
+from langfuse.api.client import FernLangfuse
+
 
 def create_uuid():
     return str(uuid4())
@@ -106,3 +106,8 @@ def get_llama_index_index(callback, force_rebuild: bool = False):
         index = load_index_from_storage(storage_context)
 
     return index
+
+
+def encode_file_to_base64(image_path) -> str:
+    with open(image_path, "rb") as file:
+        return base64.b64encode(file.read()).decode("utf-8")