perf(ingestion): make max event and batch size configurable (#1103)

Author: hassiebp

langfuse/_task_manager/ingestion_consumer.py

@@ -1,8 +1,8 @@
 import json
 import logging
+import os
 import threading
 import time
-
 from queue import Empty, Queue
 from typing import Any, List, Optional
 
@@ -21,8 +21,8 @@
 
 from .media_manager import MediaManager
 
-MAX_EVENT_SIZE_BYTES = 1_000_000
-MAX_BATCH_SIZE_BYTES = 2_500_000
+MAX_EVENT_SIZE_BYTES = int(os.environ.get("LANGFUSE_MAX_EVENT_SIZE_BYTES", 1_000_000))
+MAX_BATCH_SIZE_BYTES = int(os.environ.get("LANGFUSE_MAX_BATCH_SIZE_BYTES", 2_500_000))
 
 
 class IngestionMetadata(pydantic.BaseModel):

tests/test_langchain.py

@@ -2184,7 +2184,9 @@ def _generate_random_dict(n: int, key_length: int = 8) -> Dict[str, Any]:
     overhead = duration_with_langfuse - duration_without_langfuse
     print(f"Langfuse overhead: {overhead}ms")
 
-    assert overhead < 50, f"Langfuse tracing overhead of {overhead}ms exceeds threshold"
+    assert (
+        overhead < 100
+    ), f"Langfuse tracing overhead of {overhead}ms exceeds threshold"
 
     handler.flush()