From fa8e1ef0c212ed7746c9b6951ab908b562adfb5a Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 11:35:22 +0200 Subject: [PATCH 01/40] feat: add fetch_scores method to Langfuse client --- langfuse/client.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/langfuse/client.py b/langfuse/client.py index 998caf327..423aa5f91 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -125,6 +125,13 @@ class FetchSessionsResponse: data: typing.List[Session] meta: MetaResponse +@dataclass +class FetchScoresResponse: + """Response object for fetch_scores method.""" + + data: typing.List[ScoreBody] + meta: MetaResponse + class Langfuse(object): """Langfuse Python client. @@ -915,6 +922,43 @@ def fetch_sessions( self.log.exception(e) raise e + def fetch_sessions( + self, + *, + page: typing.Optional[int] = None, + limit: typing.Optional[int] = None, + from_timestamp: typing.Optional[dt.datetime] = None, + to_timestamp: typing.Optional[dt.datetime] = None, + ) -> FetchSessionsResponse: + """Get a list of sessions in the current project. + + Args: + page (Optional[int]): Page number of the sessions to return. Defaults to None. + limit (Optional[int]): Maximum number of sessions to return. Defaults to None. + from_timestamp (Optional[dt.datetime]): Retrieve only sessions with a timestamp on or after this datetime. Defaults to None. + to_timestamp (Optional[dt.datetime]): Retrieve only sessions with a timestamp before this datetime. Defaults to None. + + Returns: + FetchSessionsResponse, list of sessions on `data` and metadata on `meta`. + + Raises: + Exception: If an error occurred during the request. + """ + try: + self.log.debug( + f"Getting sessions... {page}, {limit}, {from_timestamp}, {to_timestamp}" + ) + res = self.client.sessions.list( + page=page, + limit=limit, + from_timestamp=from_timestamp, + to_timestamp=to_timestamp, + ) + return FetchSessionsResponse(data=res.data, meta=res.meta) + except Exception as e: + self.log.exception(e) + raise e + @overload def get_prompt( self, From a298b09f70e0cffd139d21a2f2bf5a1af9c20cda Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 11:35:27 +0200 Subject: [PATCH 02/40] feat: add fetch_scores method to Langfuse client --- tests/test_core_sdk.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index c4f809ed4..f547423aa 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -6,6 +6,7 @@ from langfuse.client import ( FetchObservationResponse, FetchObservationsResponse, + FetchScoresResponse, FetchSessionsResponse, FetchTraceResponse, FetchTracesResponse, @@ -1465,3 +1466,39 @@ def test_fetch_sessions(): response = langfuse.fetch_sessions(limit=1, page=2) assert len(response.data) == 1 assert response.data[0].id in [session1, session2, session3] + +def test_fetch_scores(): + langfuse = Langfuse() + + # Create a trace with a score + name = create_uuid() + trace = langfuse + trace = langfuse.trace(name=name) + trace.score(name="harmfulness", value=0.5) + trace.score(name="quality", value=1) + trace.score(name="relevance", value=0.8) + langfuse.flush() + + # Fetch scores + response = langfuse.fetch_scores() + + # Assert the structure of the response + assert isinstance(response, FetchScoresResponse) + assert hasattr(response, "data") + assert hasattr(response, "meta") + assert isinstance(response.data, list) + assert len(response.data) == 3 + assert response.meta.total_items == 3 + assert response.data[0].name == "harmfulness" + assert response.data[0].value == 0.5 + assert response.data[1].name == "quality" + assert response.data[1].value == 1 + assert response.data[2].name == "relevance" + assert response.data[2].value == 0.8 + + # fetch only one + response = langfuse.fetch_scores(limit=1, page=2) + assert len(response.data) == 1 + + + \ No newline at end of file From 5f1e761847ed4efaccc69d312bb0e0ea0248657a Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 11:41:13 +0200 Subject: [PATCH 03/40] feat: update fetch_scores method in Langfuse client --- langfuse/client.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/langfuse/client.py b/langfuse/client.py index 423aa5f91..c1bfc113a 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -922,39 +922,39 @@ def fetch_sessions( self.log.exception(e) raise e - def fetch_sessions( + def fetch_scores( self, *, page: typing.Optional[int] = None, limit: typing.Optional[int] = None, from_timestamp: typing.Optional[dt.datetime] = None, to_timestamp: typing.Optional[dt.datetime] = None, - ) -> FetchSessionsResponse: - """Get a list of sessions in the current project. + ) -> FetchScoresResponse: + """Get a list of scores in the current project. Args: - page (Optional[int]): Page number of the sessions to return. Defaults to None. - limit (Optional[int]): Maximum number of sessions to return. Defaults to None. - from_timestamp (Optional[dt.datetime]): Retrieve only sessions with a timestamp on or after this datetime. Defaults to None. - to_timestamp (Optional[dt.datetime]): Retrieve only sessions with a timestamp before this datetime. Defaults to None. + page (Optional[int]): Page number of the scores to return. Defaults to None. + limit (Optional[int]): Maximum number of scores to return. Defaults to None. + from_timestamp (Optional[dt.datetime]): Retrieve only scores with a timestamp on or after this datetime. Defaults to None. + to_timestamp (Optional[dt.datetime]): Retrieve only scores with a timestamp before this datetime. Defaults to None. Returns: - FetchSessionsResponse, list of sessions on `data` and metadata on `meta`. + FetchScoresResponse, list of scores on `data` and metadata on `meta`. Raises: Exception: If an error occurred during the request. """ try: self.log.debug( - f"Getting sessions... {page}, {limit}, {from_timestamp}, {to_timestamp}" + f"Getting scores... {page}, {limit}, {from_timestamp}, {to_timestamp}" ) - res = self.client.sessions.list( + res = self.client.scores.list( page=page, limit=limit, from_timestamp=from_timestamp, to_timestamp=to_timestamp, ) - return FetchSessionsResponse(data=res.data, meta=res.meta) + return FetchScoresResponse(data=res.data, meta=res.meta) except Exception as e: self.log.exception(e) raise e From e092a8347d88f50564cc1e8373fb85ea87ee835b Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 11:58:11 +0200 Subject: [PATCH 04/40] feat: update fetch_scores method in Langfuse client --- langfuse/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langfuse/client.py b/langfuse/client.py index c1bfc113a..6e95690bb 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -948,7 +948,7 @@ def fetch_scores( self.log.debug( f"Getting scores... {page}, {limit}, {from_timestamp}, {to_timestamp}" ) - res = self.client.scores.list( + res = self.client.score.list( page=page, limit=limit, from_timestamp=from_timestamp, From 4983e4346bad04d420773050e5e57940bb1cd99e Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 12:16:51 +0200 Subject: [PATCH 05/40] feat: add list method to ScoreClient --- langfuse/api/resources/score/client.py | 157 +++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/langfuse/api/resources/score/client.py b/langfuse/api/resources/score/client.py index 29674f622..de62d1d0c 100644 --- a/langfuse/api/resources/score/client.py +++ b/langfuse/api/resources/score/client.py @@ -451,6 +451,163 @@ def delete( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + def list( + self, + *, + page: typing.Optional[int] = None, + limit: typing.Optional[int] = None, + user_id: typing.Optional[str] = None, + name: typing.Optional[str] = None, + from_timestamp: typing.Optional[dt.datetime] = None, + to_timestamp: typing.Optional[dt.datetime] = None, + source: typing.Optional[ScoreSource] = None, + operator: typing.Optional[str] = None, + value: typing.Optional[float] = None, + score_ids: typing.Optional[str] = None, + config_id: typing.Optional[str] = None, + data_type: typing.Optional[ScoreDataType] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> Scores: + """ + Get a list of scores + + Parameters: + - page: typing.Optional[int]. Page number, starts at 1. + + - limit: typing.Optional[int]. Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit. + + - user_id: typing.Optional[str]. Retrieve only scores with this userId associated to the trace. + + - name: typing.Optional[str]. Retrieve only scores with this name. + + - from_timestamp: typing.Optional[dt.datetime]. Optional filter to only include scores created on or after a certain datetime (ISO 8601) + + - to_timestamp: typing.Optional[dt.datetime]. Optional filter to only include scores created before a certain datetime (ISO 8601) + + - source: typing.Optional[ScoreSource]. Retrieve only scores from a specific source. + + - operator: typing.Optional[str]. Retrieve only scores with value. + + - value: typing.Optional[float]. Retrieve only scores with value. + + - score_ids: typing.Optional[str]. Comma-separated list of score IDs to limit the results to. + + - config_id: typing.Optional[str]. Retrieve only scores with a specific configId. + + - data_type: typing.Optional[ScoreDataType]. Retrieve only scores with a specific dataType. + + - request_options: typing.Optional[RequestOptions]. Request-specific configuration. + --- + import datetime + + from finto import ScoreDataType, ScoreSource + from finto.client import FernLangfuse + + client = FernLangfuse( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.score.list( + page=1, + limit=1, + user_id="string", + name="string", + from_timestamp=datetime.datetime.fromisoformat( + "2024-01-15 09:30:00+00:00", + ), + to_timestamp=datetime.datetime.fromisoformat( + "2024-01-15 09:30:00+00:00", + ), + source=ScoreSource.ANNOTATION, + operator="string", + value=1.1, + score_ids="string", + config_id="string", + data_type=ScoreDataType.NUMERIC, + ) + """ + _response = self._client_wrapper.httpx_client.request( + "GET", + urllib.parse.urljoin( + f"{self._client_wrapper.get_base_url()}/", "api/public/scores" + ), + params=jsonable_encoder( + remove_none_from_dict( + { + "page": page, + "limit": limit, + "userId": user_id, + "name": name, + "fromTimestamp": serialize_datetime(from_timestamp) + if from_timestamp is not None + else None, + "toTimestamp": serialize_datetime(to_timestamp) + if to_timestamp is not None + else None, + "source": source, + "operator": operator, + "value": value, + "scoreIds": score_ids, + "configId": config_id, + "dataType": data_type, + **( + request_options.get("additional_query_parameters", {}) + if request_options is not None + else {} + ), + } + ) + ), + headers=jsonable_encoder( + remove_none_from_dict( + { + **self._client_wrapper.get_headers(), + **( + request_options.get("additional_headers", {}) + if request_options is not None + else {} + ), + } + ) + ), + timeout=request_options.get("timeout_in_seconds") + if request_options is not None + and request_options.get("timeout_in_seconds") is not None + else self._client_wrapper.get_timeout(), + retries=0, + max_retries=request_options.get("max_retries") + if request_options is not None + else 0, # type: ignore + ) + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as(Scores, _response.json()) # type: ignore + if _response.status_code == 400: + raise Error(pydantic_v1.parse_obj_as(typing.Any, _response.json())) # type: ignore + if _response.status_code == 401: + raise UnauthorizedError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 403: + raise AccessDeniedError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 405: + raise MethodNotAllowedError( + pydantic_v1.parse_obj_as(typing.Any, _response.json()) + ) # type: ignore + if _response.status_code == 404: + raise NotFoundError(pydantic_v1.parse_obj_as(typing.Any, _response.json())) # type: ignore + try: + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + class AsyncScoreClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): From 39f96a0a2f30f7698c0a65cf5f127c234c9d98b9 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 12:38:59 +0200 Subject: [PATCH 06/40] fix: fix assertion order in test_fetch_scores --- tests/test_core_sdk.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index f547423aa..f589eeeec 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1487,14 +1487,12 @@ def test_fetch_scores(): assert hasattr(response, "data") assert hasattr(response, "meta") assert isinstance(response.data, list) - assert len(response.data) == 3 - assert response.meta.total_items == 3 - assert response.data[0].name == "harmfulness" - assert response.data[0].value == 0.5 + assert response.data[2].name == "harmfulness" + assert response.data[2].value == 0.5 assert response.data[1].name == "quality" assert response.data[1].value == 1 - assert response.data[2].name == "relevance" - assert response.data[2].value == 0.8 + assert response.data[0].name == "relevance" + assert response.data[0].value == 0.8 # fetch only one response = langfuse.fetch_scores(limit=1, page=2) From 7ec13682def31a62fabc16d2cc54eb579bbba78c Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 13:36:06 +0200 Subject: [PATCH 07/40] feat: add fetch_sessions method to Langfuse client --- langfuse/client.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/langfuse/client.py b/langfuse/client.py index 6e95690bb..d7ba5ad6e 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -958,6 +958,43 @@ def fetch_scores( except Exception as e: self.log.exception(e) raise e + + def fetch_sessions( + self, + *, + page: typing.Optional[int] = None, + limit: typing.Optional[int] = None, + from_timestamp: typing.Optional[dt.datetime] = None, + to_timestamp: typing.Optional[dt.datetime] = None, + ) -> FetchSessionsResponse: + """Get a list of sessions in the current project. + + Args: + page (Optional[int]): Page number of the sessions to return. Defaults to None. + limit (Optional[int]): Maximum number of sessions to return. Defaults to None. + from_timestamp (Optional[dt.datetime]): Retrieve only sessions with a timestamp on or after this datetime. Defaults to None. + to_timestamp (Optional[dt.datetime]): Retrieve only sessions with a timestamp before this datetime. Defaults to None. + + Returns: + FetchSessionsResponse, list of sessions on `data` and metadata on `meta`. + + Raises: + Exception: If an error occurred during the request. + """ + try: + self.log.debug( + f"Getting sessions... {page}, {limit}, {from_timestamp}, {to_timestamp}" + ) + res = self.client.sessions.list( + page=page, + limit=limit, + from_timestamp=from_timestamp, + to_timestamp=to_timestamp, + ) + return FetchSessionsResponse(data=res.data, meta=res.meta) + except Exception as e: + self.log.exception(e) + raise e @overload def get_prompt( From c99055d81de3a23b889d5b1f01113d5c7ced649f Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 13:36:10 +0200 Subject: [PATCH 08/40] feat: add test_fetch_sessions to Langfuse client --- tests/test_core_sdk.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index f589eeeec..9eb35c0c5 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1498,5 +1498,33 @@ def test_fetch_scores(): response = langfuse.fetch_scores(limit=1, page=2) assert len(response.data) == 1 +def test_fetch_sessions(): + langfuse = Langfuse() + + # unique name + name = create_uuid() + session1 = create_uuid() + session2 = create_uuid() + session3 = create_uuid() + + # Create multiple traces + langfuse.trace(name=name, session_id=session1) + langfuse.trace(name=name, session_id=session2) + langfuse.trace(name=name, session_id=session3) + langfuse.flush() + # Fetch traces + response = langfuse.fetch_sessions() + + # Assert the structure of the response, cannot check for the exact number of sessions as the table is not cleared between tests + assert isinstance(response, FetchSessionsResponse) + assert hasattr(response, "data") + assert hasattr(response, "meta") + assert isinstance(response.data, list) + assert response.data[0].id in [session1, session2, session3] + + # fetch only one, cannot check for the exact number of sessions as the table is not cleared between tests + response = langfuse.fetch_sessions(limit=1, page=2) + assert len(response.data) == 1 + assert response.data[0].id in [session1, session2, session3] \ No newline at end of file From 62baa1a8d55e62cbdd801e3e431ba8027bc12053 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 13:39:10 +0200 Subject: [PATCH 09/40] feat: refactor fetch_sessions method in Langfuse client --- langfuse/client.py | 37 ------------------------------------- tests/test_core_sdk.py | 31 ------------------------------- 2 files changed, 68 deletions(-) diff --git a/langfuse/client.py b/langfuse/client.py index d7ba5ad6e..6e95690bb 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -958,43 +958,6 @@ def fetch_scores( except Exception as e: self.log.exception(e) raise e - - def fetch_sessions( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - from_timestamp: typing.Optional[dt.datetime] = None, - to_timestamp: typing.Optional[dt.datetime] = None, - ) -> FetchSessionsResponse: - """Get a list of sessions in the current project. - - Args: - page (Optional[int]): Page number of the sessions to return. Defaults to None. - limit (Optional[int]): Maximum number of sessions to return. Defaults to None. - from_timestamp (Optional[dt.datetime]): Retrieve only sessions with a timestamp on or after this datetime. Defaults to None. - to_timestamp (Optional[dt.datetime]): Retrieve only sessions with a timestamp before this datetime. Defaults to None. - - Returns: - FetchSessionsResponse, list of sessions on `data` and metadata on `meta`. - - Raises: - Exception: If an error occurred during the request. - """ - try: - self.log.debug( - f"Getting sessions... {page}, {limit}, {from_timestamp}, {to_timestamp}" - ) - res = self.client.sessions.list( - page=page, - limit=limit, - from_timestamp=from_timestamp, - to_timestamp=to_timestamp, - ) - return FetchSessionsResponse(data=res.data, meta=res.meta) - except Exception as e: - self.log.exception(e) - raise e @overload def get_prompt( diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 9eb35c0c5..97ca23094 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1497,34 +1497,3 @@ def test_fetch_scores(): # fetch only one response = langfuse.fetch_scores(limit=1, page=2) assert len(response.data) == 1 - -def test_fetch_sessions(): - langfuse = Langfuse() - - # unique name - name = create_uuid() - session1 = create_uuid() - session2 = create_uuid() - session3 = create_uuid() - - # Create multiple traces - langfuse.trace(name=name, session_id=session1) - langfuse.trace(name=name, session_id=session2) - langfuse.trace(name=name, session_id=session3) - langfuse.flush() - - # Fetch traces - response = langfuse.fetch_sessions() - - # Assert the structure of the response, cannot check for the exact number of sessions as the table is not cleared between tests - assert isinstance(response, FetchSessionsResponse) - assert hasattr(response, "data") - assert hasattr(response, "meta") - assert isinstance(response.data, list) - assert response.data[0].id in [session1, session2, session3] - - # fetch only one, cannot check for the exact number of sessions as the table is not cleared between tests - response = langfuse.fetch_sessions(limit=1, page=2) - assert len(response.data) == 1 - assert response.data[0].id in [session1, session2, session3] - \ No newline at end of file From c5c2d5d25759285bf0b3de124d5ecf5e38e8008b Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 13:53:00 +0200 Subject: [PATCH 10/40] feat: add FetchPromptsResponse class and fetch_prompts method to Langfuse client --- langfuse/client.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/langfuse/client.py b/langfuse/client.py index 6e95690bb..d92c18c10 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -21,6 +21,7 @@ import urllib.parse import warnings from dataclasses import dataclass +from langfuse.api.core.request_options import RequestOptions from langfuse.api.resources.commons.types.dataset_run_with_items import ( @@ -132,6 +133,13 @@ class FetchScoresResponse: data: typing.List[ScoreBody] meta: MetaResponse +@dataclass +class FetchPromptsResponse: + """Response object for fetch_prompts method.""" + + data: typing.List[Union[Prompt_Text, Prompt_Chat]] + meta: MetaResponse + class Langfuse(object): """Langfuse Python client. @@ -958,6 +966,54 @@ def fetch_scores( except Exception as e: self.log.exception(e) raise e + def fetch_prompts( + self, + *, + page: typing.Optional[int] = None, + limit: typing.Optional[int] = None, + name: typing.Optional[str] = None, + user_id: typing.Optional[str] = None, + trace_id: typing.Optional[str] = None, + from_start_time: typing.Optional[dt.datetime] = None, + to_start_time: typing.Optional[dt.datetime] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> FetchPromptsResponse: + """Get a list of prompts in the current project matching the given parameters. + + Args: + page (Optional[int]): Page number of the prompts to return. Defaults to None. + limit (Optional[int]): Maximum number of prompts to return. Defaults to None. + name (Optional[str]): Name of the prompts to return. Defaults to None. + user_id (Optional[str]): User identifier of the prompts to return. Defaults to None. + trace_id (Optional[str]): Trace identifier + from_start_time (Optional[dt.datetime]): Retrieve only prompts with a start_time on or after this datetime. Defaults to None. + to_start_time (Optional[dt.datetime]): Retrieve only prompts with a start_time before this datetime. Defaults to None. + request_options (Optional[RequestOptions]): Type of the prompt. Defaults to None. + + Returns: + FetchPromptsResponse, list of prompts on `data` and metadata on `meta`. + + Raises: + Exception: If an error occurred during the request. + """ + try: + self.log.debug( + f"Getting prompts... {page}, {limit}, {name}, {user_id}, {trace_id}, {from_start_time}, {to_start_time}, {request_options}" + ) + res = self.client.prompts.list( + page=page, + limit=limit, + name=name, + user_id=user_id, + trace_id=trace_id, + from_start_time=from_start_time, + to_start_time=to_start_time, + request_options=request_options, + ) + return FetchPromptsResponse(data=res.data, meta=res.meta) + except Exception as e: + self.log.exception(e) + raise e @overload def get_prompt( From 30073ede51cbb5e071a13d6c21d42931a42686ec Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 13:53:04 +0200 Subject: [PATCH 11/40] feat: add test_fetch_prompts to Langfuse client --- tests/test_core_sdk.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 97ca23094..4fbea6eda 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -6,6 +6,7 @@ from langfuse.client import ( FetchObservationResponse, FetchObservationsResponse, + FetchPromptsResponse, FetchScoresResponse, FetchSessionsResponse, FetchTraceResponse, @@ -1497,3 +1498,25 @@ def test_fetch_scores(): # fetch only one response = langfuse.fetch_scores(limit=1, page=2) assert len(response.data) == 1 + +def test_fetch_prompts(): + langfuse = Langfuse() + + # Create a trace with a prompt + name = create_uuid() + trace = langfuse + trace = langfuse.trace(name=name) + trace.prompt(name="prompt", input="prompt input", output="prompt output") + langfuse.flush() + + # Fetch prompts + response = langfuse.fetch_prompts() + + # Assert the structure of the response + assert isinstance(response, FetchPromptsResponse) + assert hasattr(response, "data") + assert hasattr(response, "meta") + assert isinstance(response.data, list) + assert response.data[0].name == "prompt" + assert response.data[0].input == "prompt input" + assert response.data[0].output == "prompt output" \ No newline at end of file From 1997259ac96209fccb83b8cb93ceb76defd73a2d Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:02:07 +0200 Subject: [PATCH 12/40] refactor: remove unnecessary blank lines in Langfuse client --- langfuse/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/langfuse/client.py b/langfuse/client.py index d92c18c10..cd68eb05a 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -126,6 +126,7 @@ class FetchSessionsResponse: data: typing.List[Session] meta: MetaResponse + @dataclass class FetchScoresResponse: """Response object for fetch_scores method.""" @@ -133,6 +134,7 @@ class FetchScoresResponse: data: typing.List[ScoreBody] meta: MetaResponse + @dataclass class FetchPromptsResponse: """Response object for fetch_prompts method.""" @@ -966,6 +968,7 @@ def fetch_scores( except Exception as e: self.log.exception(e) raise e + def fetch_prompts( self, *, @@ -989,7 +992,7 @@ def fetch_prompts( from_start_time (Optional[dt.datetime]): Retrieve only prompts with a start_time on or after this datetime. Defaults to None. to_start_time (Optional[dt.datetime]): Retrieve only prompts with a start_time before this datetime. Defaults to None. request_options (Optional[RequestOptions]): Type of the prompt. Defaults to None. - + Returns: FetchPromptsResponse, list of prompts on `data` and metadata on `meta`. @@ -1014,7 +1017,7 @@ def fetch_prompts( except Exception as e: self.log.exception(e) raise e - + @overload def get_prompt( self, From fe7c4c6aaf25d37472c43199abcdfc333a4d0357 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:02:11 +0200 Subject: [PATCH 13/40] refactor: remove unnecessary blank lines in Langfuse client --- langfuse/api/resources/score/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/langfuse/api/resources/score/client.py b/langfuse/api/resources/score/client.py index de62d1d0c..b88bcd8ce 100644 --- a/langfuse/api/resources/score/client.py +++ b/langfuse/api/resources/score/client.py @@ -608,7 +608,6 @@ def list( raise ApiError(status_code=_response.status_code, body=_response_json) - class AsyncScoreClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): self._client_wrapper = client_wrapper From 42694c4231fc4a80cdd5d8d98143139f35ed2683 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:02:14 +0200 Subject: [PATCH 14/40] refactor: remove unnecessary blank lines in Langfuse client --- tests/test_core_sdk.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 4fbea6eda..088bc1dc1 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1468,6 +1468,7 @@ def test_fetch_sessions(): assert len(response.data) == 1 assert response.data[0].id in [session1, session2, session3] + def test_fetch_scores(): langfuse = Langfuse() @@ -1499,14 +1500,24 @@ def test_fetch_scores(): response = langfuse.fetch_scores(limit=1, page=2) assert len(response.data) == 1 + def test_fetch_prompts(): langfuse = Langfuse() # Create a trace with a prompt name = create_uuid() - trace = langfuse - trace = langfuse.trace(name=name) - trace.prompt(name="prompt", input="prompt input", output="prompt output") + langfuse.create_prompt( + name="event-planner", + prompt="Plan an event titled {{Event Name}}. The event will be about: {{Event Description}}. " + "The event will be held in {{Location}} on {{Date}}. " + "Consider the following factors: audience, budget, venue, catering options, and entertainment. " + "Provide a detailed plan including potential vendors and logistics.", + config={ + "model": "gpt-3.5-turbo-1106", + "temperature": 0, + }, + labels=["production"], + ) langfuse.flush() # Fetch prompts @@ -1517,6 +1528,9 @@ def test_fetch_prompts(): assert hasattr(response, "data") assert hasattr(response, "meta") assert isinstance(response.data, list) - assert response.data[0].name == "prompt" - assert response.data[0].input == "prompt input" - assert response.data[0].output == "prompt output" \ No newline at end of file + assert response.data[0].name == "event-planner" + assert ( + response.data[0].prompt + == "Plan an event titled {{Event Name}}. The event will be about: {{Event Description}}. The event will be held in {{Location}} on {{Date}}. Consider the following factors: audience, budget, venue, catering options, and entertainment. Provide a detailed plan including potential vendors and logistics." + ) + assert response.data[0].labels == ["production"] From b53ac5ea4239defd6771b7f41a0100308ce0e933 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:17:43 +0200 Subject: [PATCH 15/40] refactor: update Langfuse client's get_prompts method parameters --- langfuse/client.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/langfuse/client.py b/langfuse/client.py index cd68eb05a..c478b0bc5 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -975,10 +975,8 @@ def fetch_prompts( page: typing.Optional[int] = None, limit: typing.Optional[int] = None, name: typing.Optional[str] = None, - user_id: typing.Optional[str] = None, - trace_id: typing.Optional[str] = None, - from_start_time: typing.Optional[dt.datetime] = None, - to_start_time: typing.Optional[dt.datetime] = None, + label: typing.Optional[str] = None, + tag: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None, ) -> FetchPromptsResponse: """Get a list of prompts in the current project matching the given parameters. @@ -987,10 +985,8 @@ def fetch_prompts( page (Optional[int]): Page number of the prompts to return. Defaults to None. limit (Optional[int]): Maximum number of prompts to return. Defaults to None. name (Optional[str]): Name of the prompts to return. Defaults to None. - user_id (Optional[str]): User identifier of the prompts to return. Defaults to None. - trace_id (Optional[str]): Trace identifier - from_start_time (Optional[dt.datetime]): Retrieve only prompts with a start_time on or after this datetime. Defaults to None. - to_start_time (Optional[dt.datetime]): Retrieve only prompts with a start_time before this datetime. Defaults to None. + label (Optional[str]): Label of the prompts to return. Defaults to None. + tag (Optional[str]): Tag of the prompts to return. Defaults to None. request_options (Optional[RequestOptions]): Type of the prompt. Defaults to None. Returns: @@ -1001,16 +997,14 @@ def fetch_prompts( """ try: self.log.debug( - f"Getting prompts... {page}, {limit}, {name}, {user_id}, {trace_id}, {from_start_time}, {to_start_time}, {request_options}" + f"Getting prompts... {page}, {limit}, {name}, {label}, {tag}, {request_options}" ) res = self.client.prompts.list( page=page, limit=limit, name=name, - user_id=user_id, - trace_id=trace_id, - from_start_time=from_start_time, - to_start_time=to_start_time, + label=label, + tag=tag, request_options=request_options, ) return FetchPromptsResponse(data=res.data, meta=res.meta) From 9a410aee00de8a2acc0f8340b3b73de641d4575d Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:17:49 +0200 Subject: [PATCH 16/40] refactor: remove unnecessary blank lines in Langfuse client --- tests/test_core_sdk.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 088bc1dc1..607fcddc1 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1504,8 +1504,7 @@ def test_fetch_scores(): def test_fetch_prompts(): langfuse = Langfuse() - # Create a trace with a prompt - name = create_uuid() + # Create a prompt langfuse.create_prompt( name="event-planner", prompt="Plan an event titled {{Event Name}}. The event will be about: {{Event Description}}. " From 50ad9d5579cfeb639cab853e9305dea1677b2ec9 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:44:18 +0200 Subject: [PATCH 17/40] refactor: add user_id and name parameters to Langfuse client's fetch_scores method --- langfuse/client.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/langfuse/client.py b/langfuse/client.py index c478b0bc5..8847d8a08 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -28,6 +28,7 @@ DatasetRunWithItems, ) from langfuse.api.resources.commons.types.observations_view import ObservationsView +from langfuse.api.resources.commons.types.score_source import ScoreSource from langfuse.api.resources.commons.types.session import Session from langfuse.api.resources.commons.types.trace_with_details import TraceWithDetails from langfuse.api.resources.datasets.types.paginated_dataset_runs import ( @@ -937,16 +938,35 @@ def fetch_scores( *, page: typing.Optional[int] = None, limit: typing.Optional[int] = None, + user_id: str | None = None, + name: str | None = None, from_timestamp: typing.Optional[dt.datetime] = None, to_timestamp: typing.Optional[dt.datetime] = None, + source: ScoreSource | None = None, + operator: str | None = None, + value: float | None = None, + score_ids: str | None = None, + config_id: str | None = None, + data_type: ScoreDataType | None = None, + request_options: RequestOptions | None = None, + ) -> FetchScoresResponse: """Get a list of scores in the current project. Args: page (Optional[int]): Page number of the scores to return. Defaults to None. limit (Optional[int]): Maximum number of scores to return. Defaults to None. + user_id (Optional[str]): User identifier. Defaults to None. + name (Optional[str]): Name of the scores to return. Defaults to None. from_timestamp (Optional[dt.datetime]): Retrieve only scores with a timestamp on or after this datetime. Defaults to None. to_timestamp (Optional[dt.datetime]): Retrieve only scores with a timestamp before this datetime. Defaults to None. + source (Optional[ScoreSource]): Source of the scores. Defaults to None. + operator (Optional[str]): Operator of the scores. Defaults to None. + value (Optional[float]): Value of the scores. Defaults to None. + score_ids (Optional[str]): Score identifier. Defaults to None. + config_id (Optional[str]): Configuration identifier. Defaults to None. + data_type (Optional[ScoreDataType]): Data type of the scores. Defaults to None. + request_options (Optional[RequestOptions]): Type of the score. Defaults to None. Returns: FetchScoresResponse, list of scores on `data` and metadata on `meta`. @@ -956,13 +976,22 @@ def fetch_scores( """ try: self.log.debug( - f"Getting scores... {page}, {limit}, {from_timestamp}, {to_timestamp}" + f"Getting scores... {page}, {limit}, {user_id}, {name}, {from_timestamp}, {to_timestamp}, {source}, {operator}, {value}, {score_ids}, {config_id}, {data_type}, {request_options}" ) - res = self.client.score.list( + res = self.client.score.get( page=page, limit=limit, + user_id=user_id, + name=name, from_timestamp=from_timestamp, to_timestamp=to_timestamp, + source=source, + operator=operator, + value=value, + score_ids=score_ids, + config_id=config_id, + data_type=data_type, + request_options=request_options, ) return FetchScoresResponse(data=res.data, meta=res.meta) except Exception as e: From 71452dc1e0bfb51ac90398e123199e18a1513537 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:44:22 +0200 Subject: [PATCH 18/40] refactor: update Langfuse client's list method parameters --- langfuse/api/resources/score/client.py | 157 ------------------------- 1 file changed, 157 deletions(-) diff --git a/langfuse/api/resources/score/client.py b/langfuse/api/resources/score/client.py index b88bcd8ce..5e4a7adb9 100644 --- a/langfuse/api/resources/score/client.py +++ b/langfuse/api/resources/score/client.py @@ -451,163 +451,6 @@ def delete( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) - def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - user_id: typing.Optional[str] = None, - name: typing.Optional[str] = None, - from_timestamp: typing.Optional[dt.datetime] = None, - to_timestamp: typing.Optional[dt.datetime] = None, - source: typing.Optional[ScoreSource] = None, - operator: typing.Optional[str] = None, - value: typing.Optional[float] = None, - score_ids: typing.Optional[str] = None, - config_id: typing.Optional[str] = None, - data_type: typing.Optional[ScoreDataType] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> Scores: - """ - Get a list of scores - - Parameters: - - page: typing.Optional[int]. Page number, starts at 1. - - - limit: typing.Optional[int]. Limit of items per page. If you encounter api issues due to too large page sizes, try to reduce the limit. - - - user_id: typing.Optional[str]. Retrieve only scores with this userId associated to the trace. - - - name: typing.Optional[str]. Retrieve only scores with this name. - - - from_timestamp: typing.Optional[dt.datetime]. Optional filter to only include scores created on or after a certain datetime (ISO 8601) - - - to_timestamp: typing.Optional[dt.datetime]. Optional filter to only include scores created before a certain datetime (ISO 8601) - - - source: typing.Optional[ScoreSource]. Retrieve only scores from a specific source. - - - operator: typing.Optional[str]. Retrieve only scores with value. - - - value: typing.Optional[float]. Retrieve only scores with value. - - - score_ids: typing.Optional[str]. Comma-separated list of score IDs to limit the results to. - - - config_id: typing.Optional[str]. Retrieve only scores with a specific configId. - - - data_type: typing.Optional[ScoreDataType]. Retrieve only scores with a specific dataType. - - - request_options: typing.Optional[RequestOptions]. Request-specific configuration. - --- - import datetime - - from finto import ScoreDataType, ScoreSource - from finto.client import FernLangfuse - - client = FernLangfuse( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.score.list( - page=1, - limit=1, - user_id="string", - name="string", - from_timestamp=datetime.datetime.fromisoformat( - "2024-01-15 09:30:00+00:00", - ), - to_timestamp=datetime.datetime.fromisoformat( - "2024-01-15 09:30:00+00:00", - ), - source=ScoreSource.ANNOTATION, - operator="string", - value=1.1, - score_ids="string", - config_id="string", - data_type=ScoreDataType.NUMERIC, - ) - """ - _response = self._client_wrapper.httpx_client.request( - "GET", - urllib.parse.urljoin( - f"{self._client_wrapper.get_base_url()}/", "api/public/scores" - ), - params=jsonable_encoder( - remove_none_from_dict( - { - "page": page, - "limit": limit, - "userId": user_id, - "name": name, - "fromTimestamp": serialize_datetime(from_timestamp) - if from_timestamp is not None - else None, - "toTimestamp": serialize_datetime(to_timestamp) - if to_timestamp is not None - else None, - "source": source, - "operator": operator, - "value": value, - "scoreIds": score_ids, - "configId": config_id, - "dataType": data_type, - **( - request_options.get("additional_query_parameters", {}) - if request_options is not None - else {} - ), - } - ) - ), - headers=jsonable_encoder( - remove_none_from_dict( - { - **self._client_wrapper.get_headers(), - **( - request_options.get("additional_headers", {}) - if request_options is not None - else {} - ), - } - ) - ), - timeout=request_options.get("timeout_in_seconds") - if request_options is not None - and request_options.get("timeout_in_seconds") is not None - else self._client_wrapper.get_timeout(), - retries=0, - max_retries=request_options.get("max_retries") - if request_options is not None - else 0, # type: ignore - ) - if 200 <= _response.status_code < 300: - return pydantic_v1.parse_obj_as(Scores, _response.json()) # type: ignore - if _response.status_code == 400: - raise Error(pydantic_v1.parse_obj_as(typing.Any, _response.json())) # type: ignore - if _response.status_code == 401: - raise UnauthorizedError( - pydantic_v1.parse_obj_as(typing.Any, _response.json()) - ) # type: ignore - if _response.status_code == 403: - raise AccessDeniedError( - pydantic_v1.parse_obj_as(typing.Any, _response.json()) - ) # type: ignore - if _response.status_code == 405: - raise MethodNotAllowedError( - pydantic_v1.parse_obj_as(typing.Any, _response.json()) - ) # type: ignore - if _response.status_code == 404: - raise NotFoundError(pydantic_v1.parse_obj_as(typing.Any, _response.json())) # type: ignore - try: - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, body=_response.text) - raise ApiError(status_code=_response.status_code, body=_response_json) - - class AsyncScoreClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): self._client_wrapper = client_wrapper From c13f56a79c268a8a64b0290427c0b4ab23b2f462 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:44:25 +0200 Subject: [PATCH 19/40] refactor: create multiple versions of a prompt in Langfuse client --- tests/test_core_sdk.py | 48 +++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 607fcddc1..b348afe0c 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1504,19 +1504,37 @@ def test_fetch_scores(): def test_fetch_prompts(): langfuse = Langfuse() - # Create a prompt + # Create multiple versions of a prompt langfuse.create_prompt( - name="event-planner", - prompt="Plan an event titled {{Event Name}}. The event will be about: {{Event Description}}. " - "The event will be held in {{Location}} on {{Date}}. " - "Consider the following factors: audience, budget, venue, catering options, and entertainment. " - "Provide a detailed plan including potential vendors and logistics.", + name="simple-prompt", + prompt="What is the weather like today?", config={ "model": "gpt-3.5-turbo-1106", "temperature": 0, }, labels=["production"], ) + + langfuse.create_prompt( + name="simple-prompt", + prompt="What is the weather like today?", + config={ + "model": "gpt-3.5-turbo-1106", + "temperature": 0.7, + }, + labels=["staging"], + ) + + langfuse.create_prompt( + name="simple-prompt", + prompt="What is the weather like today?", + config={ + "model": "gpt-4o-mini", + "temperature": 0.5, + }, + labels=["development"], + ) + langfuse.flush() # Fetch prompts @@ -1527,9 +1545,15 @@ def test_fetch_prompts(): assert hasattr(response, "data") assert hasattr(response, "meta") assert isinstance(response.data, list) - assert response.data[0].name == "event-planner" - assert ( - response.data[0].prompt - == "Plan an event titled {{Event Name}}. The event will be about: {{Event Description}}. The event will be held in {{Location}} on {{Date}}. Consider the following factors: audience, budget, venue, catering options, and entertainment. Provide a detailed plan including potential vendors and logistics." - ) - assert response.data[0].labels == ["production"] + + # Check that all versions and labels are present + assert response.data[0].name == "simple-prompt" + assert response.data[0].labels == ["development"] + assert response.data[0].version == 3 + assert response.data[1].name == "simple-prompt" + assert response.data[1].labels == ["staging"] + assert response.data[1].version == 2 + assert response.data[2].name == "simple-prompt" + assert response.data[2].labels == ["production"] + assert response.data[2].version == 1 + assert len(response.data) == 3 From 400ffeaab5dcfc5f91b67d0ae7911cb9a711fba3 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:53:18 +0200 Subject: [PATCH 20/40] refactor: update Langfuse client's fetch_scores method parameters --- langfuse/client.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/langfuse/client.py b/langfuse/client.py index 8847d8a08..97ebec581 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -938,17 +938,17 @@ def fetch_scores( *, page: typing.Optional[int] = None, limit: typing.Optional[int] = None, - user_id: str | None = None, - name: str | None = None, + user_id: typing.Optional[str] = None, + name: typing.Optional[str] = None, from_timestamp: typing.Optional[dt.datetime] = None, to_timestamp: typing.Optional[dt.datetime] = None, - source: ScoreSource | None = None, - operator: str | None = None, - value: float | None = None, - score_ids: str | None = None, - config_id: str | None = None, - data_type: ScoreDataType | None = None, - request_options: RequestOptions | None = None, + source: typing.Optional[ScoreSource] = None, + operator: typing.Optional[str] = None, + value: typing.Optional[float] = None, + score_ids: typing.Optional[str] = None, + config_id: typing.Optional[str] = None, + data_type: typing.Optional[ScoreDataType] = None, + request_options: typing.Optional[RequestOptions] = None, ) -> FetchScoresResponse: """Get a list of scores in the current project. From a0b41da28bdcb21a5ec6ef132fcefabc4ea811b0 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 14:53:22 +0200 Subject: [PATCH 21/40] refactor: update Langfuse client's fetch_scores method parameters --- langfuse/api/resources/score/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/langfuse/api/resources/score/client.py b/langfuse/api/resources/score/client.py index 5e4a7adb9..29674f622 100644 --- a/langfuse/api/resources/score/client.py +++ b/langfuse/api/resources/score/client.py @@ -451,6 +451,7 @@ def delete( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + class AsyncScoreClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): self._client_wrapper = client_wrapper From a88b6485acc7ec39c65b8a8eba7f47b59bcc33d8 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 15:08:20 +0200 Subject: [PATCH 22/40] refactor: update Langfuse client's fetch_scores method parameters --- tests/test_core_sdk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index b348afe0c..5e4c08d0a 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1548,12 +1548,12 @@ def test_fetch_prompts(): # Check that all versions and labels are present assert response.data[0].name == "simple-prompt" - assert response.data[0].labels == ["development"] + assert response.data[0].labels == ["latest", "production", "staging", "development"] assert response.data[0].version == 3 assert response.data[1].name == "simple-prompt" - assert response.data[1].labels == ["staging"] + assert response.data[1].labels == ["production", "staging", "development"] assert response.data[1].version == 2 assert response.data[2].name == "simple-prompt" - assert response.data[2].labels == ["production"] + assert response.data[2].labels == ["production", "staging", "development"] assert response.data[2].version == 1 assert len(response.data) == 3 From c5c216c9429561153a74e2ca75bbb8eda1e9d398 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 15:23:28 +0200 Subject: [PATCH 23/40] refactor: Update Langfuse client's fetch_scores method parameters --- tests/test_core_sdk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 5e4c08d0a..d70c15f49 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1548,12 +1548,12 @@ def test_fetch_prompts(): # Check that all versions and labels are present assert response.data[0].name == "simple-prompt" - assert response.data[0].labels == ["latest", "production", "staging", "development"] + assert set(response.data[0].labels) == {"latest", "production", "staging", "development"} assert response.data[0].version == 3 assert response.data[1].name == "simple-prompt" - assert response.data[1].labels == ["production", "staging", "development"] + assert set(response.data[1].labels) == {"production", "staging", "development"} assert response.data[1].version == 2 assert response.data[2].name == "simple-prompt" - assert response.data[2].labels == ["production", "staging", "development"] + assert set(response.data[2].labels) == {"production", "staging", "development"} assert response.data[2].version == 1 assert len(response.data) == 3 From 49cb6c61b16b7d2f3079b89c95b70b8f001500d8 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 15:40:46 +0200 Subject: [PATCH 24/40] refactor: Update Langfuse client's fetch_scores method parameters --- tests/test_core_sdk.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index d70c15f49..d1a981210 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1549,11 +1549,11 @@ def test_fetch_prompts(): # Check that all versions and labels are present assert response.data[0].name == "simple-prompt" assert set(response.data[0].labels) == {"latest", "production", "staging", "development"} - assert response.data[0].version == 3 + assert response.data[0].versions == [3, 2, 1] assert response.data[1].name == "simple-prompt" assert set(response.data[1].labels) == {"production", "staging", "development"} - assert response.data[1].version == 2 + assert response.data[1].versions == [2, 1] assert response.data[2].name == "simple-prompt" assert set(response.data[2].labels) == {"production", "staging", "development"} - assert response.data[2].version == 1 + assert response.data[2].versions == [1] assert len(response.data) == 3 From e8da5e121c287897b2cc82e9f94f682931223f65 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 15:57:17 +0200 Subject: [PATCH 25/40] refactor: Update Langfuse client's fetch_scores method parameters --- tests/test_core_sdk.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index d1a981210..4bfeebd35 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1502,7 +1502,10 @@ def test_fetch_scores(): def test_fetch_prompts(): - langfuse = Langfuse() + langfuse = Langfuse( + public_key="pk-lf-1234567890", + secret_key="sk-lf-1234567890", + ) # Create multiple versions of a prompt langfuse.create_prompt( @@ -1545,7 +1548,6 @@ def test_fetch_prompts(): assert hasattr(response, "data") assert hasattr(response, "meta") assert isinstance(response.data, list) - # Check that all versions and labels are present assert response.data[0].name == "simple-prompt" assert set(response.data[0].labels) == {"latest", "production", "staging", "development"} From 7c58106e4b16f47b52e9c42a82e20d17e47c71e3 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 16:25:35 +0200 Subject: [PATCH 26/40] refactor: Update Langfuse client's fetch_scores method parameters --- tests/test_core_sdk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 4bfeebd35..72e58505d 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1551,10 +1551,10 @@ def test_fetch_prompts(): # Check that all versions and labels are present assert response.data[0].name == "simple-prompt" assert set(response.data[0].labels) == {"latest", "production", "staging", "development"} - assert response.data[0].versions == [3, 2, 1] + assert response.data[0].versions == [1, 2, 3] assert response.data[1].name == "simple-prompt" assert set(response.data[1].labels) == {"production", "staging", "development"} - assert response.data[1].versions == [2, 1] + assert response.data[1].versions == [1, 2] assert response.data[2].name == "simple-prompt" assert set(response.data[2].labels) == {"production", "staging", "development"} assert response.data[2].versions == [1] From 2fb63253660975c5ab096c0d6647fc861b7fc08e Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 16:37:43 +0200 Subject: [PATCH 27/40] refactor: Update Langfuse client's fetch_scores method parameters --- tests/test_core_sdk.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 72e58505d..06b6dc5b8 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1502,10 +1502,7 @@ def test_fetch_scores(): def test_fetch_prompts(): - langfuse = Langfuse( - public_key="pk-lf-1234567890", - secret_key="sk-lf-1234567890", - ) + langfuse = Langfuse() # Create multiple versions of a prompt langfuse.create_prompt( From 410bd7a23b57cfcd0d5d20b6c2212901575e4124 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 16:51:47 +0200 Subject: [PATCH 28/40] refactor: Update Langfuse client's fetch_prompts method to fetch a specific prompt by name --- tests/test_core_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 06b6dc5b8..cfaf1ac69 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1538,7 +1538,7 @@ def test_fetch_prompts(): langfuse.flush() # Fetch prompts - response = langfuse.fetch_prompts() + response = langfuse.fetch_prompts(name="simple-prompt") # Assert the structure of the response assert isinstance(response, FetchPromptsResponse) From e4a342e1dfc031529f1334b4f44e7dae3af2a55e Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Thu, 1 Aug 2024 17:09:59 +0200 Subject: [PATCH 29/40] refactor: Update Langfuse client's fetch_prompts method to fetch a specific prompt by name --- tests/test_core_sdk.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index cfaf1ac69..7e521bb73 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1549,10 +1549,4 @@ def test_fetch_prompts(): assert response.data[0].name == "simple-prompt" assert set(response.data[0].labels) == {"latest", "production", "staging", "development"} assert response.data[0].versions == [1, 2, 3] - assert response.data[1].name == "simple-prompt" - assert set(response.data[1].labels) == {"production", "staging", "development"} - assert response.data[1].versions == [1, 2] - assert response.data[2].name == "simple-prompt" - assert set(response.data[2].labels) == {"production", "staging", "development"} - assert response.data[2].versions == [1] - assert len(response.data) == 3 + assert len(response.data[0].versions) == 3 From 4f74d3163500bca808a3068a242fad555ecc4890 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 13:32:10 +0200 Subject: [PATCH 30/40] refactor: Add methods to Langfuse client for retrieving datasets and dataset items --- langfuse/client.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/langfuse/client.py b/langfuse/client.py index 97ebec581..0557aea9c 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -368,6 +368,26 @@ def get_dataset( except Exception as e: self.log.exception(e) raise e + + def get_datasets( + self, *, page: Optional[int] = None, limit: Optional[int] = None, request_options: Optional[RequestOptions] = None + ) -> typing.List[Dataset]: + """Get all datasets. + + Args: + page (Optional[int]): Page number of the datasets to return, starts at 1. Defaults to None. + limit (Optional[int]): Maximum number of datasets to return. Defaults to 50. + request_options (Optional[RequestOptions]): Request options. Defaults to None. + + Returns: + List[Dataset]: The datasets. + """ + try: + self.log.debug("Getting datasets") + return self.client.datasets.list(page=page, limit=limit, request_options=request_options) + except Exception as e: + self.log.exception(e) + raise e def get_dataset_item(self, id: str) -> "DatasetItemClient": """Get the dataset item with the given id.""" @@ -539,6 +559,38 @@ def create_dataset_item( except Exception as e: self.log.exception(e) raise e + + def get_dataset_items( + self, + dataset_name: typing.Optional[str] = None, + source_trace_id: typing.Optional[str] = None, + source_observation_id: typing.Optional[str] = None, + page: typing.Optional[int] = None, + limit: typing.Optional[int] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> typing.List[DatasetItem]: + """Get all dataset items. + + Args: + dataset_name (Optional[str]): Name of the dataset. + source_trace_id (Optional[str]): Source trace id. + source_observation_id (Optional[str]): Source observation id. + page (Optional[int]): Page number of the dataset items to return, starts at 1. Defaults to None. + limit (Optional[int]): Maximum number of dataset items to return. Defaults to 50. + request_options (Optional[RequestOptions]): Request options. Defaults to None. + + Returns: + List[DatasetItem]: The dataset items. + """ + try: + self.log.debug("Getting dataset items") + return self.client.dataset_items.list( + dataset_name=dataset_name, source_trace_id=source_trace_id, source_observation_id=source_observation_id, page=page, limit=limit, request_options=request_options + ) + except Exception as e: + self.log.exception(e) + raise e + def fetch_trace( self, From 4b04131c2cc9fc83329fafaa25c9262093823f95 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 13:32:16 +0200 Subject: [PATCH 31/40] refactor: Add methods to Langfuse client for retrieving datasets and dataset items --- tests/test_datasets.py | 70 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 9265167f3..c746dad42 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -555,3 +555,73 @@ def execute_dataset_item(item, run_name, trace_id): assert next_trace.output == "non-dataset-run-afterwards" assert len(next_trace.observations) == 0 assert next_trace.id != trace_id + +def test_get_dataset_items(): + langfuse = Langfuse(debug=False) + name = create_uuid() + langfuse.create_dataset(name=name) + + input = {"input": "Hello World"} + for _ in range(99): + langfuse.create_dataset_item(dataset_name=name, input=input) + + + dataset_items = langfuse.get_dataset_items(name) + assert len(dataset_items.data) == 99 + assert dataset_items.meta.total_items == 99 + assert dataset_items.meta.total_pages == 1 + assert dataset_items.meta.page == 1 + assert dataset_items.meta.limit == 50 + + dataset_items_2 = langfuse.get_dataset_items(name, page=1, limit=50) + assert len(dataset_items_2.data) == 50 + assert dataset_items_2.meta.total_items == 99 + assert dataset_items_2.meta.total_pages == 2 + assert dataset_items_2.meta.page == 1 + assert dataset_items_2.meta.limit == 50 + + dataset_items_3 = langfuse.get_dataset_items(name, page=2, limit=50) + assert len(dataset_items_3.data) == 49 + assert dataset_items_3.meta.total_items == 99 + assert dataset_items_3.meta.total_pages == 2 + assert dataset_items_3.meta.page == 2 + assert dataset_items_3.meta.limit == 50 + +def test_get_datasets(): + langfuse = Langfuse(debug=False) + name = create_uuid() + langfuse.create_dataset(name=name) + + datasets = langfuse.get_datasets() + assert len(datasets.data) == 1 + assert datasets.meta.total_items == 1 + assert datasets.meta.total_pages == 1 + assert datasets.meta.page == 1 + assert datasets.meta.limit == 50 + + name_2 = create_uuid() + langfuse.create_dataset(name=name_2) + + datasets_2 = langfuse.get_datasets() + assert len(datasets_2.data) == 2 + assert datasets_2.meta.total_items == 2 + assert datasets_2.meta.total_pages == 1 + assert datasets_2.meta.page == 1 + assert datasets_2.meta.limit == 50 + + name_3 = create_uuid() + langfuse.create_dataset(name=name_3) + + datasets_3 = langfuse.get_datasets(page=1, limit=2) + assert len(datasets_3.data) == 2 + assert datasets_3.meta.total_items == 3 + assert datasets_3.meta.total_pages == 2 + assert datasets_3.meta.page == 1 + assert datasets_3.meta.limit == 2 + + datasets_4 = langfuse.get_datasets(page=2, limit=2) + assert len(datasets_4.data) == 1 + assert datasets_4.meta.total_items == 3 + assert datasets_4.meta.total_pages == 2 + assert datasets_4.meta.page == 2 + assert datasets_4.meta.limit == 2 \ No newline at end of file From 807ec4aef3c1b8a77a103ae5c4d2d4734332cb43 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 13:52:58 +0200 Subject: [PATCH 32/40] refactor: Update Langfuse client's fetch_scores method parameters --- tests/test_datasets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index c746dad42..173a8eba5 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -565,14 +565,15 @@ def test_get_dataset_items(): for _ in range(99): langfuse.create_dataset_item(dataset_name=name, input=input) - + # Fetch all dataset items without pagination dataset_items = langfuse.get_dataset_items(name) assert len(dataset_items.data) == 99 assert dataset_items.meta.total_items == 99 - assert dataset_items.meta.total_pages == 1 + assert dataset_items.meta.total_pages == 2 assert dataset_items.meta.page == 1 assert dataset_items.meta.limit == 50 + # Fetch dataset items with pagination dataset_items_2 = langfuse.get_dataset_items(name, page=1, limit=50) assert len(dataset_items_2.data) == 50 assert dataset_items_2.meta.total_items == 99 @@ -587,6 +588,7 @@ def test_get_dataset_items(): assert dataset_items_3.meta.page == 2 assert dataset_items_3.meta.limit == 50 + def test_get_datasets(): langfuse = Langfuse(debug=False) name = create_uuid() From 55a7bc3d755ad95d61ff58b9de252237a1e11de5 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 14:29:19 +0200 Subject: [PATCH 33/40] refactor: Add tests for retrieving dataset items and datasets --- tests/test_core_sdk.py | 73 ++++++++++++++++++++++++++++++++++++++++++ tests/test_datasets.py | 72 ----------------------------------------- 2 files changed, 73 insertions(+), 72 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 7e521bb73..f5bb2aa00 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1550,3 +1550,76 @@ def test_fetch_prompts(): assert set(response.data[0].labels) == {"latest", "production", "staging", "development"} assert response.data[0].versions == [1, 2, 3] assert len(response.data[0].versions) == 3 + +def test_get_dataset_items(): + langfuse = Langfuse() + name = create_uuid() + langfuse.create_dataset(name=name) + + input = {"input": "Hello World"} + for _ in range(99): + langfuse.create_dataset_item(dataset_name=name, input=input) + + # Fetch all dataset items without pagination + dataset_items = langfuse.get_dataset_items(name) + assert len(dataset_items.data) == 50 + assert dataset_items.meta.total_items == 99 + assert dataset_items.meta.total_pages == 2 + assert dataset_items.meta.page == 1 + assert dataset_items.meta.limit == 50 + + # Fetch dataset items with pagination + dataset_items_2 = langfuse.get_dataset_items(dataset_name=name, page=1, limit=49) + assert len(dataset_items_2.data) == 49 + assert dataset_items_2.meta.total_items == 99 + assert dataset_items_2.meta.total_pages == 3 + assert dataset_items_2.meta.page == 1 + assert dataset_items_2.meta.limit == 49 + + dataset_items_3 = langfuse.get_dataset_items(name, page=2, limit=50) + assert len(dataset_items_3.data) == 49 + assert dataset_items_3.meta.total_items == 99 + assert dataset_items_3.meta.total_pages == 2 + assert dataset_items_3.meta.page == 2 + assert dataset_items_3.meta.limit == 50 + + +def test_get_datasets(): + langfuse = Langfuse() + name = create_uuid() + langfuse.create_dataset(name=name) + + datasets = langfuse.get_datasets() + print(datasets) + assert len(datasets.data) == 1 + assert datasets.meta.total_items == 1 + assert datasets.meta.total_pages == 1 + assert datasets.meta.page == 1 + assert datasets.meta.limit == 50 + + name_2 = create_uuid() + langfuse.create_dataset(name=name_2) + + datasets_2 = langfuse.get_datasets() + assert len(datasets_2.data) == 2 + assert datasets_2.meta.total_items == 2 + assert datasets_2.meta.total_pages == 1 + assert datasets_2.meta.page == 1 + assert datasets_2.meta.limit == 50 + + name_3 = create_uuid() + langfuse.create_dataset(name=name_3) + + datasets_3 = langfuse.get_datasets(page=1, limit=2) + assert len(datasets_3.data) == 2 + assert datasets_3.meta.total_items == 3 + assert datasets_3.meta.total_pages == 2 + assert datasets_3.meta.page == 1 + assert datasets_3.meta.limit == 2 + + datasets_4 = langfuse.get_datasets(page=2, limit=2) + assert len(datasets_4.data) == 1 + assert datasets_4.meta.total_items == 3 + assert datasets_4.meta.total_pages == 2 + assert datasets_4.meta.page == 2 + assert datasets_4.meta.limit == 2 \ No newline at end of file diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 173a8eba5..9265167f3 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -555,75 +555,3 @@ def execute_dataset_item(item, run_name, trace_id): assert next_trace.output == "non-dataset-run-afterwards" assert len(next_trace.observations) == 0 assert next_trace.id != trace_id - -def test_get_dataset_items(): - langfuse = Langfuse(debug=False) - name = create_uuid() - langfuse.create_dataset(name=name) - - input = {"input": "Hello World"} - for _ in range(99): - langfuse.create_dataset_item(dataset_name=name, input=input) - - # Fetch all dataset items without pagination - dataset_items = langfuse.get_dataset_items(name) - assert len(dataset_items.data) == 99 - assert dataset_items.meta.total_items == 99 - assert dataset_items.meta.total_pages == 2 - assert dataset_items.meta.page == 1 - assert dataset_items.meta.limit == 50 - - # Fetch dataset items with pagination - dataset_items_2 = langfuse.get_dataset_items(name, page=1, limit=50) - assert len(dataset_items_2.data) == 50 - assert dataset_items_2.meta.total_items == 99 - assert dataset_items_2.meta.total_pages == 2 - assert dataset_items_2.meta.page == 1 - assert dataset_items_2.meta.limit == 50 - - dataset_items_3 = langfuse.get_dataset_items(name, page=2, limit=50) - assert len(dataset_items_3.data) == 49 - assert dataset_items_3.meta.total_items == 99 - assert dataset_items_3.meta.total_pages == 2 - assert dataset_items_3.meta.page == 2 - assert dataset_items_3.meta.limit == 50 - - -def test_get_datasets(): - langfuse = Langfuse(debug=False) - name = create_uuid() - langfuse.create_dataset(name=name) - - datasets = langfuse.get_datasets() - assert len(datasets.data) == 1 - assert datasets.meta.total_items == 1 - assert datasets.meta.total_pages == 1 - assert datasets.meta.page == 1 - assert datasets.meta.limit == 50 - - name_2 = create_uuid() - langfuse.create_dataset(name=name_2) - - datasets_2 = langfuse.get_datasets() - assert len(datasets_2.data) == 2 - assert datasets_2.meta.total_items == 2 - assert datasets_2.meta.total_pages == 1 - assert datasets_2.meta.page == 1 - assert datasets_2.meta.limit == 50 - - name_3 = create_uuid() - langfuse.create_dataset(name=name_3) - - datasets_3 = langfuse.get_datasets(page=1, limit=2) - assert len(datasets_3.data) == 2 - assert datasets_3.meta.total_items == 3 - assert datasets_3.meta.total_pages == 2 - assert datasets_3.meta.page == 1 - assert datasets_3.meta.limit == 2 - - datasets_4 = langfuse.get_datasets(page=2, limit=2) - assert len(datasets_4.data) == 1 - assert datasets_4.meta.total_items == 3 - assert datasets_4.meta.total_pages == 2 - assert datasets_4.meta.page == 2 - assert datasets_4.meta.limit == 2 \ No newline at end of file From 3c1dc9b47a46ff33b81b83e0821bce1f8b61ccb3 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 14:52:07 +0200 Subject: [PATCH 34/40] refactor: Update Langfuse client's fetch_datasets method to handle pagination --- tests/test_core_sdk.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index f5bb2aa00..b79ac30f8 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1589,11 +1589,12 @@ def test_get_datasets(): name = create_uuid() langfuse.create_dataset(name=name) + # Fetch datasets, considering that datasets from previous tests might still exist datasets = langfuse.get_datasets() + initial_count = len(datasets.data) print(datasets) - assert len(datasets.data) == 1 - assert datasets.meta.total_items == 1 - assert datasets.meta.total_pages == 1 + assert datasets.meta.total_items == initial_count + assert datasets.meta.total_pages == (initial_count // 50) + 1 assert datasets.meta.page == 1 assert datasets.meta.limit == 50 @@ -1601,9 +1602,9 @@ def test_get_datasets(): langfuse.create_dataset(name=name_2) datasets_2 = langfuse.get_datasets() - assert len(datasets_2.data) == 2 - assert datasets_2.meta.total_items == 2 - assert datasets_2.meta.total_pages == 1 + assert len(datasets_2.data) == initial_count + 1 + assert datasets_2.meta.total_items == initial_count + 1 + assert datasets_2.meta.total_pages == ((initial_count + 1) // 50) + 1 assert datasets_2.meta.page == 1 assert datasets_2.meta.limit == 50 @@ -1612,14 +1613,14 @@ def test_get_datasets(): datasets_3 = langfuse.get_datasets(page=1, limit=2) assert len(datasets_3.data) == 2 - assert datasets_3.meta.total_items == 3 - assert datasets_3.meta.total_pages == 2 + assert datasets_3.meta.total_items == initial_count + 2 + assert datasets_3.meta.total_pages == ((initial_count + 2) // 2) + 1 assert datasets_3.meta.page == 1 assert datasets_3.meta.limit == 2 datasets_4 = langfuse.get_datasets(page=2, limit=2) - assert len(datasets_4.data) == 1 - assert datasets_4.meta.total_items == 3 - assert datasets_4.meta.total_pages == 2 + assert len(datasets_4.data) == min(2, initial_count + 2 - 2) + assert datasets_4.meta.total_items == initial_count + 2 + assert datasets_4.meta.total_pages == ((initial_count + 2) // 2) + 1 assert datasets_4.meta.page == 2 assert datasets_4.meta.limit == 2 \ No newline at end of file From 2ecaecc1258af0d7ce49ddbcc156d8cf38b691b0 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 15:10:00 +0200 Subject: [PATCH 35/40] refactor: Update Langfuse client's fetch_datasets method to handle pagination --- tests/test_core_sdk.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index b79ac30f8..49410a362 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1592,7 +1592,6 @@ def test_get_datasets(): # Fetch datasets, considering that datasets from previous tests might still exist datasets = langfuse.get_datasets() initial_count = len(datasets.data) - print(datasets) assert datasets.meta.total_items == initial_count assert datasets.meta.total_pages == (initial_count // 50) + 1 assert datasets.meta.page == 1 @@ -1604,7 +1603,7 @@ def test_get_datasets(): datasets_2 = langfuse.get_datasets() assert len(datasets_2.data) == initial_count + 1 assert datasets_2.meta.total_items == initial_count + 1 - assert datasets_2.meta.total_pages == ((initial_count + 1) // 50) + 1 + assert datasets_2.meta.total_pages == ((initial_count + 1) // 50) assert datasets_2.meta.page == 1 assert datasets_2.meta.limit == 50 @@ -1614,13 +1613,13 @@ def test_get_datasets(): datasets_3 = langfuse.get_datasets(page=1, limit=2) assert len(datasets_3.data) == 2 assert datasets_3.meta.total_items == initial_count + 2 - assert datasets_3.meta.total_pages == ((initial_count + 2) // 2) + 1 + assert datasets_3.meta.total_pages == ((initial_count + 2) // 2) assert datasets_3.meta.page == 1 assert datasets_3.meta.limit == 2 datasets_4 = langfuse.get_datasets(page=2, limit=2) assert len(datasets_4.data) == min(2, initial_count + 2 - 2) assert datasets_4.meta.total_items == initial_count + 2 - assert datasets_4.meta.total_pages == ((initial_count + 2) // 2) + 1 + assert datasets_4.meta.total_pages == ((initial_count + 2) // 2) assert datasets_4.meta.page == 2 assert datasets_4.meta.limit == 2 \ No newline at end of file From 412ea056f720bf477a632b93db0a9e13a69fb21c Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 15:28:22 +0200 Subject: [PATCH 36/40] refactor: Update Langfuse client's fetch_datasets method to handle pagination --- tests/test_core_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 49410a362..443749e81 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1603,7 +1603,7 @@ def test_get_datasets(): datasets_2 = langfuse.get_datasets() assert len(datasets_2.data) == initial_count + 1 assert datasets_2.meta.total_items == initial_count + 1 - assert datasets_2.meta.total_pages == ((initial_count + 1) // 50) + assert datasets_2.meta.total_pages == ((initial_count + 1) // 50) + 1 assert datasets_2.meta.page == 1 assert datasets_2.meta.limit == 50 From d1509d88b4ca674f290ebcfa80cc9f1b0a85e393 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 16:22:39 +0200 Subject: [PATCH 37/40] refactor: Update Langfuse client's fetch_score method to fetch a specific score by id --- langfuse/client.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/langfuse/client.py b/langfuse/client.py index 0557aea9c..436357d8f 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -127,6 +127,12 @@ class FetchSessionsResponse: data: typing.List[Session] meta: MetaResponse +@dataclass +class FetchScoreResponse: + """Response object for fetch_score method.""" + + data: ScoreBody + meta: MetaResponse @dataclass class FetchScoresResponse: @@ -984,6 +990,29 @@ def fetch_sessions( except Exception as e: self.log.exception(e) raise e + + def fetch_score( + self, + id: str, + ) -> FetchScoreResponse: + """Get a score in the current project with the given identifier. + + Args: + id: The identifier of the score to fetch. + + Returns: + FetchScoreResponse: The score with the given id on `data`. + + Raises: + Exception: If the score with the given id could not be found within the authenticated project or if an error occurred during the request. + """ + try: + self.log.debug(f"Getting score {id}") + res = self.client.score.get_by_id(id) + return FetchScoreResponse(data=res.data, meta=res.meta) + except Exception as e: + self.log.exception(e) + raise e def fetch_scores( self, From cd90b20a7df516b417c06d0ffcf62ab206c8adf5 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 16:22:42 +0200 Subject: [PATCH 38/40] refactor: Update Langfuse client's fetch_score method to fetch a specific score by id --- tests/test_core_sdk.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 443749e81..3e52c8cc8 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1500,6 +1500,31 @@ def test_fetch_scores(): response = langfuse.fetch_scores(limit=1, page=2) assert len(response.data) == 1 +def test_fetch_score_by_id(): + langfuse = Langfuse( public_key="pk-lf-1234567890", + secret_key="sk-lf-1234567890",) + + # Create a trace with a score + name = create_uuid() + trace = langfuse.trace(name=name) + score_1 = trace.score(name="harmfulness", value=0.5) + score_2 = trace.score(name="quality", value=1) + langfuse.flush() + + # Fetch scores + res_1 = langfuse.fetch_score(id=score_1.id) + res_2 = langfuse.fetch_score(id=score_2.id) + + # Assert the structure of the response + assert isinstance(res_1, FetchScoresResponse) + assert hasattr(res_1, "data") + assert hasattr(res_1, "meta") + + # Check that the correct score is returned + assert res_1.data[0].name == "harmfulness" + assert res_1.data[0].value == 0.5 + assert res_2.data[0].name == "quality" + assert res_2.data[0].value == 1 def test_fetch_prompts(): langfuse = Langfuse() From 49bde62f09f703491503a91164fedb4dacd9eec5 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Fri, 2 Aug 2024 16:41:46 +0200 Subject: [PATCH 39/40] Refactor Langfuse client's fetch_score method to return specific score by id --- langfuse/client.py | 3 +-- tests/test_core_sdk.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/langfuse/client.py b/langfuse/client.py index 436357d8f..a58eb6c8e 100644 --- a/langfuse/client.py +++ b/langfuse/client.py @@ -132,7 +132,6 @@ class FetchScoreResponse: """Response object for fetch_score method.""" data: ScoreBody - meta: MetaResponse @dataclass class FetchScoresResponse: @@ -1009,7 +1008,7 @@ def fetch_score( try: self.log.debug(f"Getting score {id}") res = self.client.score.get_by_id(id) - return FetchScoreResponse(data=res.data, meta=res.meta) + return FetchScoreResponse(data=res) except Exception as e: self.log.exception(e) raise e diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index 3e52c8cc8..bc741bec1 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -7,6 +7,7 @@ FetchObservationResponse, FetchObservationsResponse, FetchPromptsResponse, + FetchScoreResponse, FetchScoresResponse, FetchSessionsResponse, FetchTraceResponse, @@ -1507,24 +1508,26 @@ def test_fetch_score_by_id(): # Create a trace with a score name = create_uuid() trace = langfuse.trace(name=name) - score_1 = trace.score(name="harmfulness", value=0.5) - score_2 = trace.score(name="quality", value=1) + trace.score(name="harmfulness", value=0.55) + trace.score(name="quality", value=0.99) langfuse.flush() + scores = langfuse.fetch_scores() + score_1 = scores.data[1] + score_2 = scores.data[0] # Fetch scores res_1 = langfuse.fetch_score(id=score_1.id) res_2 = langfuse.fetch_score(id=score_2.id) # Assert the structure of the response - assert isinstance(res_1, FetchScoresResponse) + assert isinstance(res_1, FetchScoreResponse) assert hasattr(res_1, "data") - assert hasattr(res_1, "meta") # Check that the correct score is returned - assert res_1.data[0].name == "harmfulness" - assert res_1.data[0].value == 0.5 - assert res_2.data[0].name == "quality" - assert res_2.data[0].value == 1 + assert res_1.data.name == "harmfulness" + assert res_1.data.value == 0.55 + assert res_2.data.name == "quality" + assert res_2.data.value == 0.99 def test_fetch_prompts(): langfuse = Langfuse() From 616fa424e512c70b944ad4b5a5d665f7124400b1 Mon Sep 17 00:00:00 2001 From: RichardKruemmel Date: Tue, 20 Aug 2024 11:52:10 +0200 Subject: [PATCH 40/40] refactor: simplify test_fetch_score_by_id function --- tests/test_core_sdk.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_core_sdk.py b/tests/test_core_sdk.py index ad8970cb9..e59fe7dcc 100644 --- a/tests/test_core_sdk.py +++ b/tests/test_core_sdk.py @@ -1500,8 +1500,7 @@ def test_fetch_scores(): assert len(response.data) == 1 def test_fetch_score_by_id(): - langfuse = Langfuse( public_key="pk-lf-1234567890", - secret_key="sk-lf-1234567890",) + langfuse = Langfuse() # Create a trace with a score name = create_uuid()