Skip to content

feat(scores): add session and dataset run scores #1201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hassiebp merged 5 commits into from
Jun 2, 2025
Merged

feat(scores): add session and dataset run scores #1201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6d03f47
feat(scores): add session and dataset run scores
hassiebp May 28, 2025
e9413e2
push
hassiebp May 28, 2025
1c83406
Merge branch 'main' into hassieb/lfe-4865-add-score-metadata
hassiebp May 28, 2025
807f7db
push
hassiebp Jun 2, 2025
b356fc0
push
hassiebp Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions langfuse/_client/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ def __init__(
self._project_id = None
sample_rate = sample_rate or float(os.environ.get(LANGFUSE_SAMPLE_RATE, 1.0))
if not 0.0 <= sample_rate <= 1.0:
raise ValueError(f"Sample rate must be between 0.0 and 1.0, got {sample_rate}")
raise ValueError(
f"Sample rate must be between 0.0 and 1.0, got {sample_rate}"
)

self._tracing_enabled = (
tracing_enabled
Expand Down Expand Up @@ -1214,12 +1216,15 @@ def create_score(
*,
name: str,
value: float,
trace_id: str,
session_id: Optional[str] = None,
dataset_run_id: Optional[str] = None,
trace_id: Optional[str] = None,
observation_id: Optional[str] = None,
score_id: Optional[str] = None,
data_type: Optional[Literal["NUMERIC", "BOOLEAN"]] = None,
comment: Optional[str] = None,
config_id: Optional[str] = None,
metadata: Optional[Any] = None,
) -> None: ...

@overload
Expand All @@ -1228,25 +1233,31 @@ def create_score(
*,
name: str,
value: str,
trace_id: str,
session_id: Optional[str] = None,
dataset_run_id: Optional[str] = None,
trace_id: Optional[str] = None,
score_id: Optional[str] = None,
observation_id: Optional[str] = None,
data_type: Optional[Literal["CATEGORICAL"]] = "CATEGORICAL",
comment: Optional[str] = None,
config_id: Optional[str] = None,
metadata: Optional[Any] = None,
) -> None: ...

def create_score(
self,
*,
name: str,
value: Union[float, str],
trace_id: str,
session_id: Optional[str] = None,
dataset_run_id: Optional[str] = None,
trace_id: Optional[str] = None,
observation_id: Optional[str] = None,
score_id: Optional[str] = None,
data_type: Optional[ScoreDataType] = None,
comment: Optional[str] = None,
config_id: Optional[str] = None,
metadata: Optional[Any] = None,
) -> None:
"""Create a score for a specific trace or observation.

Expand All @@ -1256,12 +1267,15 @@ def create_score(
Args:
name: Name of the score (e.g., "relevance", "accuracy")
value: Score value (can be numeric for NUMERIC/BOOLEAN types or string for CATEGORICAL)
session_id: ID of the Langfuse session to associate the score with
dataset_run_id: ID of the Langfuse dataset run to associate the score with
trace_id: ID of the Langfuse trace to associate the score with
observation_id: Optional ID of the specific observation to score
observation_id: Optional ID of the specific observation to score. Trace ID must be provided too.
score_id: Optional custom ID for the score (auto-generated if not provided)
data_type: Type of score (NUMERIC, BOOLEAN, or CATEGORICAL)
comment: Optional comment or explanation for the score
config_id: Optional ID of a score config defined in Langfuse
metadata: Optional metadata to be attached to the score

Example:
```python
Expand Down Expand Up @@ -1292,6 +1306,8 @@ def create_score(
try:
score_event = {
"id": score_id,
"session_id": session_id,
"dataset_run_id": dataset_run_id,
"trace_id": trace_id,
"observation_id": observation_id,
"name": name,
Expand All @@ -1300,6 +1316,7 @@ def create_score(
"comment": comment,
"config_id": config_id,
"environment": self._environment,
"metadata": metadata,
}

new_body = ScoreBody(**score_event)
Expand Down
19 changes: 12 additions & 7 deletions langfuse/_client/resource_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,13 +256,18 @@ def add_score_task(self, event: dict):
# Sample scores with the same sampler that is used for tracing
tracer_provider = cast(TracerProvider, otel_trace_api.get_tracer_provider())
should_sample = (
tracer_provider.sampler.should_sample(
parent_context=None,
trace_id=int(event["body"].trace_id, 16),
name="score",
).decision
== Decision.RECORD_AND_SAMPLE
if hasattr(event["body"], "trace_id")
(
tracer_provider.sampler.should_sample(
parent_context=None,
trace_id=int(event["body"].trace_id, 16),
name="score",
).decision
== Decision.RECORD_AND_SAMPLE
if hasattr(event["body"], "trace_id")
else True
)
if event["body"].trace_id
is not None # do not sample out session / dataset run scores
else True
)

Expand Down
90 changes: 84 additions & 6 deletions langfuse/api/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,20 @@
AnnotationQueueItem,
AnnotationQueueObjectType,
AnnotationQueueStatus,
ApiKeyDeletionResponse,
ApiKeyList,
ApiKeyResponse,
ApiKeySummary,
AuthenticationScheme,
BaseEvent,
BasePrompt,
BaseScore,
BaseScoreV1,
BooleanScore,
BooleanScoreV1,
BulkConfig,
CategoricalScore,
CategoricalScoreV1,
ChatMessage,
ChatPrompt,
Comment,
Expand Down Expand Up @@ -39,8 +48,6 @@
CreateSpanBody,
CreateSpanEvent,
CreateTextPromptRequest,
DailyMetrics,
DailyMetricsDetails,
Dataset,
DatasetItem,
DatasetRun,
Expand All @@ -51,7 +58,9 @@
DeleteDatasetItemResponse,
DeleteDatasetRunResponse,
DeleteTraceResponse,
EmptyResponse,
Error,
FilterConfig,
GetCommentsResponse,
GetMediaResponse,
GetMediaUploadUrlRequest,
Expand Down Expand Up @@ -83,11 +92,18 @@
IngestionUsage,
MapValue,
MediaContentType,
MembershipRequest,
MembershipResponse,
MembershipRole,
MembershipsResponse,
MethodNotAllowedError,
MetricsResponse,
Model,
ModelPrice,
ModelUsageUnit,
NotFoundError,
NumericScore,
NumericScoreV1,
Observation,
ObservationBody,
ObservationLevel,
Expand All @@ -99,33 +115,53 @@
OpenAiResponseUsageSchema,
OpenAiUsage,
OptionalObservationBody,
OrganizationProject,
OrganizationProjectsResponse,
PaginatedAnnotationQueueItems,
PaginatedAnnotationQueues,
PaginatedDatasetItems,
PaginatedDatasetRunItems,
PaginatedDatasetRuns,
PaginatedDatasets,
PaginatedModels,
PaginatedSessions,
PatchMediaBody,
Project,
ProjectDeletionResponse,
Projects,
Prompt,
PromptMeta,
PromptMetaListResponse,
Prompt_Chat,
Prompt_Text,
ResourceMeta,
ResourceType,
ResourceTypesResponse,
SchemaExtension,
SchemaResource,
SchemasResponse,
ScimEmail,
ScimFeatureSupport,
ScimName,
ScimUser,
ScimUsersListResponse,
Score,
ScoreBody,
ScoreConfig,
ScoreConfigs,
ScoreDataType,
ScoreEvent,
ScoreSource,
ScoreV1,
ScoreV1_Boolean,
ScoreV1_Categorical,
ScoreV1_Numeric,
Score_Boolean,
Score_Categorical,
Score_Numeric,
SdkLogBody,
SdkLogEvent,
ServiceProviderConfig,
ServiceUnavailableError,
Session,
SessionWithTraces,
Expand All @@ -146,8 +182,8 @@
UpdateSpanBody,
UpdateSpanEvent,
Usage,
UsageByModel,
UsageDetails,
UserMeta,
annotation_queues,
comments,
commons,
Expand All @@ -160,11 +196,14 @@
metrics,
models,
observations,
organizations,
projects,
prompt_version,
prompts,
scim,
score,
score_configs,
score_v_2,
sessions,
trace,
utils,
Expand All @@ -176,11 +215,20 @@
"AnnotationQueueItem",
"AnnotationQueueObjectType",
"AnnotationQueueStatus",
"ApiKeyDeletionResponse",
"ApiKeyList",
"ApiKeyResponse",
"ApiKeySummary",
"AuthenticationScheme",
"BaseEvent",
"BasePrompt",
"BaseScore",
"BaseScoreV1",
"BooleanScore",
"BooleanScoreV1",
"BulkConfig",
"CategoricalScore",
"CategoricalScoreV1",
"ChatMessage",
"ChatPrompt",
"Comment",
Expand Down Expand Up @@ -209,8 +257,6 @@
"CreateSpanBody",
"CreateSpanEvent",
"CreateTextPromptRequest",
"DailyMetrics",
"DailyMetricsDetails",
"Dataset",
"DatasetItem",
"DatasetRun",
Expand All @@ -221,7 +267,9 @@
"DeleteDatasetItemResponse",
"DeleteDatasetRunResponse",
"DeleteTraceResponse",
"EmptyResponse",
"Error",
"FilterConfig",
"GetCommentsResponse",
"GetMediaResponse",
"GetMediaUploadUrlRequest",
Expand Down Expand Up @@ -253,11 +301,18 @@
"IngestionUsage",
"MapValue",
"MediaContentType",
"MembershipRequest",
"MembershipResponse",
"MembershipRole",
"MembershipsResponse",
"MethodNotAllowedError",
"MetricsResponse",
"Model",
"ModelPrice",
"ModelUsageUnit",
"NotFoundError",
"NumericScore",
"NumericScoreV1",
"Observation",
"ObservationBody",
"ObservationLevel",
Expand All @@ -269,33 +324,53 @@
"OpenAiResponseUsageSchema",
"OpenAiUsage",
"OptionalObservationBody",
"OrganizationProject",
"OrganizationProjectsResponse",
"PaginatedAnnotationQueueItems",
"PaginatedAnnotationQueues",
"PaginatedDatasetItems",
"PaginatedDatasetRunItems",
"PaginatedDatasetRuns",
"PaginatedDatasets",
"PaginatedModels",
"PaginatedSessions",
"PatchMediaBody",
"Project",
"ProjectDeletionResponse",
"Projects",
"Prompt",
"PromptMeta",
"PromptMetaListResponse",
"Prompt_Chat",
"Prompt_Text",
"ResourceMeta",
"ResourceType",
"ResourceTypesResponse",
"SchemaExtension",
"SchemaResource",
"SchemasResponse",
"ScimEmail",
"ScimFeatureSupport",
"ScimName",
"ScimUser",
"ScimUsersListResponse",
"Score",
"ScoreBody",
"ScoreConfig",
"ScoreConfigs",
"ScoreDataType",
"ScoreEvent",
"ScoreSource",
"ScoreV1",
"ScoreV1_Boolean",
"ScoreV1_Categorical",
"ScoreV1_Numeric",
"Score_Boolean",
"Score_Categorical",
"Score_Numeric",
"SdkLogBody",
"SdkLogEvent",
"ServiceProviderConfig",
"ServiceUnavailableError",
"Session",
"SessionWithTraces",
Expand All @@ -316,8 +391,8 @@
"UpdateSpanBody",
"UpdateSpanEvent",
"Usage",
"UsageByModel",
"UsageDetails",
"UserMeta",
"annotation_queues",
"comments",
"commons",
Expand All @@ -330,11 +405,14 @@
"metrics",
"models",
"observations",
"organizations",
"projects",
"prompt_version",
"prompts",
"scim",
"score",
"score_configs",
"score_v_2",
"sessions",
"trace",
"utils",
Expand Down
Loading