From 19b3a1c804dce349196b26ed1a3e11dd8dfce712 Mon Sep 17 00:00:00 2001 From: "wangluming.wlm" Date: Wed, 27 Aug 2025 11:27:46 +0800 Subject: [PATCH 1/8] feat-mul --- cozeloop/entities/prompt.py | 18 ++ cozeloop/internal/prompt/converter.py | 27 ++- cozeloop/internal/prompt/openapi.py | 12 ++ cozeloop/internal/prompt/prompt.py | 54 +++++- .../multipart/prompt_hub_with_multipart.py | 163 ++++++++++++++++++ 5 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 examples/prompt/multipart/prompt_hub_with_multipart.py diff --git a/cozeloop/entities/prompt.py b/cozeloop/entities/prompt.py index f708523..72a7ca9 100644 --- a/cozeloop/entities/prompt.py +++ b/cozeloop/entities/prompt.py @@ -36,6 +36,7 @@ class VariableType(str, Enum): ARRAY_INTEGER = "array" ARRAY_FLOAT = "array" ARRAY_OBJECT = "array" + MULTI_PART = "multi_part" class ToolChoiceType(str, Enum): @@ -43,9 +44,26 @@ class ToolChoiceType(str, Enum): NONE = "none" +class ContentType(str, Enum): + TEXT = "text" + IMAGE_URL = "image_url" + MULTI_PART_VARIABLE = "multi_part_variable" + + +class ImageURL(BaseModel): + url: str + + +class ContentPart(BaseModel): + type: ContentType + text: Optional[str] = None + image_url: Optional[ImageURL] = None + + class Message(BaseModel): role: Role content: Optional[str] = None + parts: Optional[List[ContentPart]] = None class VariableDef(BaseModel): diff --git a/cozeloop/internal/prompt/converter.py b/cozeloop/internal/prompt/converter.py index 92e8738..1e6b9de 100644 --- a/cozeloop/internal/prompt/converter.py +++ b/cozeloop/internal/prompt/converter.py @@ -19,6 +19,8 @@ VariableType as EntityVariableType, ToolType as EntityToolType, PromptVariable, + ContentType as EntityContentType, + ContentPart as EntityContentPart, ) from cozeloop.internal.prompt.openapi import ( @@ -34,7 +36,9 @@ ToolType as OpenAPIToolType, Role as OpenAPIRole, ToolChoiceType as OpenAPIChoiceType, - TemplateType as OpenAPITemplateType + TemplateType as OpenAPITemplateType, + ContentType as OpenAPIContentType, + ContentPart as OpenAPIContentPart, ) @@ -49,10 +53,26 @@ def _convert_role(openapi_role: OpenAPIRole) -> EntityRole: return role_mapping.get(openapi_role, EntityRole.USER) # Default to USER type +def _convert_content_type(openapi_type: OpenAPIContentType) -> EntityContentType: + content_type_mapping = { + OpenAPIContentType.TEXT: EntityContentType.TEXT, + OpenAPIContentType.MULTI_PART_VARIABLE: EntityContentType.MULTI_PART_VARIABLE, + } + return content_type_mapping.get(openapi_type, EntityContentType.TEXT) + + +def to_content_part(openapi_part: OpenAPIContentPart) -> EntityContentPart: + return EntityContentPart( + type=_convert_content_type(openapi_part.type), + text=openapi_part.text + ) + + def _convert_message(msg: OpenAPIMessage) -> EntityMessage: return EntityMessage( role=_convert_role(msg.role), - content=msg.content + content=msg.content, + parts=[to_content_part(part) for part in msg.parts] if msg.parts else None ) @@ -68,7 +88,8 @@ def _convert_variable_type(openapi_type: OpenAPIVariableType) -> EntityVariableT OpenAPIVariableType.ARRAY_INTEGER: EntityVariableType.ARRAY_INTEGER, OpenAPIVariableType.ARRAY_FLOAT: EntityVariableType.ARRAY_FLOAT, OpenAPIVariableType.ARRAY_BOOLEAN: EntityVariableType.ARRAY_BOOLEAN, - OpenAPIVariableType.ARRAY_OBJECT: EntityVariableType.ARRAY_OBJECT + OpenAPIVariableType.ARRAY_OBJECT: EntityVariableType.ARRAY_OBJECT, + OpenAPIVariableType.MULTI_PART: EntityVariableType.MULTI_PART, } return type_mapping.get(openapi_type, EntityVariableType.STRING) # Default to STRING type diff --git a/cozeloop/internal/prompt/openapi.py b/cozeloop/internal/prompt/openapi.py index 7335d4b..5bfef9a 100644 --- a/cozeloop/internal/prompt/openapi.py +++ b/cozeloop/internal/prompt/openapi.py @@ -41,6 +41,7 @@ class VariableType(str, Enum): ARRAY_INTEGER = "array" ARRAY_FLOAT = "array" ARRAY_OBJECT = "array" + MULTI_PART = "multi_part" class ToolChoiceType(str, Enum): @@ -48,9 +49,20 @@ class ToolChoiceType(str, Enum): NONE = "none" +class ContentType(str, Enum): + TEXT = "text" + MULTI_PART_VARIABLE = "multi_part_variable" + + +class ContentPart(BaseModel): + type: ContentType + text: Optional[str] = None + + class Message(BaseModel): role: Role content: Optional[str] = None + parts: Optional[List[ContentPart]] = None class VariableDef(BaseModel): diff --git a/cozeloop/internal/prompt/prompt.py b/cozeloop/internal/prompt/prompt.py index fd04232..815ef9a 100644 --- a/cozeloop/internal/prompt/prompt.py +++ b/cozeloop/internal/prompt/prompt.py @@ -10,7 +10,7 @@ from cozeloop.spec.tracespec import PROMPT_KEY, INPUT, PROMPT_VERSION, V_SCENE_PROMPT_TEMPLATE, V_SCENE_PROMPT_HUB from cozeloop.entities.prompt import (Prompt, Message, VariableDef, VariableType, TemplateType, Role, - PromptVariable) + PromptVariable, ContentPart, ContentType) from cozeloop.internal import consts from cozeloop.internal.consts.error import RemoteServiceError from cozeloop.internal.httpclient.client import Client @@ -175,6 +175,9 @@ def _validate_variable_values_type(self, variable_defs: List[VariableDef], varia elif var_def.type == VariableType.ARRAY_FLOAT: if not isinstance(val, list) or not all(isinstance(item, float) for item in val): raise ValueError(f"type of variable '{var_def.key}' should be array") + elif var_def.type == VariableType.MULTI_PART: + if not isinstance(val, list) or not all(isinstance(item, ContentPart) for item in val): + raise ValueError(f"type of variable '{var_def.key}' should be multi_part") def _format_normal_messages( self, @@ -204,11 +207,60 @@ def _format_normal_messages( variables ) message.content = rendered_content + # Render parts + if message.parts: + message.parts = self.format_multi_part( + template_type, + message.parts, + variable_def_map, + variables + ) results.append(message) return results + def format_multi_part( + self, + template_type: TemplateType, + parts: List[Optional[ContentPart]], + def_map: Dict[str, VariableDef], + val_map: Dict[str, Any]) -> List[ContentPart]: + formatted_parts: List[ContentPart] = [] + + # Render text + for part in parts: + if part is None: + continue + if part.type == ContentType.TEXT and part.text is not None: + rendered_text = self._render_text_content( + template_type, part.text, def_map, val_map + ) + part.text = rendered_text + + # Render multi-part variable + for part in parts: + if part is None: + continue + if part.type == ContentType.MULTI_PART_VARIABLE and part.text is not None: + multi_part_key = part.text + if multi_part_key in def_map and multi_part_key in val_map: + vardef = def_map[multi_part_key] + value = val_map[multi_part_key] + if vardef is not None and value is not None and vardef.type == VariableType.MULTI_PART: + formatted_parts.extend(value) + else: + formatted_parts.append(part) + + # Filter + filtered: List[ContentPart] = [] + for pt in formatted_parts: + if pt is None: + continue + if pt.text is not None or pt.image_url is not None: + filtered.append(pt) + return filtered + def _format_placeholder_messages( self, messages: List[Message], diff --git a/examples/prompt/multipart/prompt_hub_with_multipart.py b/examples/prompt/multipart/prompt_hub_with_multipart.py new file mode 100644 index 0000000..d9e087c --- /dev/null +++ b/examples/prompt/multipart/prompt_hub_with_multipart.py @@ -0,0 +1,163 @@ +# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates +# SPDX-License-Identifier: MIT + +import json +import os +import time +from typing import List + +import cozeloop +from cozeloop import Message +from cozeloop.entities.prompt import Role, ContentPart, ContentType, ImageURL +from cozeloop.spec.tracespec import CALL_OPTIONS, ModelCallOption, ModelMessage, ModelInput + + +def convert_model_input(messages: List[Message]) -> ModelInput: + model_messages = [] + for message in messages: + model_messages.append(ModelMessage( + role=str(message.role), + content=message.content if message.content is not None else "" + )) + + return ModelInput( + messages=model_messages + ) + + +class LLMRunner: + def __init__(self, client): + self.client = client + + def llm_call(self, input_data): + """ + Simulate an LLM call and set relevant span tags. + """ + span = self.client.start_span("llmCall", "model") + try: + # Assuming llm is processing + # output = ChatOpenAI().invoke(input=input_data) + + # mock resp + time.sleep(1) + output = "I'm a robot. I don't have a specific name. You can give me one." + input_token = 232 + output_token = 1211 + + # set tag key: `input` + span.set_input(convert_model_input(input_data)) + # set tag key: `output` + span.set_output(output) + # set tag key: `model_provider`, e.g., openai, etc. + span.set_model_provider("openai") + # set tag key: `start_time_first_resp` + # Timestamp of the first packet return from LLM, unit: microseconds. + # When `start_time_first_resp` is set, a tag named `latency_first_resp` calculated + # based on the span's StartTime will be added, meaning the latency for the first packet. + span.set_start_time_first_resp(int(time.time() * 1000000)) + # set tag key: `input_tokens`. The amount of input tokens. + # when the `input_tokens` value is set, it will automatically sum with the `output_tokens` to calculate the `tokens` tag. + span.set_input_tokens(input_token) + # set tag key: `output_tokens`. The amount of output tokens. + # when the `output_tokens` value is set, it will automatically sum with the `input_tokens` to calculate the `tokens` tag. + span.set_output_tokens(output_token) + # set tag key: `model_name`, e.g., gpt-4-1106-preview, etc. + span.set_model_name("gpt-4-1106-preview") + span.set_tags({CALL_OPTIONS: ModelCallOption( + temperature=0.5, + top_p=0.5, + top_k=10, + presence_penalty=0.5, + frequency_penalty=0.5, + max_tokens=1024, + )}) + + return None + except Exception as e: + raise e + finally: + span.finish() + + +# If you want to use the multipart variable such as image in prompts, you can refer to the following. +if __name__ == '__main__': + # 1.Create a prompt on the platform + # You can create a Prompt on the platform's Prompt development page (set Prompt Key to 'prompt_hub_demo'), + # add the following messages to the template, and submit a version. + # System: You are a helpful bot, the conversation topic is {{var1}}. + # Placeholder: placeholder1 + # User: My question is {{var2}} + # Placeholder: placeholder2 + + # Set the following environment variables first. + # COZELOOP_WORKSPACE_ID=your workspace id + # COZELOOP_API_TOKEN=your token + # 2.New loop client + client = cozeloop.new_client( + # Set whether to report a trace span when get or format prompt. + # Default value is false. + prompt_trace=True, + workspace_id="7496795200791511052", + api_token="pat_MncpzaGch5UIHuModf3mv7S6IpNkG8uer265shnDPML8MRiG0gJrYoT9izOAOhdd") + + os.environ["x_tt_env"] = "ppe_multipart" + os.environ["x_use_ppe"] = "1" + + # 3. new root span + rootSpan = client.start_span("root_span", "main_span") + + # 4. Get the prompt + # If no specific version is specified, the latest version of the corresponding prompt will be obtained + prompt = client.get_prompt(prompt_key="image1", version="0.0.3") + if prompt is not None: + # Get messages of the prompt + if prompt.prompt_template is not None: + messages = prompt.prompt_template.messages + print( + f"prompt messages: {json.dumps([message.model_dump(exclude_none=True) for message in messages], ensure_ascii=False)}") + # Get llm config of the prompt + if prompt.llm_config is not None: + llm_config = prompt.llm_config + print(f"prompt llm_config: {llm_config.model_dump_json(exclude_none=True)}") + + # 5.Format messages of the prompt + formatted_messages = client.prompt_format(prompt, { + "num": "2", + "count": 10, + "format": { + "image1": { + "city": "bejing", + "street": "123 Main", + }, + "image2": { + "city": "bejing", + "street": "123 Main", + }, + }, + # im1 is a multi-part variable, and the value is a list of ContentPart + "im1": [ + ContentPart(type=ContentType.TEXT, text="图片示例"), + ContentPart(type=ContentType.IMAGE_URL,image_url=ImageURL(url="https://example.com/image.jpg")), + ContentPart(type=ContentType.TEXT), + ContentPart(type=ContentType.IMAGE_URL), + ], + # Placeholder variable type should be Message/List[Message] + "placeholder1": [Message(role=Role.USER, content="Hello!"), + Message(role=Role.ASSISTANT, content="Hello!")] + # Other variables in the prompt template that are not provided with corresponding values will be + # considered as empty values. + }) + print( + f"formatted_messages: {json.dumps([message.model_dump(exclude_none=True) for message in formatted_messages], ensure_ascii=False)}") + + # 6.LLM call + llm_runner = LLMRunner(client) + llm_runner.llm_call(formatted_messages) + + rootSpan.finish() + # 4. (optional) flush or close + # -- force flush, report all traces in the queue + # Warning! In general, this method is not needed to be call, as spans will be automatically reported in batches. + # Note that flush will block and wait for the report to complete, and it may cause frequent reporting, + # affecting performance. + client.flush() From bb47d858a8be801704293f4acf0b1bc3158a9ec9 Mon Sep 17 00:00:00 2001 From: "wangluming.wlm" Date: Wed, 27 Aug 2025 17:08:11 +0800 Subject: [PATCH 2/8] feat-mul-test --- tests/internal/prompt/test_converter.py | 108 +++++++- tests/internal/prompt/test_openapi.py | 59 ++++- tests/internal/prompt/test_prompt.py | 312 +++++++++++++++++++++++- 3 files changed, 471 insertions(+), 8 deletions(-) diff --git a/tests/internal/prompt/test_converter.py b/tests/internal/prompt/test_converter.py index 8d65d6c..a137254 100644 --- a/tests/internal/prompt/test_converter.py +++ b/tests/internal/prompt/test_converter.py @@ -4,7 +4,8 @@ from unittest.mock import MagicMock from cozeloop.entities.prompt import ( - Message as EntityMessage + Message as EntityMessage, + ContentType as EntityContentType ) from cozeloop.entities.prompt import ( Role as EntityRole, @@ -30,7 +31,9 @@ _to_span_prompt_input, _to_span_prompt_output, _to_span_messages, - _to_span_arguments + _to_span_arguments, + _convert_content_type, + to_content_part ) from cozeloop.internal.prompt.openapi import ( Prompt as OpenAPIPrompt, @@ -45,7 +48,9 @@ ToolType as OpenAPIToolType, Role as OpenAPIRole, ToolChoiceType as OpenAPIChoiceType, - TemplateType as OpenAPITemplateType + TemplateType as OpenAPITemplateType, + ContentType as OpenAPIContentType, + ContentPart as OpenAPIContentPart ) @@ -403,4 +408,99 @@ def test_to_span_prompt_output(): # Verify conversion assert len(span_output.prompts) == 1 assert span_output.prompts[0].role == EntityRole.ASSISTANT - assert span_output.prompts[0].content == "Assistant response" \ No newline at end of file + assert span_output.prompts[0].content == "Assistant response" + +def test_convert_content_type(): + # Test conversion of content types + assert _convert_content_type(OpenAPIContentType.TEXT) == EntityContentType.TEXT + assert _convert_content_type(OpenAPIContentType.MULTI_PART_VARIABLE) == EntityContentType.MULTI_PART_VARIABLE + + # Test default case with invalid type + mock_invalid_type = MagicMock() + assert _convert_content_type(mock_invalid_type) == EntityContentType.TEXT + +def test_to_content_part(): + # Test text content part conversion + openapi_text_part = OpenAPIContentPart( + type=OpenAPIContentType.TEXT, + text="Hello world" + ) + + entity_part = to_content_part(openapi_text_part) + assert entity_part.type == EntityContentType.TEXT + assert entity_part.text == "Hello world" + assert entity_part.image_url is None + + # Test multi-part variable content part conversion + openapi_multipart_part = OpenAPIContentPart( + type=OpenAPIContentType.MULTI_PART_VARIABLE, + text="image_variable" + ) + + entity_multipart_part = to_content_part(openapi_multipart_part) + assert entity_multipart_part.type == EntityContentType.MULTI_PART_VARIABLE + assert entity_multipart_part.text == "image_variable" + assert entity_multipart_part.image_url is None + +def test_convert_message_with_parts(): + # Create OpenAPI content parts + text_part = OpenAPIContentPart(type=OpenAPIContentType.TEXT, text="Hello") + multipart_part = OpenAPIContentPart(type=OpenAPIContentType.MULTI_PART_VARIABLE, text="image_var") + + # Create OpenAPI message with parts + openapi_message = OpenAPIMessage( + role=OpenAPIRole.USER, + content="User message", + parts=[text_part, multipart_part] + ) + + # Convert message + entity_message = _convert_message(openapi_message) + + # Verify conversion + assert entity_message.role == EntityRole.USER + assert entity_message.content == "User message" + assert entity_message.parts is not None + assert len(entity_message.parts) == 2 + + # Verify first part (text) + assert entity_message.parts[0].type == EntityContentType.TEXT + assert entity_message.parts[0].text == "Hello" + assert entity_message.parts[0].image_url is None + + # Verify second part (multi-part variable) + assert entity_message.parts[1].type == EntityContentType.MULTI_PART_VARIABLE + assert entity_message.parts[1].text == "image_var" + assert entity_message.parts[1].image_url is None + +def test_convert_message_without_parts(): + # Create OpenAPI message without parts + openapi_message = OpenAPIMessage( + role=OpenAPIRole.ASSISTANT, + content="Assistant response" + ) + + # Convert message + entity_message = _convert_message(openapi_message) + + # Verify conversion + assert entity_message.role == EntityRole.ASSISTANT + assert entity_message.content == "Assistant response" + assert entity_message.parts is None + +def test_convert_variable_type_multi_part(): + # Test MULTI_PART variable type conversion + assert _convert_variable_type(OpenAPIVariableType.MULTI_PART) == EntityVariableType.MULTI_PART + + # Test all existing variable types still work + assert _convert_variable_type(OpenAPIVariableType.STRING) == EntityVariableType.STRING + assert _convert_variable_type(OpenAPIVariableType.PLACEHOLDER) == EntityVariableType.PLACEHOLDER + assert _convert_variable_type(OpenAPIVariableType.BOOLEAN) == EntityVariableType.BOOLEAN + assert _convert_variable_type(OpenAPIVariableType.INTEGER) == EntityVariableType.INTEGER + assert _convert_variable_type(OpenAPIVariableType.FLOAT) == EntityVariableType.FLOAT + assert _convert_variable_type(OpenAPIVariableType.OBJECT) == EntityVariableType.OBJECT + assert _convert_variable_type(OpenAPIVariableType.ARRAY_STRING) == EntityVariableType.ARRAY_STRING + assert _convert_variable_type(OpenAPIVariableType.ARRAY_BOOLEAN) == EntityVariableType.ARRAY_BOOLEAN + assert _convert_variable_type(OpenAPIVariableType.ARRAY_INTEGER) == EntityVariableType.ARRAY_INTEGER + assert _convert_variable_type(OpenAPIVariableType.ARRAY_FLOAT) == EntityVariableType.ARRAY_FLOAT + assert _convert_variable_type(OpenAPIVariableType.ARRAY_OBJECT) == EntityVariableType.ARRAY_OBJECT \ No newline at end of file diff --git a/tests/internal/prompt/test_openapi.py b/tests/internal/prompt/test_openapi.py index 23ba89e..010daf4 100644 --- a/tests/internal/prompt/test_openapi.py +++ b/tests/internal/prompt/test_openapi.py @@ -25,7 +25,9 @@ Tool, ToolCallConfig, LLMConfig, - PromptTemplate + PromptTemplate, + ContentType, + ContentPart ) @@ -225,23 +227,76 @@ def test_enum_values(): # Test VariableType enum values assert VariableType.STRING == "string" assert VariableType.PLACEHOLDER == "placeholder" - + assert VariableType.MULTI_PART == "multi_part" + # Test ToolChoiceType enum values assert ToolChoiceType.AUTO == "auto" assert ToolChoiceType.NONE == "none" + # Test ContentType enum values + assert ContentType.TEXT == "text" + assert ContentType.MULTI_PART_VARIABLE == "multi_part_variable" def test_message_model(): # Test Message model with required fields message = Message(role=Role.USER) assert message.role == Role.USER assert message.content is None + assert message.parts is None + # Test Message model with all fields message = Message(role=Role.SYSTEM, content="System message") assert message.role == Role.SYSTEM assert message.content == "System message" + assert message.parts is None + +def test_message_model_with_parts(): + # Test Message model with parts + text_part = ContentPart(type=ContentType.TEXT, text="Hello") + multipart_part = ContentPart(type=ContentType.MULTI_PART_VARIABLE, text="image_var") + + message = Message( + role=Role.USER, + content="User message", + parts=[text_part, multipart_part] + ) + + assert message.role == Role.USER + assert message.content == "User message" + assert message.parts is not None + assert len(message.parts) == 2 + + # Verify parts + assert message.parts[0].type == ContentType.TEXT + assert message.parts[0].text == "Hello" + assert message.parts[1].type == ContentType.MULTI_PART_VARIABLE + assert message.parts[1].text == "image_var" + + +def test_content_part_model(): + # Test ContentPart model with text type + text_part = ContentPart(type=ContentType.TEXT, text="Hello world") + assert text_part.type == ContentType.TEXT + assert text_part.text == "Hello world" + + # Test ContentPart model with multi-part variable type + multipart_part = ContentPart(type=ContentType.MULTI_PART_VARIABLE, text="variable_name") + assert multipart_part.type == ContentType.MULTI_PART_VARIABLE + assert multipart_part.text == "variable_name" + + # Test ContentPart model with minimal fields + minimal_part = ContentPart(type=ContentType.TEXT) + assert minimal_part.type == ContentType.TEXT + assert minimal_part.text is None + +def test_variable_def_model_multi_part(): + # Test VariableDef model with MULTI_PART type + var_def = VariableDef(key="image_parts", desc="Multi-part content", type=VariableType.MULTI_PART) + assert var_def.key == "image_parts" + assert var_def.desc == "Multi-part content" + assert var_def.type == VariableType.MULTI_PART def test_variable_def_model(): # Test VariableDef model with required fields diff --git a/tests/internal/prompt/test_prompt.py b/tests/internal/prompt/test_prompt.py index b309c9f..479bb5d 100644 --- a/tests/internal/prompt/test_prompt.py +++ b/tests/internal/prompt/test_prompt.py @@ -6,7 +6,8 @@ from unittest.mock import MagicMock, patch, call from cozeloop.entities.prompt import ( - Prompt, Message, VariableDef, VariableType, TemplateType, Role, PromptVariable + Prompt, Message, VariableDef, VariableType, TemplateType, Role, PromptVariable, + ContentType, ContentPart, ImageURL ) from cozeloop.internal import consts from cozeloop.internal.consts.error import RemoteServiceError @@ -1123,4 +1124,311 @@ def test_prompt_format_jinja2_edge_cases(prompt_provider): assert len(result) == 3 assert result[0].content == "" assert result[1].content == " " - assert result[2].content == "Static text" \ No newline at end of file + assert result[2].content == "Static text" + +# ============================================================================= +# multi_part相关测试 +# ============================================================================= +def test_validate_variable_values_type_multi_part_valid(prompt_provider): + """测试有效的 MULTI_PART 类型变量""" + # 创建有效的 ContentPart 列表 + content_parts = [ + ContentPart(type=ContentType.TEXT, text="Hello"), + ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/image.jpg")) + ] + + var_defs = [VariableDef(key="multi_content", desc="Multi-part content", type=VariableType.MULTI_PART)] + variables = {"multi_content": content_parts} + + # 应该不抛出异常 + prompt_provider._validate_variable_values_type(var_defs, variables) + +def test_validate_variable_values_type_multi_part_invalid_not_list(prompt_provider): + """测试无效的 MULTI_PART 类型变量 - 不是列表""" + var_defs = [VariableDef(key="multi_content", desc="Multi-part content", type=VariableType.MULTI_PART)] + variables = {"multi_content": "not a list"} # 字符串而不是列表 + + with pytest.raises(ValueError) as excinfo: + prompt_provider._validate_variable_values_type(var_defs, variables) + + assert "type of variable 'multi_content' should be multi_part" in str(excinfo.value) + +def test_validate_variable_values_type_multi_part_invalid_wrong_element_type(prompt_provider): + """测试无效的 MULTI_PART 类型变量 - 元素类型错误""" + var_defs = [VariableDef(key="multi_content", desc="Multi-part content", type=VariableType.MULTI_PART)] + variables = {"multi_content": ["not a ContentPart", ContentPart(type=ContentType.TEXT, text="Hello")]} + + with pytest.raises(ValueError) as excinfo: + prompt_provider._validate_variable_values_type(var_defs, variables) + + assert "type of variable 'multi_content' should be multi_part" in str(excinfo.value) + +def test_format_multi_part_text_rendering(prompt_provider): + """测试多媒体内容中文本部分的渲染""" + # 创建包含模板变量的 ContentPart 列表 + parts = [ + ContentPart(type=ContentType.TEXT, text="Hello {{name}}!"), + ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/image.jpg")) + ] + + variable_def_map = {"name": VariableDef(key="name", desc="User name", type=VariableType.STRING)} + variables = {"name": "Alice"} + + result = prompt_provider.format_multi_part( + TemplateType.NORMAL, + parts, + variable_def_map, + variables + ) + + # 验证结果 + assert len(result) == 2 + assert result[0].type == ContentType.TEXT + assert result[0].text == "Hello Alice!" + assert result[1].type == ContentType.IMAGE_URL + assert result[1].image_url.url == "https://example.com/image.jpg" + +def test_format_multi_part_variable_expansion(prompt_provider): + """测试多媒体变量的展开""" + # 创建包含多媒体变量的 ContentPart 列表 + parts = [ + ContentPart(type=ContentType.TEXT, text="Introduction:"), + ContentPart(type=ContentType.MULTI_PART_VARIABLE, text="dynamic_content") + ] + + # 创建动态内容 + dynamic_parts = [ + ContentPart(type=ContentType.TEXT, text="Dynamic text"), + ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/dynamic.jpg")) + ] + + variable_def_map = { + "dynamic_content": VariableDef(key="dynamic_content", desc="Dynamic content", type=VariableType.MULTI_PART) + } + variables = {"dynamic_content": dynamic_parts} + + result = prompt_provider.format_multi_part( + TemplateType.NORMAL, + parts, + variable_def_map, + variables + ) + + # 验证结果 + assert len(result) == 3 + assert result[0].type == ContentType.TEXT + assert result[0].text == "Introduction:" + assert result[1].type == ContentType.TEXT + assert result[1].text == "Dynamic text" + assert result[2].type == ContentType.IMAGE_URL + assert result[2].image_url.url == "https://example.com/dynamic.jpg" + +def test_format_multi_part_empty_parts(prompt_provider): + """测试空的多媒体内容列表""" + parts = [] + variable_def_map = {} + variables = {} + + result = prompt_provider.format_multi_part( + TemplateType.NORMAL, + parts, + variable_def_map, + variables + ) + + assert result == [] + +def test_format_multi_part_none_parts(prompt_provider): + """测试包含 None 元素的多媒体内容列表""" + parts = [ + ContentPart(type=ContentType.TEXT, text="Valid part"), + None, + ContentPart(type=ContentType.TEXT, text="Another valid part") + ] + + variable_def_map = {} + variables = {} + + result = prompt_provider.format_multi_part( + TemplateType.NORMAL, + parts, + variable_def_map, + variables + ) + + # 应该过滤掉 None 元素 + assert len(result) == 2 + assert result[0].text == "Valid part" + assert result[1].text == "Another valid part" + +def test_format_multi_part_filter_empty_content(prompt_provider): + """测试过滤空内容的多媒体部分""" + parts = [ + ContentPart(type=ContentType.TEXT, text="Valid text"), + ContentPart(type=ContentType.TEXT, text=None), # 空文本 + ContentPart(type=ContentType.IMAGE_URL, image_url=None), # 空图片 + ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/image.jpg")) + ] + + variable_def_map = {} + variables = {} + + result = prompt_provider.format_multi_part( + TemplateType.NORMAL, + parts, + variable_def_map, + variables + ) + + # 应该过滤掉空内容的部分 + assert len(result) == 2 + assert result[0].text == "Valid text" + assert result[1].image_url.url == "https://example.com/image.jpg" + +def test_format_multi_part_jinja2_template(prompt_provider): + """测试使用 Jinja2 模板的多媒体内容渲染""" + parts = [ + ContentPart(type=ContentType.TEXT, text="Hello {{ name }}! You have {{ count }} messages."), + ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/avatar.jpg")) + ] + + variable_def_map = { + "name": VariableDef(key="name", desc="User name", type=VariableType.STRING), + "count": VariableDef(key="count", desc="Message count", type=VariableType.INTEGER) + } + variables = {"name": "Bob", "count": 3} + + result = prompt_provider.format_multi_part( + TemplateType.JINJA2, + parts, + variable_def_map, + variables + ) + + # 验证结果 + assert len(result) == 2 + assert result[0].type == ContentType.TEXT + assert result[0].text == "Hello Bob! You have 3 messages." + assert result[1].type == ContentType.IMAGE_URL + assert result[1].image_url.url == "https://example.com/avatar.jpg" + +def test_format_normal_messages_with_parts(prompt_provider): + """测试包含 parts 字段的消息格式化""" + # 创建包含 parts 的消息 + parts = [ + ContentPart(type=ContentType.TEXT, text="Hello {{name}}!"), + ContentPart(type=ContentType.MULTI_PART_VARIABLE, text="images") + ] + + message = Message(role=Role.USER, content="User message with {{greeting}}", parts=parts) + + # 创建变量定义和值 + var_defs = [ + VariableDef(key="name", desc="User name", type=VariableType.STRING), + VariableDef(key="greeting", desc="Greeting", type=VariableType.STRING), + VariableDef(key="images", desc="Image parts", type=VariableType.MULTI_PART) + ] + + image_parts = [ + ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/photo.jpg")) + ] + + variables = { + "name": "Alice", + "greeting": "Welcome", + "images": image_parts + } + + # 调用方法 + result = prompt_provider._format_normal_messages( + TemplateType.NORMAL, + [message], + var_defs, + variables + ) + + # 验证结果 + assert len(result) == 1 + assert result[0].role == Role.USER + assert result[0].content == "User message with Welcome" + assert result[0].parts is not None + assert len(result[0].parts) == 2 + + # 验证第一个部分(文本) + assert result[0].parts[0].type == ContentType.TEXT + assert result[0].parts[0].text == "Hello Alice!" + + # 验证第二个部分(展开的图片) + assert result[0].parts[1].type == ContentType.IMAGE_URL + assert result[0].parts[1].image_url.url == "https://example.com/photo.jpg" + +def test_prompt_format_with_multi_part_integration(prompt_provider): + """测试多媒体内容的完整集成""" + # 创建包含多媒体内容的消息 + system_parts = [ + ContentPart(type=ContentType.TEXT, text="You are analyzing images for {{task_type}}."), + ] + + user_parts = [ + ContentPart(type=ContentType.TEXT, text="Please analyze this image:"), + ContentPart(type=ContentType.MULTI_PART_VARIABLE, text="user_images") + ] + + system_message = Message(role=Role.SYSTEM, parts=system_parts) + user_message = Message(role=Role.USER, parts=user_parts) + + # 创建变量定义 + var_defs = [ + VariableDef(key="task_type", desc="Task type", type=VariableType.STRING), + VariableDef(key="user_images", desc="User images", type=VariableType.MULTI_PART) + ] + + # 创建 prompt 模板 + prompt_template = MagicMock() + prompt_template.template_type = TemplateType.NORMAL + prompt_template.messages = [system_message, user_message] + prompt_template.variable_defs = var_defs + + prompt = MagicMock(spec=Prompt) + prompt.prompt_template = prompt_template + + # 创建用户图片内容 + user_image_parts = [ + ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/user_photo.jpg")), + ContentPart(type=ContentType.TEXT, text="What do you see in this image?") + ] + + variables = { + "task_type": "object detection", + "user_images": user_image_parts + } + + # 调用方法 + result = prompt_provider.prompt_format(prompt, variables) + + # 验证结果 + assert len(result) == 2 + + # 验证系统消息 + assert result[0].role == Role.SYSTEM + assert result[0].parts is not None + assert len(result[0].parts) == 1 + assert result[0].parts[0].type == ContentType.TEXT + assert result[0].parts[0].text == "You are analyzing images for object detection." + + # 验证用户消息 + assert result[1].role == Role.USER + assert result[1].parts is not None + assert len(result[1].parts) == 3 + + # 第一个部分是原始文本 + assert result[1].parts[0].type == ContentType.TEXT + assert result[1].parts[0].text == "Please analyze this image:" + + # 第二个部分是展开的图片 + assert result[1].parts[1].type == ContentType.IMAGE_URL + assert result[1].parts[1].image_url.url == "https://example.com/user_photo.jpg" + + # 第三个部分是展开的文本 + assert result[1].parts[2].type == ContentType.TEXT + assert result[1].parts[2].text == "What do you see in this image?" \ No newline at end of file From 2678d5d941205af5ef93db1418ebac9f246d4d79 Mon Sep 17 00:00:00 2001 From: "wangluming.wlm" Date: Tue, 2 Sep 2025 15:49:45 +0800 Subject: [PATCH 3/8] feat-mul --- cozeloop/internal/prompt/converter.py | 54 ++++++++++++++++--- cozeloop/spec/tracespec/model.py | 1 + cozeloop/spec/tracespec/prompt.py | 10 +++- .../multipart/prompt_hub_with_multipart.py | 53 ++++++++++-------- 4 files changed, 90 insertions(+), 28 deletions(-) diff --git a/cozeloop/internal/prompt/converter.py b/cozeloop/internal/prompt/converter.py index 1e6b9de..8d0b49f 100644 --- a/cozeloop/internal/prompt/converter.py +++ b/cozeloop/internal/prompt/converter.py @@ -3,7 +3,8 @@ from typing import List, Dict -from cozeloop.spec.tracespec import PromptInput, PromptOutput, ModelMessage, PromptArgument +from cozeloop.spec.tracespec import PromptInput, PromptOutput, ModelMessage, PromptArgument, ModelMessagePart, \ + ModelMessagePartType, ModelImageURL, PromptArgumentValueType from cozeloop.entities.prompt import ( Prompt as EntityPrompt, Message as EntityMessage, @@ -197,15 +198,56 @@ def _to_span_messages(messages: List[EntityMessage]) -> List[ModelMessage]: return [ ModelMessage( role=msg.role, - content=msg.content + content=msg.content, + parts=[_to_span_content_part(part) for part in msg.parts] if msg.parts else None ) for msg in messages ] def _to_span_arguments(arguments: Dict[str, PromptVariable]) -> List[PromptArgument]: return [ - PromptArgument( - key=key, - value=value - ) for key, value in arguments.items() + to_span_argument(key, value) for key, value in arguments.items() ] + + +def to_span_argument(key: str, value: any) -> PromptArgument: + converted_value = str(value) + value_type = PromptArgumentValueType.TEXT + # 判断是否是多模态变量 + if isinstance(value, list) and all(isinstance(part, EntityContentPart) for part in value): + value_type = PromptArgumentValueType.MODEL_MESSAGE_PART + converted_value = [_to_span_content_part(part) for part in value] + + # 判断是否是placeholder变量 + if isinstance(value, list) and all(isinstance(part, EntityMessage) for part in value): + value_type = PromptArgumentValueType.MODEL_MESSAGE + converted_value = _to_span_messages(value) + + return PromptArgument( + key=key, + value=converted_value, + value_type=value_type, + source="input" + ) + + +def _to_span_content_type(entity_type: EntityContentType) -> ModelMessagePartType: + span_content_type_mapping = { + EntityContentType.TEXT: ModelMessagePartType.TEXT, + EntityContentType.IMAGE_URL: ModelMessagePartType.IMAGE, + EntityContentType.MULTI_PART_VARIABLE: ModelMessagePartType.MULTI_PART_VARIABLE, + } + return span_content_type_mapping.get(entity_type, ModelMessagePartType.TEXT) + + +def _to_span_content_part(entity_part: EntityContentPart) -> ModelMessagePart: + image_url = None + if entity_part.image_url is not None: + image_url = ModelImageURL( + url=entity_part.image_url.url + ) + return ModelMessagePart( + type=_to_span_content_type(entity_part.type), + text=entity_part.text, + image_url=image_url, + ) diff --git a/cozeloop/spec/tracespec/model.py b/cozeloop/spec/tracespec/model.py index 8318fb8..24bb4f4 100644 --- a/cozeloop/spec/tracespec/model.py +++ b/cozeloop/spec/tracespec/model.py @@ -46,6 +46,7 @@ class ModelMessagePartType(str, Enum): TEXT = "text" IMAGE = "image_url" FILE = "file_url" + MULTI_PART_VARIABLE = "multi_part_variable" # Internal use only, unless you fully comprehend its functionality and risks class ModelMessagePart(BaseModel): diff --git a/cozeloop/spec/tracespec/prompt.py b/cozeloop/spec/tracespec/prompt.py index 59a32e6..2425107 100644 --- a/cozeloop/spec/tracespec/prompt.py +++ b/cozeloop/spec/tracespec/prompt.py @@ -1,6 +1,6 @@ # Copyright (c) 2025 Bytedance Ltd. and/or its affiliates # SPDX-License-Identifier: MIT - +from enum import Enum from typing import List, Optional, Any from pydantic import BaseModel @@ -13,9 +13,17 @@ class PromptInput(BaseModel): arguments: Optional[List['PromptArgument']] = None +class PromptArgumentValueType(str, Enum): + TEXT = "text" + MODEL_MESSAGE = "model_message" + MODEL_MESSAGE_PART = "model_message_part" + + class PromptArgument(BaseModel): key: str = "" value: Optional[Any] = None + source: Optional[str] = None + value_type: PromptArgumentValueType class PromptOutput(BaseModel): diff --git a/examples/prompt/multipart/prompt_hub_with_multipart.py b/examples/prompt/multipart/prompt_hub_with_multipart.py index d9e087c..b218c46 100644 --- a/examples/prompt/multipart/prompt_hub_with_multipart.py +++ b/examples/prompt/multipart/prompt_hub_with_multipart.py @@ -2,14 +2,35 @@ # SPDX-License-Identifier: MIT import json -import os import time from typing import List import cozeloop from cozeloop import Message from cozeloop.entities.prompt import Role, ContentPart, ContentType, ImageURL -from cozeloop.spec.tracespec import CALL_OPTIONS, ModelCallOption, ModelMessage, ModelInput +from cozeloop.spec.tracespec import CALL_OPTIONS, ModelCallOption, ModelMessage, ModelInput, ModelMessagePartType, ModelMessagePart, ModelImageURL + + +def _to_span_content_type(entity_type: ContentType) -> ModelMessagePartType: + span_content_type_mapping = { + ContentType.TEXT: ModelMessagePartType.TEXT, + ContentType.IMAGE_URL: ModelMessagePartType.IMAGE, + ContentType.MULTI_PART_VARIABLE: ModelMessagePartType.MULTI_PART_VARIABLE, + } + return span_content_type_mapping.get(entity_type, ModelMessagePartType.TEXT) + + +def _to_span_content_part(entity_part: ContentPart) -> ModelMessagePart: + image_url = None + if entity_part.image_url is not None: + image_url = ModelImageURL( + url=entity_part.image_url.url + ) + return ModelMessagePart( + type=_to_span_content_type(entity_part.type), + text=entity_part.text, + image_url=image_url, + ) def convert_model_input(messages: List[Message]) -> ModelInput: @@ -17,7 +38,8 @@ def convert_model_input(messages: List[Message]) -> ModelInput: for message in messages: model_messages.append(ModelMessage( role=str(message.role), - content=message.content if message.content is not None else "" + content=message.content if message.content is not None else "", + parts=[_to_span_content_part(part) for part in message.parts] if message.parts else None )) return ModelInput( @@ -45,7 +67,7 @@ def llm_call(self, input_data): output_token = 1211 # set tag key: `input` - span.set_input(convert_model_input(input_data)) + span.set_input(convert_model_input(input_data).model_dump_json(exclude_none=True)) # set tag key: `output` span.set_output(output) # set tag key: `model_provider`, e.g., openai, etc. @@ -100,15 +122,12 @@ def llm_call(self, input_data): workspace_id="7496795200791511052", api_token="pat_MncpzaGch5UIHuModf3mv7S6IpNkG8uer265shnDPML8MRiG0gJrYoT9izOAOhdd") - os.environ["x_tt_env"] = "ppe_multipart" - os.environ["x_use_ppe"] = "1" - # 3. new root span rootSpan = client.start_span("root_span", "main_span") # 4. Get the prompt # If no specific version is specified, the latest version of the corresponding prompt will be obtained - prompt = client.get_prompt(prompt_key="image1", version="0.0.3") + prompt = client.get_prompt(prompt_key="image1", version="0.0.4") if prompt is not None: # Get messages of the prompt if prompt.prompt_template is not None: @@ -124,26 +143,17 @@ def llm_call(self, input_data): formatted_messages = client.prompt_format(prompt, { "num": "2", "count": 10, - "format": { - "image1": { - "city": "bejing", - "street": "123 Main", - }, - "image2": { - "city": "bejing", - "street": "123 Main", - }, - }, # im1 is a multi-part variable, and the value is a list of ContentPart "im1": [ ContentPart(type=ContentType.TEXT, text="图片示例"), - ContentPart(type=ContentType.IMAGE_URL,image_url=ImageURL(url="https://example.com/image.jpg")), + ContentPart(type=ContentType.IMAGE_URL, + image_url=ImageURL(url="https://dummyimage.com/600x400/4CAF50/fff&text=")), ContentPart(type=ContentType.TEXT), ContentPart(type=ContentType.IMAGE_URL), ], # Placeholder variable type should be Message/List[Message] - "placeholder1": [Message(role=Role.USER, content="Hello!"), - Message(role=Role.ASSISTANT, content="Hello!")] + "P1": [Message(role=Role.USER, content="Hello!"), + Message(role=Role.ASSISTANT, content="Hello!")] # Other variables in the prompt template that are not provided with corresponding values will be # considered as empty values. }) @@ -161,3 +171,4 @@ def llm_call(self, input_data): # Note that flush will block and wait for the report to complete, and it may cause frequent reporting, # affecting performance. client.flush() + time.sleep(2) From e0739c5bcc5fe650e91c5e79f1bdeb53773b83ee Mon Sep 17 00:00:00 2001 From: "wangluming.wlm" Date: Tue, 2 Sep 2025 16:09:53 +0800 Subject: [PATCH 4/8] feat-mul --- examples/prompt/multipart/prompt_hub_with_multipart.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/examples/prompt/multipart/prompt_hub_with_multipart.py b/examples/prompt/multipart/prompt_hub_with_multipart.py index b218c46..138cabc 100644 --- a/examples/prompt/multipart/prompt_hub_with_multipart.py +++ b/examples/prompt/multipart/prompt_hub_with_multipart.py @@ -37,7 +37,7 @@ def convert_model_input(messages: List[Message]) -> ModelInput: model_messages = [] for message in messages: model_messages.append(ModelMessage( - role=str(message.role), + role=message.role, content=message.content if message.content is not None else "", parts=[_to_span_content_part(part) for part in message.parts] if message.parts else None )) @@ -151,9 +151,6 @@ def llm_call(self, input_data): ContentPart(type=ContentType.TEXT), ContentPart(type=ContentType.IMAGE_URL), ], - # Placeholder variable type should be Message/List[Message] - "P1": [Message(role=Role.USER, content="Hello!"), - Message(role=Role.ASSISTANT, content="Hello!")] # Other variables in the prompt template that are not provided with corresponding values will be # considered as empty values. }) @@ -170,5 +167,4 @@ def llm_call(self, input_data): # Warning! In general, this method is not needed to be call, as spans will be automatically reported in batches. # Note that flush will block and wait for the report to complete, and it may cause frequent reporting, # affecting performance. - client.flush() - time.sleep(2) + client.flush() \ No newline at end of file From 2b454aac17da78762034a11615a326a7edf96015 Mon Sep 17 00:00:00 2001 From: "wangluming.wlm" Date: Tue, 2 Sep 2025 16:12:04 +0800 Subject: [PATCH 5/8] feat-mul --- examples/prompt/multipart/prompt_hub_with_multipart.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/examples/prompt/multipart/prompt_hub_with_multipart.py b/examples/prompt/multipart/prompt_hub_with_multipart.py index 138cabc..602c446 100644 --- a/examples/prompt/multipart/prompt_hub_with_multipart.py +++ b/examples/prompt/multipart/prompt_hub_with_multipart.py @@ -118,16 +118,14 @@ def llm_call(self, input_data): client = cozeloop.new_client( # Set whether to report a trace span when get or format prompt. # Default value is false. - prompt_trace=True, - workspace_id="7496795200791511052", - api_token="pat_MncpzaGch5UIHuModf3mv7S6IpNkG8uer265shnDPML8MRiG0gJrYoT9izOAOhdd") + prompt_trace=True) # 3. new root span rootSpan = client.start_span("root_span", "main_span") # 4. Get the prompt # If no specific version is specified, the latest version of the corresponding prompt will be obtained - prompt = client.get_prompt(prompt_key="image1", version="0.0.4") + prompt = client.get_prompt(prompt_key="prompt_hub_demo", version="0.0.1") if prompt is not None: # Get messages of the prompt if prompt.prompt_template is not None: @@ -147,9 +145,7 @@ def llm_call(self, input_data): "im1": [ ContentPart(type=ContentType.TEXT, text="图片示例"), ContentPart(type=ContentType.IMAGE_URL, - image_url=ImageURL(url="https://dummyimage.com/600x400/4CAF50/fff&text=")), - ContentPart(type=ContentType.TEXT), - ContentPart(type=ContentType.IMAGE_URL), + image_url=ImageURL(url="https://example.com")) ], # Other variables in the prompt template that are not provided with corresponding values will be # considered as empty values. From 8be26af697ff6db9d05eaabe3727e67c5b5553ec Mon Sep 17 00:00:00 2001 From: "wangluming.wlm" Date: Tue, 2 Sep 2025 17:56:22 +0800 Subject: [PATCH 6/8] feat-mul --- cozeloop/entities/prompt.py | 6 +--- cozeloop/internal/prompt/converter.py | 2 +- .../multipart/prompt_hub_with_multipart.py | 7 ++--- tests/internal/prompt/test_prompt.py | 28 +++++++++---------- 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/cozeloop/entities/prompt.py b/cozeloop/entities/prompt.py index 72a7ca9..157f82a 100644 --- a/cozeloop/entities/prompt.py +++ b/cozeloop/entities/prompt.py @@ -50,14 +50,10 @@ class ContentType(str, Enum): MULTI_PART_VARIABLE = "multi_part_variable" -class ImageURL(BaseModel): - url: str - - class ContentPart(BaseModel): type: ContentType text: Optional[str] = None - image_url: Optional[ImageURL] = None + image_url: Optional[str] = None class Message(BaseModel): diff --git a/cozeloop/internal/prompt/converter.py b/cozeloop/internal/prompt/converter.py index 8d0b49f..29bb38a 100644 --- a/cozeloop/internal/prompt/converter.py +++ b/cozeloop/internal/prompt/converter.py @@ -244,7 +244,7 @@ def _to_span_content_part(entity_part: EntityContentPart) -> ModelMessagePart: image_url = None if entity_part.image_url is not None: image_url = ModelImageURL( - url=entity_part.image_url.url + url=entity_part.image_url ) return ModelMessagePart( type=_to_span_content_type(entity_part.type), diff --git a/examples/prompt/multipart/prompt_hub_with_multipart.py b/examples/prompt/multipart/prompt_hub_with_multipart.py index 602c446..a15faf8 100644 --- a/examples/prompt/multipart/prompt_hub_with_multipart.py +++ b/examples/prompt/multipart/prompt_hub_with_multipart.py @@ -7,7 +7,7 @@ import cozeloop from cozeloop import Message -from cozeloop.entities.prompt import Role, ContentPart, ContentType, ImageURL +from cozeloop.entities.prompt import Role, ContentPart, ContentType from cozeloop.spec.tracespec import CALL_OPTIONS, ModelCallOption, ModelMessage, ModelInput, ModelMessagePartType, ModelMessagePart, ModelImageURL @@ -24,7 +24,7 @@ def _to_span_content_part(entity_part: ContentPart) -> ModelMessagePart: image_url = None if entity_part.image_url is not None: image_url = ModelImageURL( - url=entity_part.image_url.url + url=entity_part.image_url ) return ModelMessagePart( type=_to_span_content_type(entity_part.type), @@ -144,8 +144,7 @@ def llm_call(self, input_data): # im1 is a multi-part variable, and the value is a list of ContentPart "im1": [ ContentPart(type=ContentType.TEXT, text="图片示例"), - ContentPart(type=ContentType.IMAGE_URL, - image_url=ImageURL(url="https://example.com")) + ContentPart(type=ContentType.IMAGE_URL, image_url="https://dummyimage.com/600x400/4CAF50/fff&text="), ], # Other variables in the prompt template that are not provided with corresponding values will be # considered as empty values. diff --git a/tests/internal/prompt/test_prompt.py b/tests/internal/prompt/test_prompt.py index 479bb5d..3262e76 100644 --- a/tests/internal/prompt/test_prompt.py +++ b/tests/internal/prompt/test_prompt.py @@ -7,7 +7,7 @@ from cozeloop.entities.prompt import ( Prompt, Message, VariableDef, VariableType, TemplateType, Role, PromptVariable, - ContentType, ContentPart, ImageURL + ContentType, ContentPart ) from cozeloop.internal import consts from cozeloop.internal.consts.error import RemoteServiceError @@ -1134,7 +1134,7 @@ def test_validate_variable_values_type_multi_part_valid(prompt_provider): # 创建有效的 ContentPart 列表 content_parts = [ ContentPart(type=ContentType.TEXT, text="Hello"), - ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/image.jpg")) + ContentPart(type=ContentType.IMAGE_URL, image_url="https://example.com/image.jpg") ] var_defs = [VariableDef(key="multi_content", desc="Multi-part content", type=VariableType.MULTI_PART)] @@ -1168,7 +1168,7 @@ def test_format_multi_part_text_rendering(prompt_provider): # 创建包含模板变量的 ContentPart 列表 parts = [ ContentPart(type=ContentType.TEXT, text="Hello {{name}}!"), - ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/image.jpg")) + ContentPart(type=ContentType.IMAGE_URL, image_url="https://example.com/image.jpg") ] variable_def_map = {"name": VariableDef(key="name", desc="User name", type=VariableType.STRING)} @@ -1186,7 +1186,7 @@ def test_format_multi_part_text_rendering(prompt_provider): assert result[0].type == ContentType.TEXT assert result[0].text == "Hello Alice!" assert result[1].type == ContentType.IMAGE_URL - assert result[1].image_url.url == "https://example.com/image.jpg" + assert result[1].image_url == "https://example.com/image.jpg" def test_format_multi_part_variable_expansion(prompt_provider): """测试多媒体变量的展开""" @@ -1199,7 +1199,7 @@ def test_format_multi_part_variable_expansion(prompt_provider): # 创建动态内容 dynamic_parts = [ ContentPart(type=ContentType.TEXT, text="Dynamic text"), - ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/dynamic.jpg")) + ContentPart(type=ContentType.IMAGE_URL, image_url="https://example.com/dynamic.jpg") ] variable_def_map = { @@ -1221,7 +1221,7 @@ def test_format_multi_part_variable_expansion(prompt_provider): assert result[1].type == ContentType.TEXT assert result[1].text == "Dynamic text" assert result[2].type == ContentType.IMAGE_URL - assert result[2].image_url.url == "https://example.com/dynamic.jpg" + assert result[2].image_url == "https://example.com/dynamic.jpg" def test_format_multi_part_empty_parts(prompt_provider): """测试空的多媒体内容列表""" @@ -1267,7 +1267,7 @@ def test_format_multi_part_filter_empty_content(prompt_provider): ContentPart(type=ContentType.TEXT, text="Valid text"), ContentPart(type=ContentType.TEXT, text=None), # 空文本 ContentPart(type=ContentType.IMAGE_URL, image_url=None), # 空图片 - ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/image.jpg")) + ContentPart(type=ContentType.IMAGE_URL, image_url="https://example.com/image.jpg") ] variable_def_map = {} @@ -1283,13 +1283,13 @@ def test_format_multi_part_filter_empty_content(prompt_provider): # 应该过滤掉空内容的部分 assert len(result) == 2 assert result[0].text == "Valid text" - assert result[1].image_url.url == "https://example.com/image.jpg" + assert result[1].image_url == "https://example.com/image.jpg" def test_format_multi_part_jinja2_template(prompt_provider): """测试使用 Jinja2 模板的多媒体内容渲染""" parts = [ ContentPart(type=ContentType.TEXT, text="Hello {{ name }}! You have {{ count }} messages."), - ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/avatar.jpg")) + ContentPart(type=ContentType.IMAGE_URL, image_url="https://example.com/avatar.jpg") ] variable_def_map = { @@ -1310,7 +1310,7 @@ def test_format_multi_part_jinja2_template(prompt_provider): assert result[0].type == ContentType.TEXT assert result[0].text == "Hello Bob! You have 3 messages." assert result[1].type == ContentType.IMAGE_URL - assert result[1].image_url.url == "https://example.com/avatar.jpg" + assert result[1].image_url == "https://example.com/avatar.jpg" def test_format_normal_messages_with_parts(prompt_provider): """测试包含 parts 字段的消息格式化""" @@ -1330,7 +1330,7 @@ def test_format_normal_messages_with_parts(prompt_provider): ] image_parts = [ - ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/photo.jpg")) + ContentPart(type=ContentType.IMAGE_URL, image_url="https://example.com/photo.jpg") ] variables = { @@ -1360,7 +1360,7 @@ def test_format_normal_messages_with_parts(prompt_provider): # 验证第二个部分(展开的图片) assert result[0].parts[1].type == ContentType.IMAGE_URL - assert result[0].parts[1].image_url.url == "https://example.com/photo.jpg" + assert result[0].parts[1].image_url == "https://example.com/photo.jpg" def test_prompt_format_with_multi_part_integration(prompt_provider): """测试多媒体内容的完整集成""" @@ -1394,7 +1394,7 @@ def test_prompt_format_with_multi_part_integration(prompt_provider): # 创建用户图片内容 user_image_parts = [ - ContentPart(type=ContentType.IMAGE_URL, image_url=ImageURL(url="https://example.com/user_photo.jpg")), + ContentPart(type=ContentType.IMAGE_URL, image_url="https://example.com/user_photo.jpg"), ContentPart(type=ContentType.TEXT, text="What do you see in this image?") ] @@ -1427,7 +1427,7 @@ def test_prompt_format_with_multi_part_integration(prompt_provider): # 第二个部分是展开的图片 assert result[1].parts[1].type == ContentType.IMAGE_URL - assert result[1].parts[1].image_url.url == "https://example.com/user_photo.jpg" + assert result[1].parts[1].image_url == "https://example.com/user_photo.jpg" # 第三个部分是展开的文本 assert result[1].parts[2].type == ContentType.TEXT From dfc2c68e9dfb517759661b885538f8babe7b0b95 Mon Sep 17 00:00:00 2001 From: "wangluming.wlm" Date: Tue, 2 Sep 2025 17:57:02 +0800 Subject: [PATCH 7/8] feat-mul --- examples/prompt/multipart/prompt_hub_with_multipart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prompt/multipart/prompt_hub_with_multipart.py b/examples/prompt/multipart/prompt_hub_with_multipart.py index a15faf8..0176b7c 100644 --- a/examples/prompt/multipart/prompt_hub_with_multipart.py +++ b/examples/prompt/multipart/prompt_hub_with_multipart.py @@ -144,7 +144,7 @@ def llm_call(self, input_data): # im1 is a multi-part variable, and the value is a list of ContentPart "im1": [ ContentPart(type=ContentType.TEXT, text="图片示例"), - ContentPart(type=ContentType.IMAGE_URL, image_url="https://dummyimage.com/600x400/4CAF50/fff&text="), + ContentPart(type=ContentType.IMAGE_URL, image_url="https://example.com"), ], # Other variables in the prompt template that are not provided with corresponding values will be # considered as empty values. From 3b33efb52c55fc8100a6da80bcac617e54db1b42 Mon Sep 17 00:00:00 2001 From: "wangluming.wlm" Date: Tue, 2 Sep 2025 19:42:36 +0800 Subject: [PATCH 8/8] feat-mul --- .../prompt/{multipart => prompt_hub}/prompt_hub_with_multipart.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/prompt/{multipart => prompt_hub}/prompt_hub_with_multipart.py (100%) diff --git a/examples/prompt/multipart/prompt_hub_with_multipart.py b/examples/prompt/prompt_hub/prompt_hub_with_multipart.py similarity index 100% rename from examples/prompt/multipart/prompt_hub_with_multipart.py rename to examples/prompt/prompt_hub/prompt_hub_with_multipart.py