Skip to content

Fix tests for Azure OpenAI models, add GPT-4.1-nano #286

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
amanjaiswal73892 merged 3 commits into from
Aug 27, 2025
Merged

Fix tests for Azure OpenAI models, add GPT-4.1-nano #286

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
88d37a5
fix tests for openai azure models after changes. add gpt-4.1-nano
patricebechard Aug 27, 2025
85b7e1d
update test models to 4.1 nano
patricebechard Aug 27, 2025
1ed6fce
formatting
patricebechard Aug 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/agentlab/agents/generic_agent/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,10 @@
AGENT_AZURE_4o_MINI_VISION,
AGENT_AZURE_41,
AGENT_AZURE_41_MINI,
AGENT_AZURE_41_NANO,
AGENT_AZURE_41_VISION,
AGENT_AZURE_41_MINI_VISION,
AGENT_AZURE_41_NANO_VISION,
AGENT_AZURE_5,
AGENT_AZURE_5_MINI,
AGENT_AZURE_5_NANO,
Expand Down Expand Up @@ -66,8 +68,10 @@
"AGENT_AZURE_4o_MINI_VISION",
"AGENT_AZURE_41",
"AGENT_AZURE_41_MINI",
"AGENT_AZURE_41_NANO",
"AGENT_AZURE_41_VISION",
"AGENT_AZURE_41_MINI_VISION",
"AGENT_AZURE_41_NANO_VISION",
"AGENT_AZURE_5",
"AGENT_AZURE_5_MINI",
"AGENT_AZURE_5_NANO",
Expand Down
8 changes: 8 additions & 0 deletions src/agentlab/agents/generic_agent/agent_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,10 @@
chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-mini-2025-04-14"],
flags=FLAGS_GPT_4o,
)
AGENT_AZURE_41_NANO = GenericAgentArgs(
chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-nano-2025-04-14"],
flags=FLAGS_GPT_4o,
)

AGENT_AZURE_5 = GenericAgentArgs(
chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-5-2025-08-07"],
Expand Down Expand Up @@ -350,6 +354,10 @@
chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-mini-2025-04-14"],
flags=FLAGS_GPT_4o_VISION,
)
AGENT_AZURE_41_NANO_VISION = GenericAgentArgs(
chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-nano-2025-04-14"],
flags=FLAGS_GPT_4o_VISION,
)

AGENT_AZURE_5_VISION = GenericAgentArgs(
chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-5-2025-08-07"],
Expand Down
19 changes: 17 additions & 2 deletions src/agentlab/llm/chat_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ def make_model(self):
class AzureModelArgs(BaseModelArgs):
"""Serializable object for instantiating a generic chat model with an Azure model."""

deployment_name: str = (
None # NOTE: deployment_name is deprecated for Azure OpenAI and won't be used.
)

def make_model(self):
return AzureChatModel(
model_name=self.model_name,
Expand Down Expand Up @@ -396,13 +400,25 @@ def __init__(
model_name,
api_key=None,
temperature=0.5,
deployment_name=None,
max_tokens=100,
max_retry=4,
min_retry_wait_time=60,
log_probs=False,
):
api_key = api_key or os.getenv("AZURE_OPENAI_API_KEY")
assert (
api_key
), "AZURE_OPENAI_API_KEY has to be defined in the environment when using AzureChatModel"
endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
assert endpoint, "AZURE_OPENAI_ENDPOINT has to be defined in the environment"
assert (
endpoint
), "AZURE_OPENAI_ENDPOINT has to be defined in the environment when using AzureChatModel"

if deployment_name is not None:
logging.info(
f"Deployment name is deprecated for Azure OpenAI and won't be used. Using model name: {model_name}."
)
Comment on lines +418 to +421
Copy link

@korbit-ai korbit-ai bot Aug 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Incorrect Log Level for Deprecation Warning category Logging

Tell me more
What is the issue?

Using INFO level for a deprecation warning is incorrect. Deprecation warnings should use WARNING level to ensure visibility.

Why this matters

Deprecation messages may be missed if logging is set to WARNING or higher, making it harder to identify and update deprecated code usage.

Suggested change ∙ Feature Preview
if deployment_name is not None:
    logging.warning(
        f"Deployment name is deprecated for Azure OpenAI and won't be used. Using model name: {model_name}."
    )
Provide feedback to improve future suggestions

Nice Catch Incorrect Not in Scope Not in coding standard Other

💬 Looking for more details? Reply to this comment to chat with Korbit.


client_args = {
"base_url": endpoint,
Expand All @@ -411,7 +427,6 @@ def __init__(
super().__init__(
model_name=model_name,
api_key=api_key,
api_key_env_var="AZURE_OPENAI_API_KEY",
temperature=temperature,
max_tokens=max_tokens,
max_retry=max_retry,
Expand Down
14 changes: 14 additions & 0 deletions src/agentlab/llm/llm_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@
temperature=1, # gpt-5 supports temperature of 1 only
vision_support=True,
),
"openai/gpt-4.1-nano-2025-04-14": OpenAIModelArgs(
model_name="gpt-4.1-nano-2025-04-14",
max_total_tokens=128_000,
max_input_tokens=128_000,
max_new_tokens=16_384,
vision_support=True,
),
"openai/gpt-4.1-mini-2025-04-14": OpenAIModelArgs(
model_name="gpt-4.1-mini-2025-04-14",
max_total_tokens=128_000,
Expand Down Expand Up @@ -160,6 +167,13 @@
max_new_tokens=16_384,
vision_support=True,
),
"azure/gpt-4.1-nano-2025-04-14": AzureModelArgs(
model_name="gpt-4.1-nano",
max_total_tokens=128_000,
max_input_tokens=128_000,
max_new_tokens=16_384,
vision_support=True,
),
Comment on lines +170 to +176
Copy link

@korbit-ai korbit-ai bot Aug 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Token Limit Configuration Inconsistency category Functionality

Tell me more
What is the issue?

The max_input_tokens plus max_new_tokens exceeds max_total_tokens. The sum (144,384) is greater than the maximum allowed (128,000).

Why this matters

This configuration could lead to runtime errors when the model attempts to generate responses that approach the token limits, as the combined input and output tokens cannot exceed max_total_tokens.

Suggested change ∙ Feature Preview

Adjust the token limits to ensure max_input_tokens + max_new_tokens <= max_total_tokens. A possible fix:

"azure/gpt-4.1-nano-2025-04-14": AzureModelArgs(
    model_name="gpt-4.1-nano",
    max_total_tokens=128_000,
    max_input_tokens=111_616,  # 128_000 - 16_384
    max_new_tokens=16_384,
    vision_support=True,
),
Provide feedback to improve future suggestions

Nice Catch Incorrect Not in Scope Not in coding standard Other

💬 Looking for more details? Reply to this comment to chat with Korbit.

"azure/gpt-5-2025-08-07": AzureModelArgs(
model_name="gpt-5",
max_total_tokens=400_000,
Expand Down
11 changes: 9 additions & 2 deletions tests/llm/test_chat_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,13 @@

@pytest.mark.pricy
@pytest.mark.skipif(skip_tests, reason="Skipping on remote as Azure is pricy")
@pytest.mark.skipif(
not os.getenv("AZURE_OPENAI_API_KEY"), reason="Skipping as Azure API key not set"
)
def test_api_model_args_azure():
model_args = AzureModelArgs(
model_name="gpt-35-turbo",
deployment_name="gpt-35-turbo",
model_name="gpt-4.1-nano",
deployment_name="gpt-4.1-nano",
max_total_tokens=8192,
max_input_tokens=8192 - 512,
max_new_tokens=512,
Expand All @@ -43,6 +46,7 @@ def test_api_model_args_azure():

@pytest.mark.pricy
@pytest.mark.skipif(skip_tests, reason="Skipping on remote as Azure is pricy")
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="Skipping as OpenAI API key not set")
def test_api_model_args_openai():
model_args = OpenAIModelArgs(
model_name="gpt-4o-mini",
Expand All @@ -64,6 +68,9 @@ def test_api_model_args_openai():

@pytest.mark.pricy
@pytest.mark.skipif(skip_tests, reason="Skipping on remote as Anthropic is pricy")
@pytest.mark.skipif(
not os.getenv("ANTHROPIC_API_KEY"), reason="Skipping as Anthropic API key not set"
)
def test_api_model_args_anthropic():
model_args = AnthropicModelArgs(
model_name="claude-3-haiku-20240307",
Expand Down
4 changes: 3 additions & 1 deletion tests/llm/test_litellm_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os
from functools import partial

import pytest

from agentlab.llm.litellm_api import LiteLLMModelArgs
from agentlab.llm.response_api import APIPayload, LLMOutput

Expand Down Expand Up @@ -106,6 +106,7 @@ def test_multi_action_tool_calls():


@pytest.mark.pricy
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="Skipping as OpenAI API key not set")
def test_single_tool_call():
"""
Test that the LLMOutput contains only one tool call when use_only_first_toolcall is True.
Expand Down Expand Up @@ -137,6 +138,7 @@ def test_single_tool_call():


@pytest.mark.pricy
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="Skipping as OpenAI API key not set")
def test_force_tool_call():
"""
Test that the model can produce a specific tool call when requested.
Expand Down
1 change: 1 addition & 0 deletions tests/llm/test_response_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,7 @@ def test_claude_model_with_multiple_messages_pricy_call():

## Test multiaction
@pytest.mark.pricy
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="Skipping as OpenAI API key not set")
def test_multi_action_tool_calls():
"""
Test that the model can produce multiple tool calls in parallel.
Expand Down
2 changes: 1 addition & 1 deletion tests/llm/test_tracking.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ def test_openai_chat_model():
not AZURE_OPENAI_API_KEY_AVAILABLE, reason="Azure OpenAI API key is not available"
)
def test_azure_chat_model():
chat_model = AzureChatModel(model_name="gpt-35-turbo", deployment_name="gpt-35-turbo")
chat_model = AzureChatModel(model_name="gpt-4.1-nano", deployment_name="gpt-4.1-nano")
assert chat_model.input_cost > 0
assert chat_model.output_cost > 0

Expand Down
Loading