From 4cd2603a3609bbb3c729c8b2cad0b24634f454ed Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 24 Feb 2025 11:49:13 -0300 Subject: [PATCH] Add test to document OpenAI o1-mini behavior on system role (#977) --- tests/conftest.py | 5 ++ ...openai_o1_mini_system_role[developer].yaml | 59 ++++++++++++++++++ ...st_openai_o1_mini_system_role[system].yaml | 61 +++++++++++++++++++ tests/models/test_openai.py | 16 ++++- 4 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml create mode 100644 tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml diff --git a/tests/conftest.py b/tests/conftest.py index 1aab6ac76..c02260bf9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -197,3 +197,8 @@ def vcr_config(): 'filter_headers': ['authorization', 'x-api-key'], 'decode_compressed_response': True, } + + +@pytest.fixture(scope='session') +def openai_key() -> str: + return os.getenv('OPENAI_API_KEY', 'mock-api-key') diff --git a/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml new file mode 100644 index 000000000..a1af485b0 --- /dev/null +++ b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml @@ -0,0 +1,59 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '149' + content-type: + - application/json + cookie: + - __cf_bm=k1xoKY2.Jtgshh4aAfJbgzfCta3eg2_pKpOubx52AqY-1740396355-1.0.1.1-piAnJhOpL2bGY4paQ8VURT3k0nvStOyXvtsja2kOYPgRmSNpE1vUf87UU.AnB_di_8lc63rtIkZjSJSSZPPgDQ; + _cfuvid=9Q7wbAVWuVxqmo8CZYy4FZvphfjrBGzqwRIt0TKETsE-1740396355005-0.0.1.1-604800000 + host: + - api.openai.com + method: POST + parsed_body: + messages: + - content: You are a helpful assistant. + role: developer + - content: Hello + role: user + model: o1-mini + n: 1 + stream: false + uri: https://api.openai.com/v1/chat/completions + response: + headers: + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + connection: + - keep-alive + content-length: + - '224' + content-type: + - application/json + openai-organization: + - pydantic-28gund + openai-processing-ms: + - '23' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + parsed_body: + error: + code: unsupported_value + message: 'Unsupported value: ''messages[0].role'' does not support ''developer'' with this model.' + param: messages[0].role + type: invalid_request_error + status: + code: 400 + message: Bad Request +version: 1 diff --git a/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml new file mode 100644 index 000000000..3a85b4cd1 --- /dev/null +++ b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml @@ -0,0 +1,61 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '146' + content-type: + - application/json + host: + - api.openai.com + method: POST + parsed_body: + messages: + - content: You are a helpful assistant. + role: system + - content: Hello + role: user + model: o1-mini + n: 1 + stream: false + uri: https://api.openai.com/v1/chat/completions + response: + headers: + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + connection: + - keep-alive + content-length: + - '221' + content-type: + - application/json + openai-organization: + - pydantic-28gund + openai-processing-ms: + - '15' + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=k1xoKY2.Jtgshh4aAfJbgzfCta3eg2_pKpOubx52AqY-1740396355-1.0.1.1-piAnJhOpL2bGY4paQ8VURT3k0nvStOyXvtsja2kOYPgRmSNpE1vUf87UU.AnB_di_8lc63rtIkZjSJSSZPPgDQ; + path=/; expires=Mon, 24-Feb-25 11:55:55 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - _cfuvid=9Q7wbAVWuVxqmo8CZYy4FZvphfjrBGzqwRIt0TKETsE-1740396355005-0.0.1.1-604800000; path=/; domain=.api.openai.com; + HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + parsed_body: + error: + code: unsupported_value + message: 'Unsupported value: ''messages[0].role'' does not support ''system'' with this model.' + param: messages[0].role + type: invalid_request_error + status: + code: 400 + message: Bad Request +version: 1 diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 45f288cad..889d00605 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -29,7 +29,7 @@ from .mock_async_stream import MockAsyncStream with try_import() as imports_successful: - from openai import NOT_GIVEN, AsyncOpenAI, OpenAIError + from openai import NOT_GIVEN, AsyncOpenAI, BadRequestError, OpenAIError from openai.types import chat from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import ( @@ -560,6 +560,20 @@ async def test_system_prompt_role( ] +@pytest.mark.parametrize('system_prompt_role', ['system', 'developer']) +@pytest.mark.vcr +async def test_openai_o1_mini_system_role( + allow_model_requests: None, + system_prompt_role: Literal['system', 'developer'], + openai_key: str, +) -> None: + model = OpenAIModel('o1-mini', api_key=openai_key, system_prompt_role=system_prompt_role) + agent = Agent(model=model, system_prompt='You are a helpful assistant.') + + with pytest.raises(BadRequestError, match=r".*Unsupported value: 'messages\[0\]\.role' does not support.*"): + await agent.run('Hello') + + @pytest.mark.parametrize('parallel_tool_calls', [True, False]) async def test_parallel_tool_calls(allow_model_requests: None, parallel_tool_calls: bool) -> None: c = completion_message(