From b18b032910c850e3a76f382831109098fb661ec8 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 24 Feb 2025 08:22:02 -0300 Subject: [PATCH 1/2] Add test to document OpenAI o1-mini behavior on system role --- ...openai_o1_mini_system_role[developer].yaml | 59 ++++++++++++++++++ ...st_openai_o1_mini_system_role[system].yaml | 61 +++++++++++++++++++ tests/models/test_openai.py | 15 ++++- 3 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml create mode 100644 tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml diff --git a/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml new file mode 100644 index 000000000..e862014e0 --- /dev/null +++ b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml @@ -0,0 +1,59 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '149' + content-type: + - application/json + cookie: + - __cf_bm=gZdCHBez2ZSET2NhA0P.khJG3GSDXc8LxMJD_qzM4J4-1740396022-1.0.1.1-cpVvQG0NAtKdehZUA1k5aziuqbZp2XuX3Hb3Alqa5xYsfzbH1Mqf6aFnc_eHLx8WC6HUo.ctmcovkowxw.1EIw; + _cfuvid=Dq6WjkA5aZURhaqUk0jDbfZ.oWhKVGE.tds5YlgAzCE-1740396022666-0.0.1.1-604800000 + host: + - api.openai.com + method: POST + parsed_body: + messages: + - content: You are a helpful assistant. + role: developer + - content: Hello + role: user + model: o1-mini + n: 1 + stream: false + uri: https://api.openai.com/v1/chat/completions + response: + headers: + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + connection: + - keep-alive + content-length: + - '224' + content-type: + - application/json + openai-organization: + - pydantic-28gund + openai-processing-ms: + - '17' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + parsed_body: + error: + code: unsupported_value + message: 'Unsupported value: ''messages[0].role'' does not support ''developer'' with this model.' + param: messages[0].role + type: invalid_request_error + status: + code: 400 + message: Bad Request +version: 1 diff --git a/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml new file mode 100644 index 000000000..10b9b2e01 --- /dev/null +++ b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml @@ -0,0 +1,61 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '146' + content-type: + - application/json + host: + - api.openai.com + method: POST + parsed_body: + messages: + - content: You are a helpful assistant. + role: system + - content: Hello + role: user + model: o1-mini + n: 1 + stream: false + uri: https://api.openai.com/v1/chat/completions + response: + headers: + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + connection: + - keep-alive + content-length: + - '221' + content-type: + - application/json + openai-organization: + - pydantic-28gund + openai-processing-ms: + - '17' + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=gZdCHBez2ZSET2NhA0P.khJG3GSDXc8LxMJD_qzM4J4-1740396022-1.0.1.1-cpVvQG0NAtKdehZUA1k5aziuqbZp2XuX3Hb3Alqa5xYsfzbH1Mqf6aFnc_eHLx8WC6HUo.ctmcovkowxw.1EIw; + path=/; expires=Mon, 24-Feb-25 11:50:22 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - _cfuvid=Dq6WjkA5aZURhaqUk0jDbfZ.oWhKVGE.tds5YlgAzCE-1740396022666-0.0.1.1-604800000; path=/; domain=.api.openai.com; + HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + parsed_body: + error: + code: unsupported_value + message: 'Unsupported value: ''messages[0].role'' does not support ''system'' with this model.' + param: messages[0].role + type: invalid_request_error + status: + code: 400 + message: Bad Request +version: 1 diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 45f288cad..4c0b7e0ab 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -29,7 +29,7 @@ from .mock_async_stream import MockAsyncStream with try_import() as imports_successful: - from openai import NOT_GIVEN, AsyncOpenAI, OpenAIError + from openai import NOT_GIVEN, AsyncOpenAI, BadRequestError, OpenAIError from openai.types import chat from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import ( @@ -560,6 +560,19 @@ async def test_system_prompt_role( ] +@pytest.mark.parametrize('system_prompt_role', ['system', 'developer']) +@pytest.mark.vcr +async def test_openai_o1_mini_system_role( + allow_model_requests: None, + system_prompt_role: Literal['system', 'developer'], +) -> None: + model = OpenAIModel('o1-mini', system_prompt_role=system_prompt_role) + agent = Agent(model=model, system_prompt='You are a helpful assistant.') + + with pytest.raises(BadRequestError, match=r".*Unsupported value: 'messages\[0\]\.role' does not support.*"): + await agent.run('Hello') + + @pytest.mark.parametrize('parallel_tool_calls', [True, False]) async def test_parallel_tool_calls(allow_model_requests: None, parallel_tool_calls: bool) -> None: c = completion_message( From 90d81416c8eb52dce69757a2451a84f6fdda9a2c Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 24 Feb 2025 08:26:24 -0300 Subject: [PATCH 2/2] Add OpenAI key --- tests/conftest.py | 5 +++++ .../test_openai_o1_mini_system_role[developer].yaml | 6 +++--- .../test_openai_o1_mini_system_role[system].yaml | 8 ++++---- tests/models/test_openai.py | 3 ++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1aab6ac76..c02260bf9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -197,3 +197,8 @@ def vcr_config(): 'filter_headers': ['authorization', 'x-api-key'], 'decode_compressed_response': True, } + + +@pytest.fixture(scope='session') +def openai_key() -> str: + return os.getenv('OPENAI_API_KEY', 'mock-api-key') diff --git a/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml index e862014e0..a1af485b0 100644 --- a/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml +++ b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml @@ -12,8 +12,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=gZdCHBez2ZSET2NhA0P.khJG3GSDXc8LxMJD_qzM4J4-1740396022-1.0.1.1-cpVvQG0NAtKdehZUA1k5aziuqbZp2XuX3Hb3Alqa5xYsfzbH1Mqf6aFnc_eHLx8WC6HUo.ctmcovkowxw.1EIw; - _cfuvid=Dq6WjkA5aZURhaqUk0jDbfZ.oWhKVGE.tds5YlgAzCE-1740396022666-0.0.1.1-604800000 + - __cf_bm=k1xoKY2.Jtgshh4aAfJbgzfCta3eg2_pKpOubx52AqY-1740396355-1.0.1.1-piAnJhOpL2bGY4paQ8VURT3k0nvStOyXvtsja2kOYPgRmSNpE1vUf87UU.AnB_di_8lc63rtIkZjSJSSZPPgDQ; + _cfuvid=9Q7wbAVWuVxqmo8CZYy4FZvphfjrBGzqwRIt0TKETsE-1740396355005-0.0.1.1-604800000 host: - api.openai.com method: POST @@ -42,7 +42,7 @@ interactions: openai-organization: - pydantic-28gund openai-processing-ms: - - '17' + - '23' openai-version: - '2020-10-01' strict-transport-security: diff --git a/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml index 10b9b2e01..3a85b4cd1 100644 --- a/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml +++ b/tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml @@ -39,13 +39,13 @@ interactions: openai-organization: - pydantic-28gund openai-processing-ms: - - '17' + - '15' openai-version: - '2020-10-01' set-cookie: - - __cf_bm=gZdCHBez2ZSET2NhA0P.khJG3GSDXc8LxMJD_qzM4J4-1740396022-1.0.1.1-cpVvQG0NAtKdehZUA1k5aziuqbZp2XuX3Hb3Alqa5xYsfzbH1Mqf6aFnc_eHLx8WC6HUo.ctmcovkowxw.1EIw; - path=/; expires=Mon, 24-Feb-25 11:50:22 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=Dq6WjkA5aZURhaqUk0jDbfZ.oWhKVGE.tds5YlgAzCE-1740396022666-0.0.1.1-604800000; path=/; domain=.api.openai.com; + - __cf_bm=k1xoKY2.Jtgshh4aAfJbgzfCta3eg2_pKpOubx52AqY-1740396355-1.0.1.1-piAnJhOpL2bGY4paQ8VURT3k0nvStOyXvtsja2kOYPgRmSNpE1vUf87UU.AnB_di_8lc63rtIkZjSJSSZPPgDQ; + path=/; expires=Mon, 24-Feb-25 11:55:55 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - _cfuvid=9Q7wbAVWuVxqmo8CZYy4FZvphfjrBGzqwRIt0TKETsE-1740396355005-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None strict-transport-security: - max-age=31536000; includeSubDomains; preload diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 4c0b7e0ab..889d00605 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -565,8 +565,9 @@ async def test_system_prompt_role( async def test_openai_o1_mini_system_role( allow_model_requests: None, system_prompt_role: Literal['system', 'developer'], + openai_key: str, ) -> None: - model = OpenAIModel('o1-mini', system_prompt_role=system_prompt_role) + model = OpenAIModel('o1-mini', api_key=openai_key, system_prompt_role=system_prompt_role) agent = Agent(model=model, system_prompt='You are a helpful assistant.') with pytest.raises(BadRequestError, match=r".*Unsupported value: 'messages\[0\]\.role' does not support.*"):