From d552c058332905b54c191dc904233fb591171e84 Mon Sep 17 00:00:00 2001 From: Davor Runje Date: Tue, 27 Aug 2024 11:07:02 +0000 Subject: [PATCH 1/4] Fixes the issue caused the following error introduced by more strict JSON checking: https://community.openai.com/t/error-the-model-produced-invalid-content/747511 --- autogen/function_utils.py | 1 + test/test_function_utils.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/autogen/function_utils.py b/autogen/function_utils.py index 6b9b6f5b129..a120136d36c 100644 --- a/autogen/function_utils.py +++ b/autogen/function_utils.py @@ -93,6 +93,7 @@ class Parameters(BaseModel): type: Literal["object"] = "object" properties: Dict[str, JsonSchemaValue] required: List[str] + additionalProperties: bool = False class Function(BaseModel): diff --git a/test/test_function_utils.py b/test/test_function_utils.py index 0475044b49f..24b1c1412c8 100644 --- a/test/test_function_utils.py +++ b/test/test_function_utils.py @@ -157,6 +157,7 @@ def f(a: Annotated[str, "Parameter a"], b=1, c: Annotated[float, "Parameter c"] "c": {"type": "number", "description": "Parameter c", "default": 1.0}, }, "required": ["a"], + "additionalProperties": False, } actual = model_dump(get_parameters(required, param_annotations, default_values)) @@ -236,6 +237,7 @@ def test_get_function_schema() -> None: }, }, "required": ["a", "d"], + "additionalProperties": False, }, }, } @@ -264,6 +266,7 @@ def test_get_function_schema() -> None: }, }, "required": ["a", "d"], + "additionalProperties": False, }, }, } @@ -333,6 +336,7 @@ def currency_calculator( # type: ignore[empty-body] }, }, "required": ["base"], + "additionalProperties": False, }, }, } From 7b6566264f76b2cd5e8e3979064e15a14d5a301d Mon Sep 17 00:00:00 2001 From: Davor Runje Date: Tue, 27 Aug 2024 11:27:44 +0000 Subject: [PATCH 2/4] update tests --- test/agentchat/test_conversable_agent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/agentchat/test_conversable_agent.py b/test/agentchat/test_conversable_agent.py index c0d37a7bd7a..eace7af0653 100755 --- a/test/agentchat/test_conversable_agent.py +++ b/test/agentchat/test_conversable_agent.py @@ -689,6 +689,7 @@ def exec_python(cell: Annotated[str, "Valid Python cell to execute."]) -> str: } }, "required": ["cell"], + "additionalProperties": False, }, }, } @@ -722,6 +723,7 @@ async def exec_sh(script: Annotated[str, "Valid shell script to execute."]) -> s } }, "required": ["script"], + "additionalProperties": False, }, }, } @@ -762,6 +764,7 @@ def exec_python(cell: Annotated[str, "Valid Python cell to execute."]) -> str: } }, "required": ["cell"], + "additionalProperties": False, }, } ] @@ -794,6 +797,7 @@ async def exec_sh(script: Annotated[str, "Valid shell script to execute."]) -> s } }, "required": ["script"], + "additionalProperties": False, }, } ] @@ -916,6 +920,7 @@ def exec_python(cell: Annotated[str, "Valid Python cell to execute."]) -> str: } }, "required": ["cell"], + "additionalProperties": False, }, }, } From 31e4db9f98436ba70694625a0221d7a0f38fb97a Mon Sep 17 00:00:00 2001 From: Davor Runje Date: Tue, 27 Aug 2024 16:01:35 +0000 Subject: [PATCH 3/4] fix --- autogen/oai/client.py | 13 +++ test/oai/test_client.py | 170 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) diff --git a/autogen/oai/client.py b/autogen/oai/client.py index 3ae37257b21..21f7d1d2634 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -195,6 +195,16 @@ def message_retrieval( for choice in choices ] + @staticmethod + def _remove_names_from_messages(params: Dict[str, Any]): + def _remove_names(messages: List[Dict[str, Any]]): + def __remove_names(message: Dict[str, Any]): + return {k: v for k, v in message.items() if k != "name"} + + return [__remove_names(message) for message in messages] + + return {k: _remove_names(v) if k == "messages" else v for k, v in params.items()} + def create(self, params: Dict[str, Any]) -> ChatCompletion: """Create a completion for a given config using openai's client. @@ -207,6 +217,9 @@ def create(self, params: Dict[str, Any]) -> ChatCompletion: """ iostream = IOStream.get_default() + # names should not be in the messages (see: https://platform.openai.com/docs/guides/chat-completions) + params = self._remove_names_from_messages(params) + completions: Completions = self._oai_client.chat.completions if "messages" in params else self._oai_client.completions # type: ignore [attr-defined] # If streaming is enabled and has messages, then iterate over the chunks of the response. if params.get("stream", False) and "messages" in params: diff --git a/test/oai/test_client.py b/test/oai/test_client.py index 443ec995de4..3aa15b714b3 100755 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -322,6 +322,176 @@ def test_cache(): assert not os.path.exists(os.path.join(cache_dir, str(LEGACY_DEFAULT_CACHE_SEED))) +@pytest.mark.skipif(skip, reason="openai>=1 not installed") +def test_chat_completion_after_tool_call(): + config_list = config_list_from_json( + env_or_file=OAI_CONFIG_LIST, + file_location=KEY_LOC, + filter_dict={"tags": ["gpt-4o-mini"]}, + ) + client = OpenAIWrapper(config_list=config_list) + params = { + "messages": [ + { + "content": "You are a student writing a practice test. Your " + "task is as follows:\n" + " 1) Retrieve exam questions by calling a " + "function.\n" + " 2) Write a draft of proposed answers and engage " + "in dialogue with your tutor.\n" + " 3) Once you are done with the dialogue, register " + "the final answers by calling a function.\n" + " 4) Retrieve the final grade by calling a " + "function.\n" + "Finally, terminate the chat by saying 'TERMINATE'.", + "role": "system", + }, + {"content": "Prepare for the test about Leonardo da Vinci.", "name": "Teacher_Agent", "role": "user"}, + { + "content": None, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": '{"message":"Please ' + "provide the exam " + "questions for the " + "test about Leonardo " + 'da Vinci."}', + "name": "retrieve_exam_questions", + }, + "id": "call_WtPX2rq3saIIqk4hM4jjAhNY", + "type": "function", + } + ], + }, + { + "content": "1) Mona Lisa 2) Innovations 3) Florence at the time " + "of Leonardo 4) The Last Supper 5) Vit", + "role": "tool", + "tool_call_id": "call_WtPX2rq3saIIqk4hM4jjAhNY", + }, + { + "content": "I have retrieved the exam questions about Leonardo " + "da Vinci. Here they are:\n" + "\n" + "1. Mona Lisa\n" + "2. Innovations\n" + "3. Florence at the time of Leonardo\n" + "4. The Last Supper\n" + "5. Vitruvian Man\n" + "\n" + "Now, I'll draft proposed answers for each question. " + "Let's start with the first one: \n" + "\n" + "1. **Mona Lisa**: The Mona Lisa, painted by " + "Leonardo da Vinci in the early 16th century, is " + "arguably the most famous painting in the world. It " + "is known for the subject's enigmatic expression and " + "da Vinci's masterful use of sfumato, which creates " + "a soft transition between colors and tones.\n" + "\n" + "What do you think of this draft answer? Would you " + "like to add or change anything?", + "name": "Student_Agent", + "role": "assistant", + }, + { + "content": "Your draft answer for the Mona Lisa is " + "well-articulated and captures the essence of the " + "painting and its significance. Here are a few " + "suggestions to enhance your response:\n" + "\n" + "1. **Historical Context**: You could mention that " + "the painting was created between 1503 and 1506 and " + "is housed in the Louvre Museum in Paris.\n" + " \n" + "2. **Artistic Techniques**: You might want to " + "elaborate on the technique of sfumato, explaining " + "how it contributes to the depth and realism of the " + "portrait.\n" + "\n" + "3. **Cultural Impact**: It could be beneficial to " + "touch on the painting's influence on art and " + "culture, as well as its status as a symbol of the " + "Renaissance.\n" + "\n" + "Here's a revised version incorporating these " + "points:\n" + "\n" + "**Revised Answer**: The Mona Lisa, painted by " + "Leonardo da Vinci between 1503 and 1506, is " + "arguably the most famous painting in the world and " + "is housed in the Louvre Museum in Paris. It is " + "renowned for the subject's enigmatic expression, " + "which has intrigued viewers for centuries. Da " + "Vinci's masterful use of sfumato, a technique that " + "allows for a soft transition between colors and " + "tones, adds a remarkable depth and realism to the " + "piece. The Mona Lisa has not only influenced " + "countless artists and movements but has also become " + "a symbol of the Renaissance and a cultural icon in " + "its own right.\n" + "\n" + "Let me know if you would like to move on to the " + "next question or make further adjustments!", + "name": "Teacher_Agent", + "role": "user", + }, + ], + "model": "gpt-4o-mini", + "stream": False, + "temperature": 0.8, + "tools": [ + { + "function": { + "description": "Get exam questions from examiner", + "name": "retrieve_exam_questions", + "parameters": { + "additionalProperties": False, + "properties": {"message": {"description": "Message " "for " "examiner", "type": "string"}}, + "required": ["message"], + "type": "object", + }, + }, + "type": "function", + }, + { + "function": { + "description": "Write a final answers to exam " + "questions to examiner, but only after " + "discussing with the tutor first.", + "name": "write_final_answers", + "parameters": { + "additionalProperties": False, + "properties": {"message": {"description": "Message " "for " "examiner", "type": "string"}}, + "required": ["message"], + "type": "object", + }, + }, + "type": "function", + }, + { + "function": { + "description": "Get the final grade after submitting " "the answers.", + "name": "get_final_grade", + "parameters": { + "additionalProperties": False, + "properties": {"message": {"description": "Message " "for " "examiner", "type": "string"}}, + "required": ["message"], + "type": "object", + }, + }, + "type": "function", + }, + ], + } + + response = client.create(**params) + + assert response is not None + + if __name__ == "__main__": # test_aoai_chat_completion() # test_oai_tool_calling_extraction() From e81a6c6bdced9e50085e8918e6a46bf599d39e6f Mon Sep 17 00:00:00 2001 From: Davor Runje Date: Wed, 28 Aug 2024 05:55:59 +0000 Subject: [PATCH 4/4] removed names from assistant role only --- autogen/oai/client.py | 14 +- test/oai/test_client.py | 316 +++++++++++++++++++++++++--------------- 2 files changed, 204 insertions(+), 126 deletions(-) diff --git a/autogen/oai/client.py b/autogen/oai/client.py index 21f7d1d2634..cfade066937 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -196,14 +196,14 @@ def message_retrieval( ] @staticmethod - def _remove_names_from_messages(params: Dict[str, Any]): - def _remove_names(messages: List[Dict[str, Any]]): - def __remove_names(message: Dict[str, Any]): + def _remove_assistant_names_from_messages(params: Dict[str, Any]): + def _remove_assistant_names(messages: List[Dict[str, Any]]): + def _remove_name(message: Dict[str, Any]): return {k: v for k, v in message.items() if k != "name"} - return [__remove_names(message) for message in messages] + return [_remove_name(m) if m["role"] == "assistant" else m for m in messages] - return {k: _remove_names(v) if k == "messages" else v for k, v in params.items()} + return {k: _remove_assistant_names(v) if k == "messages" else v for k, v in params.items()} def create(self, params: Dict[str, Any]) -> ChatCompletion: """Create a completion for a given config using openai's client. @@ -217,8 +217,8 @@ def create(self, params: Dict[str, Any]) -> ChatCompletion: """ iostream = IOStream.get_default() - # names should not be in the messages (see: https://platform.openai.com/docs/guides/chat-completions) - params = self._remove_names_from_messages(params) + # name parameter in an assistant message causes sometimes an error in the API, although they are legit parameter according to the API reference (https://platform.openai.com/docs/api-reference/chat/create) + params = self._remove_assistant_names_from_messages(params) completions: Completions = self._oai_client.chat.completions if "messages" in params else self._oai_client.completions # type: ignore [attr-defined] # If streaming is enabled and has messages, then iterate over the chunks of the response. diff --git a/test/oai/test_client.py b/test/oai/test_client.py index 3aa15b714b3..e04acb8ae84 100755 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -4,12 +4,13 @@ import shutil import sys import time +from tempfile import TemporaryDirectory import pytest from autogen import OpenAIWrapper, config_list_from_json from autogen.cache.cache import Cache -from autogen.oai.client import LEGACY_CACHE_DIR, LEGACY_DEFAULT_CACHE_SEED +from autogen.oai.client import LEGACY_CACHE_DIR, LEGACY_DEFAULT_CACHE_SEED, OpenAIClient sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from conftest import skip_openai # noqa: E402 @@ -322,42 +323,21 @@ def test_cache(): assert not os.path.exists(os.path.join(cache_dir, str(LEGACY_DEFAULT_CACHE_SEED))) -@pytest.mark.skipif(skip, reason="openai>=1 not installed") -def test_chat_completion_after_tool_call(): - config_list = config_list_from_json( - env_or_file=OAI_CONFIG_LIST, - file_location=KEY_LOC, - filter_dict={"tags": ["gpt-4o-mini"]}, - ) - client = OpenAIWrapper(config_list=config_list) +def test__remove_assistant_names_from_messages(): params = { "messages": [ { - "content": "You are a student writing a practice test. Your " - "task is as follows:\n" - " 1) Retrieve exam questions by calling a " - "function.\n" - " 2) Write a draft of proposed answers and engage " - "in dialogue with your tutor.\n" - " 3) Once you are done with the dialogue, register " - "the final answers by calling a function.\n" - " 4) Retrieve the final grade by calling a " - "function.\n" - "Finally, terminate the chat by saying 'TERMINATE'.", + "content": "system message", "role": "system", }, - {"content": "Prepare for the test about Leonardo da Vinci.", "name": "Teacher_Agent", "role": "user"}, + {"content": "Initial message", "name": "Teacher_Agent", "role": "user"}, { "content": None, "role": "assistant", "tool_calls": [ { "function": { - "arguments": '{"message":"Please ' - "provide the exam " - "questions for the " - "test about Leonardo " - 'da Vinci."}', + "arguments": '{"message":"function parameter"}', "name": "retrieve_exam_questions", }, "id": "call_WtPX2rq3saIIqk4hM4jjAhNY", @@ -366,75 +346,17 @@ def test_chat_completion_after_tool_call(): ], }, { - "content": "1) Mona Lisa 2) Innovations 3) Florence at the time " - "of Leonardo 4) The Last Supper 5) Vit", + "content": "function reply", "role": "tool", "tool_call_id": "call_WtPX2rq3saIIqk4hM4jjAhNY", }, { - "content": "I have retrieved the exam questions about Leonardo " - "da Vinci. Here they are:\n" - "\n" - "1. Mona Lisa\n" - "2. Innovations\n" - "3. Florence at the time of Leonardo\n" - "4. The Last Supper\n" - "5. Vitruvian Man\n" - "\n" - "Now, I'll draft proposed answers for each question. " - "Let's start with the first one: \n" - "\n" - "1. **Mona Lisa**: The Mona Lisa, painted by " - "Leonardo da Vinci in the early 16th century, is " - "arguably the most famous painting in the world. It " - "is known for the subject's enigmatic expression and " - "da Vinci's masterful use of sfumato, which creates " - "a soft transition between colors and tones.\n" - "\n" - "What do you think of this draft answer? Would you " - "like to add or change anything?", - "name": "Student_Agent", + "content": "some message", "role": "assistant", + "name": "Student_Agent", # this is problematic for some reason and we'll remove it }, { - "content": "Your draft answer for the Mona Lisa is " - "well-articulated and captures the essence of the " - "painting and its significance. Here are a few " - "suggestions to enhance your response:\n" - "\n" - "1. **Historical Context**: You could mention that " - "the painting was created between 1503 and 1506 and " - "is housed in the Louvre Museum in Paris.\n" - " \n" - "2. **Artistic Techniques**: You might want to " - "elaborate on the technique of sfumato, explaining " - "how it contributes to the depth and realism of the " - "portrait.\n" - "\n" - "3. **Cultural Impact**: It could be beneficial to " - "touch on the painting's influence on art and " - "culture, as well as its status as a symbol of the " - "Renaissance.\n" - "\n" - "Here's a revised version incorporating these " - "points:\n" - "\n" - "**Revised Answer**: The Mona Lisa, painted by " - "Leonardo da Vinci between 1503 and 1506, is " - "arguably the most famous painting in the world and " - "is housed in the Louvre Museum in Paris. It is " - "renowned for the subject's enigmatic expression, " - "which has intrigued viewers for centuries. Da " - "Vinci's masterful use of sfumato, a technique that " - "allows for a soft transition between colors and " - "tones, adds a remarkable depth and realism to the " - "piece. The Mona Lisa has not only influenced " - "countless artists and movements but has also become " - "a symbol of the Renaissance and a cultural icon in " - "its own right.\n" - "\n" - "Let me know if you would like to move on to the " - "next question or make further adjustments!", + "content": "some another message", "name": "Teacher_Agent", "role": "user", }, @@ -449,35 +371,7 @@ def test_chat_completion_after_tool_call(): "name": "retrieve_exam_questions", "parameters": { "additionalProperties": False, - "properties": {"message": {"description": "Message " "for " "examiner", "type": "string"}}, - "required": ["message"], - "type": "object", - }, - }, - "type": "function", - }, - { - "function": { - "description": "Write a final answers to exam " - "questions to examiner, but only after " - "discussing with the tutor first.", - "name": "write_final_answers", - "parameters": { - "additionalProperties": False, - "properties": {"message": {"description": "Message " "for " "examiner", "type": "string"}}, - "required": ["message"], - "type": "object", - }, - }, - "type": "function", - }, - { - "function": { - "description": "Get the final grade after submitting " "the answers.", - "name": "get_final_grade", - "parameters": { - "additionalProperties": False, - "properties": {"message": {"description": "Message " "for " "examiner", "type": "string"}}, + "properties": {"message": {"description": "Message for examiner", "type": "string"}}, "required": ["message"], "type": "object", }, @@ -486,10 +380,194 @@ def test_chat_completion_after_tool_call(): }, ], } + actual = OpenAIClient._remove_assistant_names_from_messages(params) + + assert [m for m in actual["messages"] if m.get("role") != "assistant"] == [ + m for m in params["messages"] if m.get("role") != "assistant" + ] + assert [m for m in actual["messages"] if m.get("role") == "assistant" and m.get("content") is not None] == [ + {"content": "some message", "role": "assistant"} + ] + + actual.pop("messages") + params.pop("messages") + assert actual == params + + +@pytest.mark.skipif(skip, reason="openai>=1 not installed") +def test_chat_completion_after_tool_call(): + config_list = config_list_from_json( + env_or_file=OAI_CONFIG_LIST, + file_location=KEY_LOC, + filter_dict={"tags": ["gpt-4o-mini"]}, + ) + with TemporaryDirectory() as temp_dir: + with Cache.disk(cache_seed=42, cache_path_root=temp_dir) as cache: + client = OpenAIWrapper(config_list=config_list, cache=cache) + params = { + "messages": [ + { + "content": "You are a student writing a practice test. Your " + "task is as follows:\n" + " 1) Retrieve exam questions by calling a " + "function.\n" + " 2) Write a draft of proposed answers and engage " + "in dialogue with your tutor.\n" + " 3) Once you are done with the dialogue, register " + "the final answers by calling a function.\n" + " 4) Retrieve the final grade by calling a " + "function.\n" + "Finally, terminate the chat by saying 'TERMINATE'.", + "role": "system", + }, + { + "content": "Prepare for the test about Leonardo da Vinci.", + "name": "Teacher_Agent", + "role": "user", + }, + { + "content": None, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": '{"message":"Please ' + "provide the exam " + "questions for the " + "test about Leonardo " + 'da Vinci."}', + "name": "retrieve_exam_questions", + }, + "id": "call_WtPX2rq3saIIqk4hM4jjAhNY", + "type": "function", + } + ], + }, + { + "content": "1) Mona Lisa 2) Innovations 3) Florence at the time " + "of Leonardo 4) The Last Supper 5) Vit", + "role": "tool", + "tool_call_id": "call_WtPX2rq3saIIqk4hM4jjAhNY", + }, + { + "content": "I have retrieved the exam questions about Leonardo " + "da Vinci. Here they are:\n" + "\n" + "1. Mona Lisa\n" + "2. Innovations\n" + "3. Florence at the time of Leonardo\n" + "4. The Last Supper\n" + "5. Vitruvian Man\n" + "\n" + "Now, I'll draft proposed answers for each question. " + "Let's start with the first one: \n" + "\n" + "1. **Mona Lisa**: The Mona Lisa, painted by " + "Leonardo da Vinci in the early 16th century, is " + "arguably the most famous painting in the world. It " + "is known for the subject's enigmatic expression and " + "da Vinci's masterful use of sfumato, which creates " + "a soft transition between colors and tones.\n" + "\n" + # "What do you think of this draft answer? Would you " + "like to add or change anything?", + "role": "assistant", + "name": "Student_Agent", # this is problematic for some reason + }, + { + "content": "Your draft answer for the Mona Lisa is " + "well-articulated and captures the essence of the " + "painting and its significance. Here are a few " + "suggestions to enhance your response:\n" + "\n" + "1. **Historical Context**: You could mention that " + "the painting was created between 1503 and 1506 and " + "is housed in the Louvre Museum in Paris.\n" + " \n" + "2. **Artistic Techniques**: You might want to " + "elaborate on the technique of sfumato, explaining " + "how it contributes to the depth and realism of the " + "portrait.\n" + "\n" + "3. **Cultural Impact**: It could be beneficial to " + "touch on the painting's influence on art and " + "culture, as well as its status as a symbol of the " + "Renaissance.\n" + "\n" + "Here's a revised version incorporating these " + "points:\n" + "\n" + "**Revised Answer**: The Mona Lisa, painted by " + "Leonardo da Vinci between 1503 and 1506, is " + "arguably the most famous painting in the world and " + "is housed in the Louvre Museum in Paris. It is " + "renowned for the subject's enigmatic expression, " + "which has intrigued viewers for centuries. Da " + "Vinci's masterful use of sfumato, a technique that " + "allows for a soft transition between colors and " + "tones, adds a remarkable depth and realism to the " + "piece. The Mona Lisa has not only influenced " + "countless artists and movements but has also become " + "a symbol of the Renaissance and a cultural icon in " + "its own right.\n" + "\n" + "Let me know if you would like to move on to the " + "next question or make further adjustments!", + "name": "Teacher_Agent", + "role": "user", + }, + ], + "model": "gpt-4o-mini", + "stream": False, + "temperature": 0.8, + "tools": [ + { + "function": { + "description": "Get exam questions from examiner", + "name": "retrieve_exam_questions", + "parameters": { + "additionalProperties": False, + "properties": {"message": {"description": "Message for examiner", "type": "string"}}, + "required": ["message"], + "type": "object", + }, + }, + "type": "function", + }, + { + "function": { + "description": "Write a final answers to exam " + "questions to examiner, but only after " + "discussing with the tutor first.", + "name": "write_final_answers", + "parameters": { + "additionalProperties": False, + "properties": {"message": {"description": "Message for examiner", "type": "string"}}, + "required": ["message"], + "type": "object", + }, + }, + "type": "function", + }, + { + "function": { + "description": "Get the final grade after submitting " "the answers.", + "name": "get_final_grade", + "parameters": { + "additionalProperties": False, + "properties": {"message": {"description": "Message for examiner", "type": "string"}}, + "required": ["message"], + "type": "object", + }, + }, + "type": "function", + }, + ], + } - response = client.create(**params) + response = client.create(**params) - assert response is not None + assert response is not None if __name__ == "__main__":