Skip to content

Commit

Permalink
Self feedback Improvement (Significant-Gravitas#3680)
Browse files Browse the repository at this point in the history
* Improved `Self-Feedback`

* minor tweak

* Test: Updated `test_get_self_feedback.py`
  • Loading branch information
AbTrax authored May 5, 2023
1 parent f2bef76 commit e12438d
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 12 deletions.
15 changes: 7 additions & 8 deletions autogpt/agent/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ def start_interaction_loop(self):
)

logger.info(
"Enter 'y' to authorise command, 'y -N' to run N continuous commands, 's' to run self-feedback commands"
"Enter 'y' to authorise command, 'y -N' to run N continuous commands, 's' to run self-feedback commands or "
"'n' to exit program, or enter feedback for "
f"{self.ai_name}..."
)
Expand Down Expand Up @@ -190,10 +190,8 @@ def start_interaction_loop(self):
Fore.YELLOW,
"",
)
if self_feedback_resp[0].lower().strip() == cfg.authorise_key:
user_input = "GENERATE NEXT COMMAND JSON"
else:
user_input = self_feedback_resp
user_input = self_feedback_resp
command_name = "self_feedback"
break
elif console_input.lower().strip() == "":
logger.warn("Invalid input format.")
Expand Down Expand Up @@ -244,6 +242,8 @@ def start_interaction_loop(self):
)
elif command_name == "human_feedback":
result = f"Human feedback: {user_input}"
elif command_name == "self_feedback":
result = f"Self feedback: {user_input}"
else:
for plugin in cfg.plugins:
if not plugin.can_handle_pre_command():
Expand Down Expand Up @@ -314,12 +314,11 @@ def get_self_feedback(self, thoughts: dict, llm_model: str) -> str:
"""
ai_role = self.config.ai_role

feedback_prompt = f"Below is a message from an AI agent with the role of {ai_role}. Please review the provided Thought, Reasoning, Plan, and Criticism. If these elements accurately contribute to the successful execution of the assumed role, respond with the letter 'Y' followed by a space, and then explain why it is effective. If the provided information is not suitable for achieving the role's objectives, please provide one or more sentences addressing the issue and suggesting a resolution."
feedback_prompt = f"Below is a message from me, an AI Agent, assuming the role of {ai_role}. whilst keeping knowledge of my slight limitations as an AI Agent Please evaluate my thought process, reasoning, and plan, and provide a concise paragraph outlining potential improvements. Consider adding or removing ideas that do not align with my role and explaining why, prioritizing thoughts based on their significance, or simply refining my overall thought process."
reasoning = thoughts.get("reasoning", "")
plan = thoughts.get("plan", "")
thought = thoughts.get("thoughts", "")
criticism = thoughts.get("criticism", "")
feedback_thoughts = thought + reasoning + plan + criticism
feedback_thoughts = thought + reasoning + plan
return create_chat_completion(
[{"role": "user", "content": feedback_prompt + feedback_thoughts}],
llm_model,
Expand Down
14 changes: 10 additions & 4 deletions tests/unit/test_get_self_feedback.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ def test_get_self_feedback(mocker):
"reasoning": "Sample reasoning.",
"plan": "Sample plan.",
"thoughts": "Sample thoughts.",
"criticism": "Sample criticism.",
}

# Define a fake response for the create_chat_completion function
fake_response = (
"Y The provided information is suitable for achieving the role's objectives."
"The AI Agent has demonstrated a reasonable thought process, but there is room for improvement. "
"For example, the reasoning could be elaborated to better justify the plan, and the plan itself "
"could be more detailed to ensure its effectiveness. In addition, the AI Agent should focus more "
"on its core role and prioritize thoughts that align with that role."
)

# Mock the create_chat_completion function
Expand All @@ -36,5 +38,9 @@ def test_get_self_feedback(mocker):
"gpt-3.5-turbo",
)

# Check if the response is correct
assert feedback == fake_response
# Check if the response is a non-empty string
assert isinstance(feedback, str) and len(feedback) > 0

# Check if certain keywords from input thoughts are present in the feedback response
for keyword in ["reasoning", "plan", "thoughts"]:
assert keyword in feedback

0 comments on commit e12438d

Please sign in to comment.