Merge pull request #1024 from Pythagora-io/add-routes-to-user-instruc…

…tions

Add route files content to prompt for user instructions.

core/agents/troubleshooter.py

@@ -7,6 +7,8 @@
 from core.agents.convo import AgentConvo
 from core.agents.mixins import IterationPromptMixin
 from core.agents.response import AgentResponse
+from core.config import ROUTE_FILES_AGENT_NAME
+from core.db.models.file import File
 from core.db.models.project_state import TaskStatus
 from core.llm.parser import JSONParser, OptionalCodeBlockParser
 from core.log import get_logger
@@ -23,6 +25,10 @@ class BugReportQuestions(BaseModel):
     )
 
 
+class RouteFilePaths(BaseModel):
+    files: list[str] = Field(description="List of paths for files that contain routes")
+
+
 class Troubleshooter(IterationPromptMixin, BaseAgent):
     agent_type = "troubleshooter"
     display_name = "Troubleshooter"
@@ -134,8 +140,12 @@ async def get_run_command(self) -> Optional[str]:
     async def get_user_instructions(self) -> Optional[str]:
         await self.send_message("Determining how to test the app ...")
 
+        route_files = await self._get_route_files()
+
         llm = self.get_llm()
-        convo = self._get_task_convo().template("define_user_review_goal", task=self.current_state.current_task)
+        convo = self._get_task_convo().template(
+            "define_user_review_goal", task=self.current_state.current_task, route_files=route_files
+        )
         user_instructions: str = await llm(convo)
 
         user_instructions = user_instructions.strip()
@@ -145,6 +155,21 @@ async def get_user_instructions(self) -> Optional[str]:
 
         return user_instructions
 
+    async def _get_route_files(self) -> list[File]:
+        """Returns the list of file paths that have routes defined in them."""
+
+        llm = self.get_llm(ROUTE_FILES_AGENT_NAME)
+        convo = (
+            self._get_task_convo()
+            .template("get_route_files", task=self.current_state.current_task)
+            .require_schema(RouteFilePaths)
+        )
+        file_list = await llm(convo, parser=JSONParser(RouteFilePaths))
+        route_files: set[str] = set(file_list.files)
+
+        # Sometimes LLM can return a non-existent file, let's make sure to filter those out
+        return [f for f in self.current_state.files if f.path in route_files]
+
     async def get_user_feedback(
         self,
         run_command: str,

core/config/__init__.py

@@ -34,6 +34,7 @@
 # Agents with sane setup in the default configuration
 DEFAULT_AGENT_NAME = "default"
 DESCRIBE_FILES_AGENT_NAME = "CodeMonkey.describe_files"
+ROUTE_FILES_AGENT_NAME = "Troubleshooter.get_route_files"
 
 # Endpoint for the external documentation
 EXTERNAL_DOCUMENTATION_API = "http://docs-pythagora-io-439719575.us-east-1.elb.amazonaws.com"
@@ -298,6 +299,7 @@ class Config(_StrictModel):
         default={
             DEFAULT_AGENT_NAME: AgentLLMConfig(),
             DESCRIBE_FILES_AGENT_NAME: AgentLLMConfig(model="gpt-3.5-turbo", temperature=0.0),
+            ROUTE_FILES_AGENT_NAME: AgentLLMConfig(model="gpt-4o", temperature=0.0),
         }
     )
     prompt: PromptConfig = PromptConfig()

core/prompts/troubleshooter/define_user_review_goal.prompt

@@ -1,3 +1,15 @@
+{% if route_files %}
+Here is a list of files the contain route definitions, and their contents. If any of the steps in testing instructions use URLs, use the routes defined in these files.
+
+---START_OF_FILES---
+{% for file in route_files %}
+File **`{{ file.path }}`** ({{file.content.content.splitlines()|length}} lines of code):
+```
+{{ file.content.content }}```
+
+{% endfor %}
+---END_OF_FILES---
+{% endif %}
 How can a human user test if this task was completed successfully?
 
 Please list actions, step by step, in order, that the user should take to verify the task. After each action, describe what the expected response is.

core/prompts/troubleshooter/get_route_files.prompt

@@ -0,0 +1,10 @@
+{# eval-tool test ID: 74918173-59e4-4005-bf19-d28f3bc9f06c
+
+This was added in June 2024, to improve the accuracy of user testing instructions.
+We've noticed that the LLM would often times include incorrect URLs in the user testing
+instructions, because it wasn't aware of the routes used in the application.
+The solution was to add the entire content of all the files that have routes defined in
+them, and this prompt selects those files.
+
+#}
+Your task is to identify all the files, from the above list, that have routes defined in them. Return just the file paths as a JSON list named "files", DO NOT return anything else. If there are no files with routes, return an empty list.

tests/agents/test_troubleshooter.py

@@ -0,0 +1,31 @@
+import pytest
+
+from core.agents.troubleshooter import RouteFilePaths, Troubleshooter
+from core.db.models import File, FileContent
+
+
+@pytest.mark.asyncio
+async def test_route_files_are_included_in_the_prompt(agentcontext):
+    sm, _, ui, mock_get_llm = agentcontext
+
+    sm.current_state.tasks = [{"description": "Some task", "status": "todo", "instructions": "Testing here!"}]
+    files = [
+        File(path="dir/file1.js", content=FileContent(content="File 1 content")),
+        File(path="dir/file2.js", content=FileContent(content="File 2 content")),
+    ]
+
+    await sm.commit()
+    sm.current_state.files = files
+
+    ts = Troubleshooter(sm, ui)
+    ts.get_llm = mock_get_llm(
+        side_effect=[
+            RouteFilePaths(files=["dir/file1.js"]),  # First LLM call, to select files with routes
+            "",  # Second LLM call, to generate the actual instructions
+        ]
+    )
+    await ts.get_user_instructions()
+    second_llm_call = ts.get_llm().call_args_list[1]
+    user_msg_contents = [msg["content"] for msg in second_llm_call[0][0] if msg["role"] == "user"]
+    assert "File 1 content" in user_msg_contents[1]  # Prompt at [1] has the route file contents
+    assert "File 2 content" not in user_msg_contents[1]