Merge pull request #1008 from Pythagora-io/more-telemetry

More telemetry

core/agents/architect.py

@@ -5,8 +5,11 @@
 from core.agents.base import BaseAgent
 from core.agents.convo import AgentConvo
 from core.agents.response import AgentResponse
+from core.db.models import Specification
 from core.llm.parser import JSONParser
+from core.log import get_logger
 from core.telemetry import telemetry
+from core.templates.example_project import EXAMPLE_PROJECTS
 from core.templates.registry import PROJECT_TEMPLATES, ProjectTemplateEnum
 from core.ui.base import ProjectStage
 
@@ -15,6 +18,8 @@
 WARN_FRAMEWORKS = ["next.js", "vue", "vue.js", "svelte", "angular"]
 WARN_FRAMEWORKS_URL = "https://github.com/Pythagora-io/gpt-pilot/wiki/Using-GPT-Pilot-with-frontend-frameworks"
 
+log = get_logger(__name__)
+
 
 # FIXME: all the reponse pydantic models should be strict (see config._StrictModel), also check if we
 # can disallow adding custom Python attributes to the model
@@ -74,34 +79,34 @@ class Architect(BaseAgent):
     async def run(self) -> AgentResponse:
         await self.ui.send_project_stage(ProjectStage.ARCHITECTURE)
 
-        llm = self.get_llm()
-        convo = AgentConvo(self).template("technologies", templates=PROJECT_TEMPLATES).require_schema(Architecture)
+        spec = self.current_state.specification.clone()
+
+        if spec.example_project:
+            self.prepare_example_project(spec)
+        else:
+            await self.plan_architecture(spec)
+
+        await self.check_system_dependencies(spec)
+
+        self.next_state.specification = spec
+        telemetry.set("template", spec.template)
+        self.next_state.action = ARCHITECTURE_STEP_NAME
+        return AgentResponse.done(self)
 
+    async def plan_architecture(self, spec: Specification):
         await self.send_message("Planning project architecture ...")
+
+        llm = self.get_llm()
+        convo = AgentConvo(self).template("technologies", templates=PROJECT_TEMPLATES).require_schema(Architecture)
         arch: Architecture = await llm(convo, parser=JSONParser(Architecture))
 
         await self.check_compatibility(arch)
-        await self.check_system_dependencies(arch.system_dependencies)
 
-        spec = self.current_state.specification.clone()
         spec.architecture = arch.architecture
         spec.system_dependencies = [d.model_dump() for d in arch.system_dependencies]
         spec.package_dependencies = [d.model_dump() for d in arch.package_dependencies]
         spec.template = arch.template.value if arch.template else None
 
-        self.next_state.specification = spec
-        telemetry.set(
-            "architecture",
-            {
-                "description": spec.architecture,
-                "system_dependencies": spec.system_dependencies,
-                "package_dependencies": spec.package_dependencies,
-            },
-        )
-        telemetry.set("template", spec.template)
-        self.next_state.action = ARCHITECTURE_STEP_NAME
-        return AgentResponse.done(self)
-
     async def check_compatibility(self, arch: Architecture) -> bool:
         warn_system_deps = [dep.name for dep in arch.system_dependencies if dep.name.lower() in WARN_SYSTEM_DEPS]
         warn_package_deps = [dep.name for dep in arch.package_dependencies if dep.name.lower() in WARN_FRAMEWORKS]
@@ -130,18 +135,50 @@ async def check_compatibility(self, arch: Architecture) -> bool:
         # that SpecWriter should catch and allow the user to reword the initial spec.
         return True
 
-    async def check_system_dependencies(self, deps: list[SystemDependency]):
+    def prepare_example_project(self, spec: Specification):
+        log.debug(f"Setting architecture for example project: {spec.example_project}")
+        arch = EXAMPLE_PROJECTS[spec.example_project]["architecture"]
+
+        spec.architecture = arch["architecture"]
+        spec.system_dependencies = arch["system_dependencies"]
+        spec.package_dependencies = arch["package_dependencies"]
+        spec.template = arch["template"]
+        telemetry.set("template", spec.template)
+
+    async def check_system_dependencies(self, spec: Specification):
         """
         Check whether the required system dependencies are installed.
+
+        This also stores the app architecture telemetry data, including the
+        information about whether each system dependency is installed.
+
+        :param spec: Project specification.
         """
+        deps = spec.system_dependencies
 
         for dep in deps:
-            status_code, _, _ = await self.process_manager.run_command(dep.test)
+            status_code, _, _ = await self.process_manager.run_command(dep["test"])
+            dep["installed"] = bool(status_code == 0)
             if status_code != 0:
-                if dep.required_locally:
+                if dep["required_locally"]:
                     remedy = "Please install it before proceeding with your app."
                 else:
                     remedy = "If you would like to use it locally, please install it before proceeding."
-                await self.send_message(f"❌ {dep.name} is not available. {remedy}")
+                await self.send_message(f"❌ {dep['name']} is not available. {remedy}")
+                await self.ask_question(
+                    f"Once you have installed {dep['name']}, please press Continue.",
+                    buttons={"continue": "Continue"},
+                    buttons_only=True,
+                    default="continue",
+                )
             else:
-                await self.send_message(f"✅ {dep.name} is available.")
+                await self.send_message(f"✅ {dep['name']} is available.")
+
+        telemetry.set(
+            "architecture",
+            {
+                "description": spec.architecture,
+                "system_dependencies": deps,
+                "package_dependencies": spec.package_dependencies,
+            },
+        )

core/agents/developer.py

@@ -10,6 +10,7 @@
 from core.db.models.project_state import TaskStatus
 from core.llm.parser import JSONParser
 from core.log import get_logger
+from core.telemetry import telemetry
 
 log = get_logger(__name__)
 
@@ -195,6 +196,14 @@ async def breakdown_current_task(self) -> AgentResponse:
         self.next_state.modified_files = {}
         self.set_next_steps(response, source)
         self.next_state.action = f"Task #{current_task_index + 1} start"
+        await telemetry.trace_code_event(
+            "task-start",
+            {
+                "task_index": current_task_index + 1,
+                "num_tasks": len(self.current_state.tasks),
+                "num_epics": len(self.current_state.epics),
+            },
+        )
         return AgentResponse.done(self)
 
     async def get_relevant_files(

core/agents/external_docs.py

@@ -26,7 +26,7 @@ class SelectedDocsets(BaseModel):
 class ExternalDocumentation(BaseAgent):
     """Agent in charge of collecting and storing additional documentation.
 
-    Docs are per task and are stores in the `tasks` variable in the project state.
+    Docs are per task and are stores in the `docs` variable in the project state.
     This agent ensures documentation is collected only once per task.
 
     Agent does 2 LLM interactions:
@@ -44,7 +44,12 @@ class ExternalDocumentation(BaseAgent):
     display_name = "Documentation"
 
     async def run(self) -> AgentResponse:
-        available_docsets = await self._get_available_docsets()
+        if self.current_state.specification.example_project:
+            log.debug("Example project detected, no documentation selected.")
+            available_docsets = []
+        else:
+            available_docsets = await self._get_available_docsets()
+
         selected_docsets = await self._select_docsets(available_docsets)
         await telemetry.trace_code_event("docsets_used", selected_docsets)
 
@@ -153,6 +158,8 @@ async def _store_docs(self, snippets: list[tuple], available_docsets: list[tuple
         Documentation snippets are stored as a list of dictionaries:
         {"key": docset-key, "desc": documentation-description, "snippets": list-of-snippets}
 
+        :param snippets: List of tuples: (docset_key, snippets)
+        :param available_docsets: List of available docsets from the API.
         """
 
         docsets_dict = dict(available_docsets)
@@ -161,4 +168,3 @@ async def _store_docs(self, snippets: list[tuple], available_docsets: list[tuple
             docs.append({"key": docset_key, "desc": docsets_dict[docset_key], "snippets": snip})
 
         self.next_state.docs = docs
-        self.next_state.flag_tasks_as_modified()

core/agents/orchestrator.py

@@ -187,7 +187,7 @@ def create_agent(self, prev_response: Optional[AgentResponse]) -> BaseAgent:
                 return Importer(self.state_manager, self.ui)
             else:
                 # New project: ask the Spec Writer to refine and save the project specification
-                return SpecWriter(self.state_manager, self.ui)
+                return SpecWriter(self.state_manager, self.ui, process_manager=self.process_manager)
         elif not state.specification.architecture:
             # Ask the Architect to design the project architecture and determine dependencies
             return Architect(self.state_manager, self.ui, process_manager=self.process_manager)

core/agents/spec_writer.py

@@ -3,11 +3,11 @@
 from core.agents.response import AgentResponse
 from core.db.models import Complexity
 from core.llm.parser import StringParser
+from core.log import get_logger
 from core.telemetry import telemetry
 from core.templates.example_project import (
-    EXAMPLE_PROJECT_ARCHITECTURE,
-    EXAMPLE_PROJECT_DESCRIPTION,
-    EXAMPLE_PROJECT_PLAN,
+    DEFAULT_EXAMPLE_PROJECT,
+    EXAMPLE_PROJECTS,
 )
 
 # If the project description is less than this, perform an analysis using LLM
@@ -18,6 +18,8 @@
 )
 SPEC_STEP_NAME = "Create specification"
 
+log = get_logger(__name__)
+
 
 class SpecWriter(BaseAgent):
     agent_type = "spec-writer"
@@ -41,18 +43,25 @@ async def run(self) -> AgentResponse:
             return AgentResponse.import_project(self)
 
         if response.button == "example":
-            await self.send_message("Starting example project with description:")
-            await self.send_message(EXAMPLE_PROJECT_DESCRIPTION)
-            self.prepare_example_project()
+            await self.prepare_example_project(DEFAULT_EXAMPLE_PROJECT)
             return AgentResponse.done(self)
+
         elif response.button == "continue":
             # FIXME: Workaround for the fact that VSCode "continue" button does
             # nothing but repeat the question. We reproduce this bug for bug here.
             return AgentResponse.done(self)
 
-        spec = response.text
+        spec = response.text.strip()
 
         complexity = await self.check_prompt_complexity(spec)
+        await telemetry.trace_code_event(
+            "project-description",
+            {
+                "initial_prompt": spec,
+                "complexity": complexity,
+            },
+        )
+
         if len(spec) < ANALYZE_THRESHOLD and complexity != Complexity.SIMPLE:
             spec = await self.analyze_spec(spec)
             spec = await self.review_spec(spec)
@@ -73,36 +82,21 @@ async def check_prompt_complexity(self, prompt: str) -> str:
         llm_response: str = await llm(convo, temperature=0, parser=StringParser())
         return llm_response.lower()
 
-    def prepare_example_project(self):
+    async def prepare_example_project(self, example_name: str):
+        example_description = EXAMPLE_PROJECTS[example_name]["description"].strip()
+
+        log.debug(f"Starting example project: {example_name}")
+        await self.send_message(f"Starting example project with description:\n\n{example_description}")
+
         spec = self.current_state.specification.clone()
-        spec.description = EXAMPLE_PROJECT_DESCRIPTION
-        spec.architecture = EXAMPLE_PROJECT_ARCHITECTURE["architecture"]
-        spec.system_dependencies = EXAMPLE_PROJECT_ARCHITECTURE["system_dependencies"]
-        spec.package_dependencies = EXAMPLE_PROJECT_ARCHITECTURE["package_dependencies"]
-        spec.template = EXAMPLE_PROJECT_ARCHITECTURE["template"]
-        spec.complexity = Complexity.SIMPLE
-        telemetry.set("initial_prompt", spec.description.strip())
-        telemetry.set("is_complex_app", False)
-        telemetry.set("template", spec.template)
-        telemetry.set(
-            "architecture",
-            {
-                "architecture": spec.architecture,
-                "system_dependencies": spec.system_dependencies,
-                "package_dependencies": spec.package_dependencies,
-            },
-        )
+        spec.example_project = example_name
+        spec.description = example_description
+        spec.complexity = EXAMPLE_PROJECTS[example_name]["complexity"]
         self.next_state.specification = spec
 
-        self.next_state.epics = [
-            {
-                "name": "Initial Project",
-                "description": EXAMPLE_PROJECT_DESCRIPTION,
-                "completed": False,
-                "complexity": Complexity.SIMPLE,
-            }
-        ]
-        self.next_state.tasks = EXAMPLE_PROJECT_PLAN
+        telemetry.set("initial_prompt", spec.description)
+        telemetry.set("example_project", example_name)
+        telemetry.set("is_complex_app", spec.complexity != Complexity.SIMPLE)
 
     async def analyze_spec(self, spec: str) -> str:
         msg = (
@@ -115,6 +109,8 @@ async def analyze_spec(self, spec: str) -> str:
 
         llm = self.get_llm()
         convo = AgentConvo(self).template("ask_questions").user(spec)
+        n_questions = 0
+        n_answers = 0
 
         while True:
             response: str = await llm(convo)
@@ -129,12 +125,21 @@ async def analyze_spec(self, spec: str) -> str:
                     buttons={"continue": "continue"},
                 )
                 if confirm.cancelled or confirm.button == "continue" or confirm.text == "":
+                    await self.telemetry.trace_code_event(
+                        "spec-writer-questions",
+                        {
+                            "num_questions": n_questions,
+                            "num_answers": n_answers,
+                            "new_spec": spec,
+                        },
+                    )
                     return spec
                 convo.user(confirm.text)
 
             else:
                 convo.assistant(response)
 
+                n_questions += 1
                 user_response = await self.ask_question(
                     response,
                     buttons={"skip": "Skip questions"},
@@ -147,6 +152,7 @@ async def analyze_spec(self, spec: str) -> str:
                     response: str = await llm(convo)
                     return response
 
+                n_answers += 1
                 convo.user(user_response.text)
 
     async def review_spec(self, spec: str) -> str:

core/agents/task_completer.py

@@ -1,6 +1,7 @@
 from core.agents.base import BaseAgent
 from core.agents.response import AgentResponse
 from core.log import get_logger
+from core.telemetry import telemetry
 
 log = get_logger(__name__)
 
@@ -25,5 +26,14 @@ async def run(self) -> AgentResponse:
             self.current_state.get_source_index(source),
             tasks,
         )
+        await telemetry.trace_code_event(
+            "task-end",
+            {
+                "task_index": current_task_index1,
+                "num_tasks": len(self.current_state.tasks),
+                "num_epics": len(self.current_state.epics),
+                "num_iterations": len(self.current_state.iterations),
+            },
+        )
 
         return AgentResponse.done(self)