Fix decommissioning problem in simulated test cases (project-chip#29)

* Start runner on setup and stop on clenaup

* Update SDK SHA

* Fix mypy error

* Fix unit tests

* Remove trailing whitespace

* Update python tests submodule

app/chip_tool/chip_tool.py

@@ -322,10 +322,12 @@ def __wait_for_server_exit(self) -> Optional[int]:
         return exit_code
 
     async def stop_chip_tool_server(self) -> None:
-        await self.__test_harness_runner.start()
-        await self.__test_harness_runner._client.send("quit()")
-        self.__wait_for_server_exit()
-        self.__server_started = False
+        if self.__server_started:
+            await self.start_runner()
+            await self.__test_harness_runner._client.send("quit()")
+            self.__wait_for_server_exit()
+            await self.stop_runner()
+            self.__server_started = False
 
     def __get_gateway_ip(self) -> str:
         """
@@ -391,14 +393,17 @@ async def start_container(
 
         self.__chip_tool_log = await self.start_chip_server(test_type, use_paa_certs)
 
-    def destroy_device(self) -> None:
+    async def destroy_device(self) -> None:
         """Destroy the device container."""
+        await self.stop_chip_tool_server()
+
         if self.__chip_tool_container is not None:
             container_manager.destroy(self.__chip_tool_container)
         self.__chip_tool_container = None
 
     async def start_runner(self) -> None:
-        await self.__test_harness_runner.start()
+        if not self.__test_harness_runner.is_connected:
+            await self.__test_harness_runner.start()
 
     async def stop_runner(self) -> None:
         await self.__test_harness_runner.stop()
@@ -470,12 +475,8 @@ def last_command_exit_code(self) -> Optional[int]:
         return exit_code
 
     async def send_websocket_command(self, cmd: str) -> Union[str, bytes, bytearray]:
-        response = None
-        try:
-            await self.__test_harness_runner.start()
-            response = await self.__test_harness_runner.execute(cmd)
-        finally:
-            await self.__test_harness_runner.stop()
+        await self.start_runner()
+        response = await self.__test_harness_runner.execute(cmd)
 
         # Log response
         if response:
@@ -523,12 +524,15 @@ async def run_websocket_test(
         )
         self.__runner_hooks = test_step_interface
         runner_config = TestRunnerConfig(
-            adapter, self.pseudo_clusters, runner_options, test_step_interface
+            adapter,
+            self.pseudo_clusters,
+            runner_options,
+            test_step_interface,
+            auto_start_stop=False,
         )
 
-        web_socket_config = WebSocketRunnerConfig()
-        web_socket_config.server_address = self.__get_gateway_ip()
-        self.__test_harness_runner = WebSocketRunner(config=web_socket_config)
+        await self.start_runner()
+
         return await self.__test_harness_runner.run(
             parser_builder_config, runner_config
         )

app/chip_tool/test_suite.py

@@ -163,15 +163,11 @@ async def cleanup(self) -> None:
             # the test completes. So we need to start it again to
             # perform decommissioning
             elif self.test_type == ChipToolTestType.CHIP_APP:
-                logger.info("Run simulated app instance to perform decommissioning")
-                await self.chip_tool.start_runner()
                 logger.info("Prompt user to perform decommissioning")
                 await self.__prompt_user_to_perform_decommission()
-                logger.info("Stop simulated app instance")
-                await self.chip_tool.stop_runner()
 
         logger.info("Stopping chip-tool container")
-        self.chip_tool.destroy_device()
+        await self.chip_tool.destroy_device()
         if self.border_router is not None:
             logger.info("Stopping border router container")
             self.border_router.destroy_device()

app/core/config.py

@@ -115,7 +115,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool:
     SDK_DOCKER_IMAGE: str = "connectedhomeip/chip-cert-bins"
     SDK_DOCKER_TAG: str = "f06d9520d02d68076c5accbf839f168cda89c47c"
     # SDK SHA: used to fetch test YAML from SDK.
-    SDK_SHA: str = "f06d9520d02d68076c5accbf839f168cda89c47c"
+    SDK_SHA: str = "4fde331ac902b7349653f069e9da6c933efa1466"
 
     class Config:
         case_sensitive = True

app/tests/chip_tool/test_chip_tool.py

@@ -364,7 +364,7 @@ async def test_destroy_container_running() -> None:
 
         assert chip_tool._ChipTool__chip_tool_container is not None
 
-        chip_tool.destroy_device()
+        await chip_tool.destroy_device()
 
     mock_destroy.assert_called()
     assert chip_tool._ChipTool__chip_tool_container is None
@@ -377,7 +377,7 @@ async def test_destroy_container_not_running() -> None:
     with mock.patch.object(
         target=container_manager, attribute="destroy"
     ) as mock_destroy:
-        chip_tool.destroy_device()
+        await chip_tool.destroy_device()
 
     mock_destroy.assert_not_called()
     assert chip_tool._ChipTool__chip_tool_container is None
@@ -405,8 +405,8 @@ async def test_destroy_container_once() -> None:
     ):
         await chip_tool.start_container(test_type)
 
-        chip_tool.destroy_device()
-        chip_tool.destroy_device()
+        await chip_tool.destroy_device()
+        await chip_tool.destroy_device()
 
     mock_destroy.assert_called_once()
     assert chip_tool._ChipTool__chip_tool_container is None
@@ -585,6 +585,9 @@ async def test_run_test_default_config() -> None:
         return_value=fake_container,
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
+    ), mock.patch(
+        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        return_value=True,
     ), mock.patch(
         target="app.chip_tool.chip_tool.WebSocketRunner.run",
         return_value=True,
@@ -639,6 +642,9 @@ async def test_run_test_custom_timeout() -> None:
         return_value=fake_container,
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
+    ), mock.patch(
+        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        return_value=True,
     ), mock.patch(
         target="app.chip_tool.chip_tool.WebSocketRunner.run",
         return_value=True,
@@ -687,6 +693,9 @@ async def test_run_test_with_custom_parameter() -> None:
         return_value=fake_container,
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
+    ), mock.patch(
+        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        return_value=True,
     ), mock.patch(
         target="app.chip_tool.chip_tool.WebSocketRunner.run",
         return_value=True,
@@ -736,6 +745,9 @@ async def test_run_test_with_endpoint_parameter() -> None:
         return_value=fake_container,
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
+    ), mock.patch(
+        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        return_value=True,
     ), mock.patch(
         target="app.chip_tool.chip_tool.WebSocketRunner.run",
         return_value=True,
@@ -784,6 +796,9 @@ async def test_run_test_with_nodeID_and_cluster_parameters() -> None:
         return_value=fake_container,
     ), mock.patch.object(
         target=chip_tool, attribute="start_chip_server"
+    ), mock.patch(
+        target="app.chip_tool.chip_tool.WebSocketRunner.start",
+        return_value=True,
     ), mock.patch(
         target="app.chip_tool.chip_tool.WebSocketRunner.run",
         return_value=True,