[replay] Improve compatibility with unstable builds

grizzly/replay/replay.py

@@ -56,12 +56,14 @@ class ReplayResult:
         durations: Number of seconds spent running each testcase.
         expected: Signature match.
         count: Number of times detected.
+        early: Result was detected before a testcase was requested.
     """
 
     report: Report
     durations: List[float]
     expected: bool
     count: int = 1
+    early: bool = False
 
 
 class ReplayManager:
@@ -443,30 +445,27 @@ def harness_fn(_: str) -> bytes:  # pragma: no cover
                         log_path, self.target.binary, is_hang=run_result.timeout
                     )
                     # set active signature
-                    if (
-                        not runner.startup_failure
-                        and not self._any_crash
-                        and not run_result.timeout
-                        and not sig_set
-                    ):
+                    if not self._any_crash and not run_result.timeout and not sig_set:
                         assert not expect_hang
                         assert self._signature is None
                         LOG.debug(
                             "no signature given, using short sig %r",
                             report.short_signature,
                         )
+                        if runner.startup_failure:
+                            LOG.warning(
+                                "Using signature from startup failure! "
+                                "Provide a signature to avoid this."
+                            )
                         self._signature = report.crash_signature
                         sig_set = True
                         if self._signature is not None:
                             assert not sig_hash, "sig_hash should only be set once"
                             sig_hash = Report.calc_hash(self._signature)
 
                     # bucket result
-                    if not runner.startup_failure and (
-                        self._any_crash
-                        or self.check_match(
-                            self._signature, report, expect_hang, sig_set
-                        )
+                    if self._any_crash or self.check_match(
+                        self._signature, report, expect_hang, sig_set
                     ):
                         if sig_hash is not None:
                             LOG.debug("using signature hash (%s) to bucket", sig_hash)
@@ -481,7 +480,9 @@ def harness_fn(_: str) -> bytes:  # pragma: no cover
                             report.minor[:8],
                         )
                         if bucket_hash not in reports:
-                            reports[bucket_hash] = ReplayResult(report, durations, True)
+                            reports[bucket_hash] = ReplayResult(
+                                report, durations, True, early=runner.startup_failure
+                            )
                             LOG.debug("now tracking %s", bucket_hash)
                             report = None  # don't remove report
                         else:
@@ -497,7 +498,7 @@ def harness_fn(_: str) -> bytes:  # pragma: no cover
                         self.status.ignored += 1
                         if report.crash_hash not in reports:
                             reports[report.crash_hash] = ReplayResult(
-                                report, durations, False
+                                report, durations, False, early=runner.startup_failure
                             )
                             LOG.debug("now tracking %s", report.crash_hash)
                             report = None  # don't remove report

grizzly/replay/test_replay.py

@@ -164,21 +164,6 @@ def test_replay_05(mocker, server):
         # target.close() called once in runner and once by ReplayManager.run()
         assert target.close.call_count == 2
     target.reset_mock()
-    # test target crashed
-    target.check_result.return_value = Result.FOUND
-    target.save_logs = _fake_save_logs
-    with ReplayManager([], server, target, use_harness=False) as replay:
-        results = replay.run(tests, 10, repeat=1)
-        assert replay.status
-        assert replay.status.ignored == 1
-        assert replay.status.iteration == 1
-        assert replay.status.results.total == 0
-        assert replay._signature is None
-        # target.close() called once in runner and once by ReplayManager.run()
-        assert target.close.call_count == 2
-    assert len(results) == 1
-    assert results[0].count == 1
-    assert not results[0].expected
 
 
 def test_replay_06(mocker, server):
@@ -408,23 +393,54 @@ def test_replay_12(mocker, server):
     assert report_2.cleanup.call_count == 1
 
 
-def test_replay_13(mocker, server):
-    """test ReplayManager.run() - any crash - startup failure"""
-    server.serve_path.return_value = (Served.NONE, {})
+@mark.parametrize(
+    "to_serve, sig_value, expected, unexpected",
+    [
+        # No signature provided
+        (((Served.NONE, {}), (Served.ALL, {"a.html": "/fake/path"})), None, 2, 0),
+        (((Served.ALL, {"a.html": "/fake/path"}), (Served.NONE, {})), None, 2, 0),
+        (((Served.NONE, {}), (Served.NONE, {})), None, 2, 0),
+        # Signature provided (signatures match)
+        (((Served.NONE, {}), (Served.ALL, {"a.html": "/fake/path"})), "STDERR", 2, 0),
+        (((Served.ALL, {"a.html": "/fake/path"}), (Served.NONE, {})), "STDERR", 2, 0),
+        (((Served.NONE, {}), (Served.NONE, {})), "STDERR", 2, 0),
+        # Signature provided (signatures don't match)
+        (((Served.NONE, {}), (Served.ALL, {"a.html": "/fake/path"})), "miss", 0, 1),
+        (((Served.ALL, {"a.html": "/fake/path"}), (Served.NONE, {})), "miss", 0, 1),
+        (((Served.NONE, {}), (Served.NONE, {})), "miss", 0, 1),
+    ],
+)
+def test_replay_13(mocker, server, tmp_path, to_serve, sig_value, expected, unexpected):
+    """test ReplayManager.run() - results triggered after launch before running test"""
+    server.serve_path.side_effect = to_serve
+
+    # prepare signature
+    if sig_value is not None:
+        sig_file = tmp_path / "sig.json"
+        sig_file.write_text(
+            "{\n"
+            '  "symptoms": [\n'
+            "    {\n"
+            '      "src": "stderr",\n'
+            '      "type": "output",\n'
+            f'      "value": "/{sig_value}/"\n'
+            "    }\n"
+            "  ]\n"
+            "}\n"
+        )
+        sig = CrashSignature.fromFile(str(sig_file))
+    else:
+        sig = None
+
     target = mocker.Mock(spec_set=Target, binary=Path("bin"), launch_timeout=30)
     target.check_result.return_value = Result.FOUND
     target.save_logs = _fake_save_logs
     target.monitor.is_healthy.return_value = False
     tests = [mocker.MagicMock(spec_set=TestCase, entry_point="a.html")]
-    with ReplayManager([], server, target, any_crash=True, use_harness=False) as replay:
-        results = replay.run(tests, 10, repeat=1, min_results=1)
-        assert results
-        assert not any(x.expected for x in results)
-        assert target.close.call_count == 2
-        assert replay.status
-        assert replay.status.iteration == 1
-        assert replay.status.results.total == 0
-        assert replay.status.ignored == 1
+    with ReplayManager(set(), server, target, any_crash=False, signature=sig) as replay:
+        results = replay.run(tests, 10, repeat=2, min_results=2)
+        assert sum(x.count for x in results if x.expected) == expected
+        assert sum(x.count for x in results if not x.expected) == unexpected
 
 
 def test_replay_14(mocker, server):