diff --git a/grizzly/replay/replay.py b/grizzly/replay/replay.py index 43561d28..c3dad74c 100644 --- a/grizzly/replay/replay.py +++ b/grizzly/replay/replay.py @@ -56,12 +56,14 @@ class ReplayResult: durations: Number of seconds spent running each testcase. expected: Signature match. count: Number of times detected. + early: Result was detected before a testcase was requested. """ report: Report durations: List[float] expected: bool count: int = 1 + early: bool = False class ReplayManager: @@ -443,18 +445,18 @@ def harness_fn(_: str) -> bytes: # pragma: no cover log_path, self.target.binary, is_hang=run_result.timeout ) # set active signature - if ( - not runner.startup_failure - and not self._any_crash - and not run_result.timeout - and not sig_set - ): + if not self._any_crash and not run_result.timeout and not sig_set: assert not expect_hang assert self._signature is None LOG.debug( "no signature given, using short sig %r", report.short_signature, ) + if runner.startup_failure: + LOG.warning( + "Using signature from startup failure! " + "Provide a signature to avoid this." + ) self._signature = report.crash_signature sig_set = True if self._signature is not None: @@ -462,11 +464,8 @@ def harness_fn(_: str) -> bytes: # pragma: no cover sig_hash = Report.calc_hash(self._signature) # bucket result - if not runner.startup_failure and ( - self._any_crash - or self.check_match( - self._signature, report, expect_hang, sig_set - ) + if self._any_crash or self.check_match( + self._signature, report, expect_hang, sig_set ): if sig_hash is not None: LOG.debug("using signature hash (%s) to bucket", sig_hash) @@ -481,7 +480,9 @@ def harness_fn(_: str) -> bytes: # pragma: no cover report.minor[:8], ) if bucket_hash not in reports: - reports[bucket_hash] = ReplayResult(report, durations, True) + reports[bucket_hash] = ReplayResult( + report, durations, True, early=runner.startup_failure + ) LOG.debug("now tracking %s", bucket_hash) report = None # don't remove report else: @@ -497,7 +498,7 @@ def harness_fn(_: str) -> bytes: # pragma: no cover self.status.ignored += 1 if report.crash_hash not in reports: reports[report.crash_hash] = ReplayResult( - report, durations, False + report, durations, False, early=runner.startup_failure ) LOG.debug("now tracking %s", report.crash_hash) report = None # don't remove report diff --git a/grizzly/replay/test_replay.py b/grizzly/replay/test_replay.py index 38177c11..96bb9aae 100644 --- a/grizzly/replay/test_replay.py +++ b/grizzly/replay/test_replay.py @@ -164,21 +164,6 @@ def test_replay_05(mocker, server): # target.close() called once in runner and once by ReplayManager.run() assert target.close.call_count == 2 target.reset_mock() - # test target crashed - target.check_result.return_value = Result.FOUND - target.save_logs = _fake_save_logs - with ReplayManager([], server, target, use_harness=False) as replay: - results = replay.run(tests, 10, repeat=1) - assert replay.status - assert replay.status.ignored == 1 - assert replay.status.iteration == 1 - assert replay.status.results.total == 0 - assert replay._signature is None - # target.close() called once in runner and once by ReplayManager.run() - assert target.close.call_count == 2 - assert len(results) == 1 - assert results[0].count == 1 - assert not results[0].expected def test_replay_06(mocker, server): @@ -408,23 +393,54 @@ def test_replay_12(mocker, server): assert report_2.cleanup.call_count == 1 -def test_replay_13(mocker, server): - """test ReplayManager.run() - any crash - startup failure""" - server.serve_path.return_value = (Served.NONE, {}) +@mark.parametrize( + "to_serve, sig_value, expected, unexpected", + [ + # No signature provided + (((Served.NONE, {}), (Served.ALL, {"a.html": "/fake/path"})), None, 2, 0), + (((Served.ALL, {"a.html": "/fake/path"}), (Served.NONE, {})), None, 2, 0), + (((Served.NONE, {}), (Served.NONE, {})), None, 2, 0), + # Signature provided (signatures match) + (((Served.NONE, {}), (Served.ALL, {"a.html": "/fake/path"})), "STDERR", 2, 0), + (((Served.ALL, {"a.html": "/fake/path"}), (Served.NONE, {})), "STDERR", 2, 0), + (((Served.NONE, {}), (Served.NONE, {})), "STDERR", 2, 0), + # Signature provided (signatures don't match) + (((Served.NONE, {}), (Served.ALL, {"a.html": "/fake/path"})), "miss", 0, 1), + (((Served.ALL, {"a.html": "/fake/path"}), (Served.NONE, {})), "miss", 0, 1), + (((Served.NONE, {}), (Served.NONE, {})), "miss", 0, 1), + ], +) +def test_replay_13(mocker, server, tmp_path, to_serve, sig_value, expected, unexpected): + """test ReplayManager.run() - results triggered after launch before running test""" + server.serve_path.side_effect = to_serve + + # prepare signature + if sig_value is not None: + sig_file = tmp_path / "sig.json" + sig_file.write_text( + "{\n" + ' "symptoms": [\n' + " {\n" + ' "src": "stderr",\n' + ' "type": "output",\n' + f' "value": "/{sig_value}/"\n' + " }\n" + " ]\n" + "}\n" + ) + sig = CrashSignature.fromFile(str(sig_file)) + else: + sig = None + target = mocker.Mock(spec_set=Target, binary=Path("bin"), launch_timeout=30) target.check_result.return_value = Result.FOUND target.save_logs = _fake_save_logs target.monitor.is_healthy.return_value = False tests = [mocker.MagicMock(spec_set=TestCase, entry_point="a.html")] - with ReplayManager([], server, target, any_crash=True, use_harness=False) as replay: - results = replay.run(tests, 10, repeat=1, min_results=1) - assert results - assert not any(x.expected for x in results) - assert target.close.call_count == 2 - assert replay.status - assert replay.status.iteration == 1 - assert replay.status.results.total == 0 - assert replay.status.ignored == 1 + with ReplayManager(set(), server, target, any_crash=False, signature=sig) as replay: + results = replay.run(tests, 10, repeat=2, min_results=2) + assert sum(x.count for x in results if x.expected) == expected + assert sum(x.count for x in results if not x.expected) == unexpected def test_replay_14(mocker, server):