Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow unstable browser builds #373

Merged
merged 2 commits into from
Sep 22, 2023
Merged

Allow unstable browser builds #373

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
077a7a8
Allow unstable browser builds
tysmith Sep 21, 2023
4015a5e
Remove built-in ignore list from FuzzManagerReporter
tysmith Sep 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 0 additions & 23 deletions grizzly/common/reporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,25 +226,6 @@ def _process_rr_trace(self, report):
# remove traces so they are not uploaded to FM (because they are huge)
rmtree(trace_path)

@staticmethod
def _ignored(report):
# This is here to prevent reporting stack-less crashes
# that were caused by system OOM
log_data = report.preferred.read_text("utf-8", errors="ignore")
# ignore sanitizer OOMs missing stack
if report.stack is None:
mem_errs = (
"ERROR: Failed to mmap",
# NOTE: max_allocation_size_mb can trigger a similar message
": AddressSanitizer failed to allocate",
"Sanitizer: internal allocator is out of memory trying to allocate",
)
# scan log data for memory error strings
if any(msg in log_data for msg in mem_errs):
return True
# ignore Valgrind crashes
return log_data.startswith("VEX temporary storage exhausted.")

def _submit_report(self, report, test_cases, force):
collector = Collector(tool=self.tool)

Expand All @@ -263,10 +244,6 @@ def _submit_report(self, report, test_cases, force):
else:
LOG.debug("sigCacheDir does not exist (%r)", collector.sigCacheDir)

if self._ignored(report):
LOG.info("Report is in ignore list")
return None

if report.is_hang:
self.add_extra_metadata("is_hang", True)

Expand Down
42 changes: 9 additions & 33 deletions grizzly/common/test_reporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,31 +156,22 @@ def test_fuzzmanager_reporter_01(mocker, tmp_path):


@mark.parametrize(
"tests, frequent, ignored, force, sig_cache",
"tests, frequent, force, sig_cache",
[
# report - without test
(False, False, False, False, True),
(False, False, False, True),
# report - with test
(True, False, False, False, True),
(True, False, False, True),
# report - frequent
(True, True, False, False, True),
(True, True, False, True),
# report - forced frequent
(True, True, False, True, True),
# report - ignored
(True, False, True, False, True),
(True, True, True, True),
# report - missing sigCacheDir
(False, False, False, False, False),
(False, False, False, False),
],
)
def test_fuzzmanager_reporter_02(
mocker, tmp_path, tests, frequent, ignored, force, sig_cache
):
def test_fuzzmanager_reporter_02(mocker, tmp_path, tests, frequent, force, sig_cache):
"""test FuzzManagerReporter.submit()"""
mocker.patch(
"grizzly.common.reporter.FuzzManagerReporter._ignored",
new_callable=mocker.MagicMock,
return_value=ignored,
)
mocker.patch("grizzly.common.reporter.Path.cwd", return_value=tmp_path)
mocker.patch("grizzly.common.reporter.getenv", autospec=True, return_value="0")
fake_collector = mocker.patch("grizzly.common.reporter.Collector", autospec=True)
Expand Down Expand Up @@ -213,7 +204,7 @@ def test_fuzzmanager_reporter_02(
)
assert not log_path.is_dir()
assert fake_collector.call_args == ({"tool": "fake-tool"},)
if (frequent and not force) or ignored:
if frequent and not force:
assert fake_collector.return_value.submit.call_count == 0
assert fake_test.dump.call_count == 0
else:
Expand All @@ -222,21 +213,6 @@ def test_fuzzmanager_reporter_02(
assert fake_test.dump.call_count == 1


def test_fuzzmanager_reporter_03(mocker, tmp_path):
"""test FuzzManagerReporter._ignored()"""
log_file = tmp_path / "test.log"
log_file.touch()
report = mocker.Mock(spec_set=Report, path=tmp_path, preferred=log_file, stack=None)
# not ignored
assert not FuzzManagerReporter._ignored(report)
# ignored - sanitizer OOM missing stack
log_file.write_bytes(b"ERROR: Failed to mmap")
assert FuzzManagerReporter._ignored(report)
# ignored - Valgrind OOM
log_file.write_bytes(b"VEX temporary storage exhausted.")
assert FuzzManagerReporter._ignored(report)


@mark.parametrize(
"tool, sanitized",
[
Expand All @@ -250,7 +226,7 @@ def test_fuzzmanager_reporter_03(mocker, tmp_path):
("TeSt-ToOl", "test-tool"),
],
)
def test_fuzzmanager_reporter_04(tool, sanitized):
def test_fuzzmanager_reporter_03(tool, sanitized):
"""test FuzzManagerReporter() sanitizing tool"""
assert FuzzManagerReporter(tool).tool == sanitized

Expand Down
17 changes: 7 additions & 10 deletions grizzly/session.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,13 +231,15 @@ def run(
LOG.debug("ignoring test case since nothing was served")
if current_test.landing_page not in current_test.contents:
raise SessionError("Test case is missing landing page")
if runner.initial:
if runner.initial and result.status != Result.NONE:
# since this is the first iteration since the Target launched
# something is likely wrong with the Target or Adapter
if result.status == Result.FOUND:
LOG.warning("Delayed startup failure detected")
else:
LOG.warning("Timeout too short? System too busy?")
LOG.warning(
"Failure detected before running a test case, "
"browser build is potentially unstable"
)
if result.timeout:
LOG.warning("Browser hung? Timeout too short? System too busy?")
else:
if self.adapter.IGNORE_UNSERVED:
LOG.debug("removing unserved files from the test case")
Expand Down Expand Up @@ -284,11 +286,6 @@ def run(
else:
LOG.info("Ignored - %d", self.status.ignored)

# ignore startup failure if it did not happen early on
# to avoid aborting the fuzzing session unnecessarily
if runner.startup_failure and self.status.iteration < 100:
raise SessionError("Please check Adapter and Target")

if self.adapter.remaining is not None and self.adapter.remaining < 1:
# all test cases have been replayed
LOG.info("Replay Complete")
Expand Down
41 changes: 12 additions & 29 deletions grizzly/test_session.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,23 +258,6 @@ def generate(self, _testcase, _server_map):
session.run([], 10)


def test_session_05(mocker):
"""test Target not requesting landing page"""
server = mocker.Mock(spec_set=Sapphire, port=0x1337)
server.serve_path.return_value = (Served.TIMEOUT, [])
target = mocker.Mock(
spec_set=Target,
assets=mocker.Mock(spec_set=AssetManager),
environ={},
launch_timeout=30,
)
target.monitor.launches = 1
with Session(SimpleAdapter(False), None, server, target) as session:
with raises(SessionError, match="Please check Adapter and Target"):
session.run([], 10)
assert target.handle_hang.call_count == 1


@mark.parametrize(
"harness, report_size",
[
Expand All @@ -286,7 +269,7 @@ def test_session_05(mocker):
(False, 1),
],
)
def test_session_06(mocker, harness, report_size):
def test_session_05(mocker, harness, report_size):
"""test Session - handle Target delayed failures"""
reporter = mocker.Mock(spec_set=Reporter)
report = mocker.Mock(
Expand Down Expand Up @@ -326,13 +309,13 @@ def test_session_06(mocker, harness, report_size):
@mark.parametrize(
"srv_results, target_result, ignored, results",
[
# delayed startup crash
# delayed startup crash (before requesting test)
(Served.NONE, Result.FOUND, 0, 1),
# startup hang/unresponsive
(Served.TIMEOUT, Result.NONE, 1, 0),
],
)
def test_session_07(mocker, srv_results, target_result, ignored, results):
def test_session_06(mocker, srv_results, target_result, ignored, results):
"""test Session.run() - initial test case was not served"""
report = mocker.Mock(
spec_set=Report, major="major123", minor="minor456", crash_hash="123"
Expand All @@ -347,22 +330,22 @@ def test_session_07(mocker, srv_results, target_result, ignored, results):
environ={},
launch_timeout=30,
)
target.log_size.return_value = 1
target.monitor.launches = 1
target.check_result.side_effect = (target_result,)
target.create_report.return_value = report
with Session(SimpleAdapter(False), reporter, server, target) as session:
server.serve_path.return_value = (srv_results, [])
with raises(SessionError, match="Please check Adapter and Target"):
session.run([], 10, iteration_limit=2)
session.run([], 10, iteration_limit=1)
assert session.status.iteration == 1
assert session.status.results.total == results
assert session.status.ignored == ignored
assert reporter.submit.call_count == results
assert target.check_result.call_count == results
assert target.handle_hang.call_count == ignored
assert reporter.submit.call_count == results
assert target.check_result.call_count == results
assert target.handle_hang.call_count == ignored


def test_session_08(mocker):
def test_session_07(mocker):
"""test Session.run() ignoring failures"""
result = RunResult([], 0.1, status=Result.IGNORED)
result.attempted = True
Expand All @@ -387,7 +370,7 @@ def test_session_08(mocker):
assert session.status.ignored == 1


def test_session_09(mocker):
def test_session_08(mocker):
"""test Session.run() report hang"""
result = RunResult([], 60.0, status=Result.FOUND, timeout=True)
result.attempted = True
Expand Down Expand Up @@ -426,7 +409,7 @@ def test_session_09(mocker):
(True, 1, 1, 10, 5),
],
)
def test_session_10(mocker, harness, report_size, relaunch, iters, report_limit):
def test_session_09(mocker, harness, report_size, relaunch, iters, report_limit):
"""test Session - limit report submission"""
adapter = SimpleAdapter(harness)
reporter = mocker.Mock(spec_set=Reporter)
Expand Down Expand Up @@ -473,7 +456,7 @@ def test_session_10(mocker, harness, report_size, relaunch, iters, report_limit)
(True, 2, 1, 0),
],
)
def test_session_12(mocker, harness, iters, result_limit, results):
def test_session_10(mocker, harness, iters, result_limit, results):
"""test Session - limit results"""
adapter = SimpleAdapter(harness)
reporter = mocker.Mock(spec_set=Reporter)
Expand Down
Loading