diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py index 24cd3bb479..f7f770aefd 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py @@ -246,6 +246,40 @@ def _write_to_bigquery(testcase, progression_range_start, end=progression_range_end) +_FATAL_EXECUTION_RETURN_CODES = frozenset({126, 127}) +_FATAL_LINKER_MARKERS = ( + 'error while loading shared libraries:', + 'cannot open shared object file', +) +_ABI_VERSION_TOKENS = ('GLIBC_', 'GLIBCXX_', 'CXXABI_') + + +def _is_fatal_execution_or_linker_failure(crash_result) -> bool: + """Returns True if execution failed due to OS loader / library errors.""" + if crash_result is None: + return False + + return_code = getattr(crash_result, 'return_code', None) + if isinstance(return_code, int) and not isinstance( + return_code, bool) and return_code in _FATAL_EXECUTION_RETURN_CODES: + return True + + candidate_outputs = [getattr(crash_result, 'output', None)] + if hasattr(crash_result, 'get_stacktrace'): + candidate_outputs.append(crash_result.get_stacktrace(symbolized=False)) + + for text in candidate_outputs: + if not isinstance(text, str) or not text: + continue + if 'not found' in text and any( + token in text for token in _ABI_VERSION_TOKENS): + return True + if any(marker in text for marker in _FATAL_LINKER_MARKERS): + return True + + return False + + def _log_output(revision, crash_result): """Log process output.""" logs.info( @@ -280,6 +314,16 @@ def _check_fixed_for_custom_binary(testcase: data_types.Testcase, http_flag=testcase.http_flag) _log_output(revision, result) + if _is_fatal_execution_or_linker_failure(result): + error_message = ( + f'Target execution failed with loader error ' + f'(code={result.return_code}) on custom build r{revision}.') + logs.error(error_message) + return uworker_msg_pb2.Output( # pylint: disable=no-member + progression_task_output=uworker_msg_pb2.ProgressionTaskOutput(), # pylint: disable=no-member + error_message=error_message, + error_type=uworker_msg_pb2.ErrorType.PROGRESSION_BAD_BUILD) # pylint: disable=no-member + # If this still crashes on the most recent build, it's not fixed. The task # will be rescheduled by a cron job and re-attempted eventually. if result.is_crash(): @@ -369,6 +413,15 @@ def _testcase_reproduces_in_revision( test_timeout, http_flag=testcase.http_flag) _log_output(revision, result) + if _is_fatal_execution_or_linker_failure(result): + progression_task_output.build_data_list[-1].is_bad_build = True + error_message = (f'Target execution failed with loader error ' + f'(code={result.return_code}) at r{revision}. Skipping') + logs.error(error_message) + return None, uworker_msg_pb2.Output( # pylint: disable=no-member + progression_task_output=progression_task_output, + error_message=error_message, + error_type=uworker_msg_pb2.ErrorType.PROGRESSION_BAD_BUILD) # pylint: disable=no-member return result, None diff --git a/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/progression_task_test.py b/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/progression_task_test.py index 09057af9a1..a9d89512a6 100644 --- a/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/progression_task_test.py +++ b/src/clusterfuzz/_internal/tests/core/bot/tasks/utasks/progression_task_test.py @@ -71,6 +71,7 @@ def setUp(self): ]) helpers.patch_environ(self) os.environ['APP_NAME'] = 'app_name' + os.environ['CONFIG_DIR_OVERRIDE'] = './configs/test' def test_error_on_failed_setup(self): """Ensure that we throw an exception if we fail to set up a build.""" @@ -120,6 +121,52 @@ def test_no_crash(self): self.assertEqual(len(progression_task_output.build_data_list), 1) self.assertEqual(progression_task_output.build_data_list[0], build_data) + def test_exit_127_missing_shared_library_treated_as_bad_build(self): + """Ensures exit 126/127 or missing .so aborts as PROGRESSION_BAD_BUILD.""" + from clusterfuzz._internal.crash_analysis.crash_result import \ + CrashResult # pylint: disable=import-outside-toplevel + build_data = uworker_msg_pb2.BuildData( + revision=42, + is_bad_build=False, + should_ignore_crash_result=False, + build_run_console_output='') + self.mock.check_for_bad_build.return_value = build_data + self.mock.test_for_crash_with_retries.return_value = CrashResult( + 127, 0.1, + '/out/fuzzer: error while loading shared libraries: libssl.so.1.1: ' + 'cannot open shared object file: No such file or directory') + testcase = data_types.Testcase() + progression_task_output = uworker_msg_pb2.ProgressionTaskOutput() + result, worker_output = progression_task._testcase_reproduces_in_revision( # pylint: disable=protected-access + testcase, '/tmp/blah', 'job_type', 42, None, progression_task_output) + self.assertIsNone(result) + self.assertIsNotNone(worker_output) + self.assertEqual(worker_output.error_type, + uworker_msg_pb2.ErrorType.PROGRESSION_BAD_BUILD) + self.assertTrue(progression_task_output.build_data_list[0].is_bad_build) + + def test_glibc_symbol_mismatch_with_zero_return_code_treated_as_bad_build( + self): + """Ensures ABI errors abort even after reproduce_with_retries sets 0.""" + from clusterfuzz._internal.crash_analysis.crash_result import \ + CrashResult # pylint: disable=import-outside-toplevel + build_data = uworker_msg_pb2.BuildData( + revision=43, + is_bad_build=False, + should_ignore_crash_result=False, + build_run_console_output='') + self.mock.check_for_bad_build.return_value = build_data + self.mock.test_for_crash_with_retries.return_value = CrashResult( + 0, 0, '/out/fuzzer: /lib/x86_64-linux-gnu/libc.so.6: ' + "version `GLIBC_2.38' not found") + testcase = data_types.Testcase() + progression_task_output = uworker_msg_pb2.ProgressionTaskOutput() + result, worker_output = progression_task._testcase_reproduces_in_revision( # pylint: disable=protected-access + testcase, '/tmp/blah', 'job_type', 43, None, progression_task_output) + self.assertIsNone(result) + self.assertEqual(worker_output.error_type, + uworker_msg_pb2.ErrorType.PROGRESSION_BAD_BUILD) + @test_utils.with_cloud_emulators('datastore') class UtaskPreprocessTest(unittest.TestCase):