diff --git a/src/clusterfuzz/_internal/base/utils.py b/src/clusterfuzz/_internal/base/utils.py index 580e707393..ad6e363d21 100644 --- a/src/clusterfuzz/_internal/base/utils.py +++ b/src/clusterfuzz/_internal/base/utils.py @@ -729,10 +729,13 @@ def remove_sub_strings(string, substrings): def restart_machine(): """Restart machine.""" if environment.platform() == 'WINDOWS': - os.system('shutdown /f /r /t 0') + return_code = os.system('shutdown /f /r /t 0') else: # POSIX platforms. - os.system('sudo shutdown -r now') + return_code = os.system('sudo shutdown -r now') + + if return_code: + logs.error(f'Failed to restart machine (exit={return_code}).') def search_bytes_in_file(search_bytes, file_handle) -> bool: diff --git a/src/clusterfuzz/_internal/build_management/build_manager.py b/src/clusterfuzz/_internal/build_management/build_manager.py index b2a094b73b..b1d26e4b49 100644 --- a/src/clusterfuzz/_internal/build_management/build_manager.py +++ b/src/clusterfuzz/_internal/build_management/build_manager.py @@ -20,6 +20,7 @@ import re import shutil import subprocess +import sys import time from typing import Optional @@ -164,6 +165,11 @@ def _handle_unrecoverable_error_on_windows(): logs.error('Unrecoverable error, restarting machine...') time.sleep(60) utils.restart_machine() + # Wait for OS shutdown to terminate this process. + time.sleep(60) + # Not expected to be reached: only runs if OS shutdown fails, preventing the + # failed task from being acked or a new task from being leased. + sys.exit(1) def _remove_scheme(bucket_path): diff --git a/src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py b/src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py index 5e22d811d7..c0c17281bf 100644 --- a/src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py +++ b/src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py @@ -454,6 +454,29 @@ def test_unpack_build_fails_when_out_of_space(self): 'gs://fake_bucket/fake.zip') self.assertFalse(result) + def test_unpack_build_windows_remove_directory_fails_restarts_and_exits(self): + """Test that _unpack_build restarts and exits non-zero on Windows when + clearing the build directory fails.""" + test_helpers.patch(self, [ + 'time.sleep', + 'clusterfuzz._internal.base.utils.restart_machine', + 'clusterfuzz._internal.system.environment.platform', + 'clusterfuzz._internal.system.shell.remove_directory', + ]) + self.mock.platform.return_value = 'WINDOWS' + self.mock.remove_directory.return_value = False + build = build_manager.RegularBuild('/builds/build4', 1, + 'gs://fake_bucket/fake.zip') + + with self.assertRaises(SystemExit) as cm: + build._unpack_build('/builds/build4', '/builds/build4/dir', + 'gs://fake_bucket/fake.zip') + + self.assertEqual(cm.exception.code, 1) + self.mock.restart_machine.assert_called_once_with() + self.assertEqual(self.mock.sleep.call_args_list, + [mock.call(60), mock.call(60)]) + class RegularLibFuzzerBuildTest(fake_filesystem_unittest.TestCase): """Tests for regular libFuzzer build setup."""