Prevent task ack on Windows build dir restart

src/clusterfuzz/_internal/base/utils.py

@@ -729,10 +729,13 @@ def remove_sub_strings(string, substrings):
 def restart_machine():
   """Restart machine."""
   if environment.platform() == 'WINDOWS':
-    os.system('shutdown /f /r /t 0')
+    return_code = os.system('shutdown /f /r /t 0')
   else:
     # POSIX platforms.
-    os.system('sudo shutdown -r now')
+    return_code = os.system('sudo shutdown -r now')
+
+  if return_code:
+    logs.error(f'Failed to restart machine (exit={return_code}).')
 
 
 def search_bytes_in_file(search_bytes, file_handle) -> bool:

src/clusterfuzz/_internal/build_management/build_manager.py

@@ -20,6 +20,7 @@
 import re
 import shutil
 import subprocess
+import sys
 import time
 from typing import Optional
 
@@ -164,6 +165,11 @@ def _handle_unrecoverable_error_on_windows():
   logs.error('Unrecoverable error, restarting machine...')
   time.sleep(60)
   utils.restart_machine()
+  # Wait for OS shutdown to terminate this process.
+  time.sleep(60)
+  # Not expected to be reached: only runs if OS shutdown fails, preventing the
+  # failed task from being acked or a new task from being leased.
+  sys.exit(1)
 
 
 def _remove_scheme(bucket_path):

src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py

@@ -454,6 +454,29 @@ def test_unpack_build_fails_when_out_of_space(self):
                                        'gs://fake_bucket/fake.zip')
           self.assertFalse(result)
 
+  def test_unpack_build_windows_remove_directory_fails_restarts_and_exits(self):
+    """Test that _unpack_build restarts and exits non-zero on Windows when
+    clearing the build directory fails."""
+    test_helpers.patch(self, [
+        'time.sleep',
+        'clusterfuzz._internal.base.utils.restart_machine',
+        'clusterfuzz._internal.system.environment.platform',
+        'clusterfuzz._internal.system.shell.remove_directory',
+    ])
+    self.mock.platform.return_value = 'WINDOWS'
+    self.mock.remove_directory.return_value = False
+    build = build_manager.RegularBuild('/builds/build4', 1,
+                                       'gs://fake_bucket/fake.zip')
+
+    with self.assertRaises(SystemExit) as cm:
+      build._unpack_build('/builds/build4', '/builds/build4/dir',
+                          'gs://fake_bucket/fake.zip')
+
+    self.assertEqual(cm.exception.code, 1)
+    self.mock.restart_machine.assert_called_once_with()
+    self.assertEqual(self.mock.sleep.call_args_list,
+                     [mock.call(60), mock.call(60)])
+
 
 class RegularLibFuzzerBuildTest(fake_filesystem_unittest.TestCase):
   """Tests for regular libFuzzer build setup."""