From b1d7943af3ac2e04204e5a8d16489c6bb60d1215 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Thu, 3 Sep 2026 18:32:23 -0400 Subject: [PATCH] Avoid a Buildbot bug that can hang CTest steps forever CTest.run() fetches the ctest XML results file from the worker via getFileContentFromWorker(), which uses buildbot's StringFileWriter. That class decodes each 32KB transfer chunk as UTF-8 independently (buildbot/process/remotetransfer.py), so a multi-byte character split across a chunk boundary raises UnicodeDecodeError inside a PB remote-message handler rather than through buildbot's normal command-failure path. The step is never notified, so it hangs forever, still holding its counting-mode claim on performance_lock -- which then starves any sibling step on the same worker that needs exclusive access to that lock (e.g. the performance test steps), for however long it takes someone to notice and restart the master. This is a long-standing, still-open upstream bug (buildbot/buildbot#3982); a 2019 fix attempt (buildbot/buildbot#4621) stalled in review and was closed by stale-bot without landing, and it's still present in the pinned 4.3.0 release, so there's no version bump that avoids it. Since Xml.fromstring() accepts bytes directly, we don't need decoded text here at all: read the file as raw bytes via a small custom FileWriterImpl instead of buildbot's decoding one, sidestepping the bug rather than working around its symptoms. Confirmed against a byte-for-byte reproduction of the production failure (same "position 32767: unexpected end of data" error) that the old writer raises and the new one doesn't. Co-Authored-By: Claude Sonnet 5 --- master/custom_steps.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/master/custom_steps.py b/master/custom_steps.py index 6bcf044..b823d88 100644 --- a/master/custom_steps.py +++ b/master/custom_steps.py @@ -2,11 +2,33 @@ from buildbot.process.buildstep import BuildStep, BuildStepFailed, ShellMixin from buildbot.steps.worker import CompositeStepMixin +from buildbot.worker.protocols.base import FileWriterImpl from twisted.internet import defer __all__ = ["CTest"] +class _BytesFileWriter(FileWriterImpl): + """Like buildbot's own StringFileWriter, but keeps raw bytes instead of eagerly decoding + each chunk as UTF-8. StringFileWriter.remote_write() decodes every chunk independently, so a + multi-byte character split across a chunk boundary raises UnicodeDecodeError; since that + happens inside a PB remote-message handler rather than buildbot's normal command-failure path, + the step never gets notified and hangs forever, still holding onto the worker and any locks it + acquired (https://github.com/buildbot/buildbot/issues/3982, still open on buildbot 4.3.0). + We only need the bytes to hand to Xml.fromstring, which accepts them directly, so decoding + isn't even necessary here. + """ + + def __init__(self): + self.buffer = b"" + + def remote_write(self, data): + self.buffer += data + + def remote_close(self): + pass + + class CTest(ShellMixin, CompositeStepMixin, BuildStep): name = "ctest" @@ -65,7 +87,7 @@ def run(self): if len(xml_results) != 1: raise BuildStepFailed(f"Expected to find a single XML file. Got: {xml_results}") - ctest_log = yield self.getFileContentFromWorker(xml_results[0], abandonOnFailure=True) + ctest_log = yield self.getFileBytesFromWorker(xml_results[0], abandonOnFailure=True) # Parse the result, collecting test failures into more convenient logs. root = Xml.fromstring(ctest_log) # nosec B314 @@ -98,6 +120,22 @@ def run(self): return cmd.results() + def getFileBytesFromWorker(self, filename, abandonOnFailure=False): + self.checkWorkerHasCommand("uploadFile") + writer = _BytesFileWriter() + args = {"workdir": self.workdir, "writer": writer, "maxsize": None, "blocksize": 32 * 1024} + if self.workerVersionIsOlderThan("uploadFile", "3.0"): + args["slavesrc"] = filename + else: + args["workersrc"] = filename + + def commandComplete(cmd): + return None if cmd.didFail() else writer.buffer + + return self.runRemoteCommand( + "uploadFile", args, abandonOnFailure=abandonOnFailure, evaluateCommand=commandComplete + ) + def write_xml(self, test, *sections, indent=0): for node, log in sections: text = test.findtext(node)