From d09aae5824d907bde3d600e28398447e8a3b5709 Mon Sep 17 00:00:00 2001 From: nethantd Date: Wed, 28 Oct 2026 13:45:26 +0100 Subject: [PATCH] fix(syft-job): sort submission files by posix path WindowsPath sorts case-insensitively and PosixPath case-sensitively, so a DS on Windows and a DO on Linux hashed files in a different order. The CRLF part of this PR is no longer needed since #9593 sends job files byte for byte. Co-Authored-By: Claude Opus 5.5 --- packages/syft-job/src/syft_job/submission.py | 16 ++++++---- .../syft-job/tests/test_submission_hash.py | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 packages/syft-job/tests/test_submission_hash.py diff --git a/packages/syft-job/src/syft_job/submission.py b/packages/syft-job/src/syft_job/submission.py index c682b726489..b40b8275d92 100644 --- a/packages/syft-job/src/syft_job/submission.py +++ b/packages/syft-job/src/syft_job/submission.py @@ -93,13 +93,17 @@ def _digest( skip: frozenset[str] = frozenset(), ) -> str: digest = hashlib.sha256() + # Sort on the posix string: WindowsPath sorts case-insensitively. files = sorted( - path - for path in submission_dir.rglob("*") - if path.is_file() - and not _is_skipped(relative := path.relative_to(submission_dir)) - and (entries is None or relative.parts[0] in entries) - and relative.as_posix() not in skip + ( + path + for path in submission_dir.rglob("*") + if path.is_file() + and not _is_skipped(relative := path.relative_to(submission_dir)) + and (entries is None or relative.parts[0] in entries) + and relative.as_posix() not in skip + ), + key=lambda path: path.relative_to(submission_dir).as_posix(), ) for path in files: digest.update(path.relative_to(submission_dir).as_posix().encode()) diff --git a/packages/syft-job/tests/test_submission_hash.py b/packages/syft-job/tests/test_submission_hash.py new file mode 100644 index 00000000000..0cbe548865b --- /dev/null +++ b/packages/syft-job/tests/test_submission_hash.py @@ -0,0 +1,29 @@ +"""The submission hash must not depend on how the OS sorts paths. + +WindowsPath sorts case-insensitively and PosixPath case-sensitively, so a DS on +Windows and a DO on Linux would feed files in a different order. +""" + +import hashlib +from pathlib import Path + +from syft_job.submission import code_hash + + +def test_code_hash_orders_files_by_posix_path(tmp_path: Path): + files = { + "run.sh": b"python code/a.py\n", + "code/B.py": b"b\n", + "code/a.py": b"a\n", + } + for name, content in files.items(): + (tmp_path / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / name).write_bytes(content) + + expected = hashlib.sha256() + for name in sorted(files): # "code/B.py" < "code/a.py" < "run.sh" + expected.update(name.encode()) + expected.update(b"\0") + expected.update(hashlib.sha256(files[name]).digest()) + + assert code_hash(tmp_path) == expected.hexdigest()