diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.yml index 806f8b617b..bbc3c18177 100644 --- a/.github/workflows/engine-verify.yml +++ b/.github/workflows/engine-verify.yml @@ -89,6 +89,7 @@ jobs: python3 scripts/verify/plan.py test > /dev/null python3 scripts/verify/test_diff.py python3 scripts/verify/test_perf_build.py + python3 scripts/verify/test_mtime.py if grep -rn --include='expectations*.toml' '^reason = "TODO' crates; then echo "::error::blessed expectations still have TODO reasons: describe each failure (and link its fix) before merging" exit 1 diff --git a/scripts/verify/mtime.py b/scripts/verify/mtime.py index 379f658e70..4c5694e9ad 100644 --- a/scripts/verify/mtime.py +++ b/scripts/verify/mtime.py @@ -19,13 +19,21 @@ Only content equality is trusted, never commit times, so a stale build cannot be reused for changed code. + +A run builds only the crates its suites need, but the cache it saves still +holds every crate earlier runs built, some with other feature sets. Before +recording, a cached crate this run did not rebuild loses its cargo fingerprint +when any of its sources changed since the restored manifest, so the next run +that needs it rebuilds it instead of trusting the recorded blob ids. """ import os +import shutil import subprocess import sys OLD = 946684800 # 2000-01-01 +HEADER = "# mtime manifest v2" def blobs(): @@ -47,35 +55,218 @@ def blobs(): return entries +def load(manifest): + """The blob ids a trusted manifest records, or None. Manifests written + before stale crates were dropped (no header) are not trusted.""" + if not os.path.exists(manifest): + return None + with open(manifest, encoding="utf-8", errors="surrogateescape") as f: + lines = f.read().split("\n") + if not lines or lines[0] != HEADER: + return None + saved = {} + for line in lines[1:]: + if line: + blob, _, path = line.partition(" ") + saved[path] = blob + return saved + + +def dep_info_paths(dep_file, root): + """Repository-relative paths of the sources a rustc dep-info file lists.""" + with open(dep_file, encoding="utf-8", errors="surrogateescape") as f: + text = f.read() + paths = [] + for line in text.splitlines(): + if line.endswith(":") and ": " not in line: + entries = [line[:-1]] + elif ": " in line: + entries = [] + current = "" + rest = line.split(": ", 1)[1] + i = 0 + while i < len(rest): + if rest[i] == "\\" and i + 1 < len(rest) and rest[i + 1] == " ": + current += " " + i += 2 + continue + if rest[i] == " ": + if current: + entries.append(current) + current = "" + else: + current += rest[i] + i += 1 + if current: + entries.append(current) + else: + continue + for entry in entries: + full = os.path.normpath(os.path.join(root, entry)) + try: + relative = os.path.relpath(full, root) + except ValueError: + # Another drive on Windows: never a workspace file. + continue + if not relative.startswith(".."): + paths.append(relative.replace(os.sep, "/")) + return paths + + +def package_dirs(current): + """Directory of every workspace package, keyed by package name.""" + dirs = {} + for path in current: + if os.path.basename(path) != "Cargo.toml" or not os.path.isfile(path): + continue + section = None + with open(path, encoding="utf-8", errors="surrogateescape") as f: + for line in f: + line = line.strip() + if line.startswith("["): + section = line + elif section == "[package]" and line.startswith("name") and "=" in line: + name = line.split("=", 1)[1].strip().strip('"').strip("'") + dirs[name] = os.path.dirname(path) + break + return dirs + + +def build_script_inputs(output_file, package_dir): + """Repository-relative paths a build script run watches: its + rerun-if-changed paths, or its whole package when it names none.""" + watched = [] + with open(output_file, encoding="utf-8", errors="surrogateescape") as f: + for line in f: + for prefix in ("cargo::rerun-if-changed=", "cargo:rerun-if-changed="): + if line.startswith(prefix): + watched.append(line[len(prefix) :].rstrip("\r\n")) + root = os.getcwd() + paths = [] + for entry in watched or ["."]: + full = os.path.normpath(os.path.join(root, package_dir, entry)) + try: + relative = os.path.relpath(full, root) + except ValueError: + continue + if not relative.startswith(".."): + paths.append("" if relative == "." else relative.replace(os.sep, "/")) + return paths + + +def touches(paths, changed): + for path in paths: + if path in changed: + return True + prefix = path.rstrip("/") + "/" if path else "" + if any(item.startswith(prefix) for item in changed): + return True + return False + + +def drop_fingerprint(profile, suffix): + fingerprints = os.path.join(profile, ".fingerprint") + if os.path.isdir(fingerprints): + for fingerprint in os.listdir(fingerprints): + if fingerprint.endswith(suffix): + shutil.rmtree(os.path.join(fingerprints, fingerprint)) + + +def drop_stale_crates(target, changed, applied_at, packages): + """Removes the fingerprint of every cached crate, build script or build + script run this run did not redo whose inputs changed, so cargo redoes it + the next time it is needed.""" + root = os.getcwd() + dropped = 0 + for directory, subdirectories, files in os.walk(target): + parent = os.path.dirname(directory) + if os.path.basename(directory) == "deps": + profile = parent + elif os.path.basename(parent) == "build": + profile = os.path.dirname(parent) + else: + continue + subdirectories[:] = [] + if profile != parent and "output" in files: + output = os.path.join(directory, "output") + package, _, unit = os.path.basename(directory).rpartition("-") + if ( + os.path.getmtime(output) < applied_at + and package in packages + and touches(build_script_inputs(output, packages[package]), changed) + ): + drop_fingerprint(profile, "-" + unit) + os.remove(output) + dropped += 1 + for name in files: + if not name.endswith(".d") or "-" not in name: + continue + dep_file = os.path.join(directory, name) + if os.path.getmtime(dep_file) >= applied_at: + continue + if not any(path in changed for path in dep_info_paths(dep_file, root)): + continue + drop_fingerprint(profile, "-" + name[: -len(".d")].rsplit("-", 1)[1]) + os.remove(dep_file) + dropped += 1 + return dropped + + def main(): if len(sys.argv) != 3 or sys.argv[1] not in ("record", "apply"): sys.exit(__doc__) command, manifest = sys.argv[1], sys.argv[2] current = blobs() + stamp = manifest + ".applied" if command == "record": - os.makedirs(os.path.dirname(os.path.abspath(manifest)), exist_ok=True) + target = os.path.dirname(os.path.abspath(manifest)) + os.makedirs(target, exist_ok=True) + if os.path.exists(stamp): + previous = load(manifest) + changed = { + path + for path, blob in current.items() + if previous is None or previous.get(path) != blob + } + dropped = drop_stale_crates( + target, changed, os.path.getmtime(stamp), package_dirs(current) + ) + header = HEADER + print(f"dropped {dropped} cached crates built from changed sources") + else: + # Without the time of the apply step, crates rebuilt by this run + # cannot be told from stale ones: save a manifest the next run + # does not trust. + header = None + print("no apply stamp: the next run will not trust this manifest") with open(manifest, "w", encoding="utf-8", errors="surrogateescape") as f: + if header: + f.write(header + "\n") for path, blob in sorted(current.items()): f.write(f"{blob} {path}\n") print(f"recorded {len(current)} files in {manifest}") return - saved = None - if os.path.exists(manifest): - saved = {} - with open(manifest, encoding="utf-8", errors="surrogateescape") as f: - for line in f: - blob, _, path = line.rstrip("\n").partition(" ") - saved[path] = blob - elif os.path.isdir(os.path.dirname(manifest)) and os.listdir(os.path.dirname(manifest)): - # A cached build saved without a manifest: nothing says which sources - # it was built from, and the fixed time would make cargo reuse all of - # it. Every file gets the current time, so every crate rebuilds. - print(f"no manifest at {manifest} beside a cached build: rebuilding every crate") + try: + apply(manifest, current) + finally: + os.makedirs(os.path.dirname(os.path.abspath(stamp)), exist_ok=True) + with open(stamp, "w", encoding="utf-8"): + pass + + +def apply(manifest, current): + saved = load(manifest) + target = os.path.dirname(manifest) + if saved is None and os.path.isdir(target) and os.listdir(target): + # A cached build saved without a trusted manifest: nothing says which + # sources it was built from, and the fixed time would make cargo reuse + # all of it. Every file gets the current time, so every crate rebuilds. + print(f"no trusted manifest at {manifest} beside a cached build: rebuilding every crate") for path in current: if os.path.isfile(path) and not os.path.islink(path): os.utime(path, None) return - else: + if saved is None: # Cold build: give every file the fixed time too. Build scripts # without rerun-if-changed fingerprint the newest mtime of their # package by equality, so the build that gets cached must see the diff --git a/scripts/verify/test_mtime.py b/scripts/verify/test_mtime.py new file mode 100644 index 0000000000..2f396f0b2e --- /dev/null +++ b/scripts/verify/test_mtime.py @@ -0,0 +1,148 @@ +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).with_name("mtime.py").resolve() +MANIFEST = "target/ci-verify/verify-src-manifest.txt" + + +class BuildCacheManifestTests(unittest.TestCase): + """Each CI run restores the shared build cache, builds only the crates its + suites need, and saves the cache for the next run. A run must never reuse + a cached build of a crate whose sources changed after that build.""" + + def test_a_crate_another_run_did_not_rebuild_is_rebuilt_when_needed(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.workspace_with_feature(root) + self.write(root / "a" / "src" / "lib.rs", "pub fn base() -> u32 { 1 }\n") + env = dict(os.environ, CARGO_TARGET_DIR=str(root / "target" / "ci-verify")) + self.run_command(root, env, "git", "init", "--quiet") + self.run_command(root, env, "cargo", "generate-lockfile", "--offline") + + # A run that builds both crates and saves the cache. + self.ci_run(root, env, ["b", "c"]) + + # The next saving run gets a new library function but only needs + # `b`, so the build of `a` with feature `x` stays from before. + self.write(root / "a" / "src" / "lib.rs", "pub fn base() -> u32 { 1 }\npub fn added() -> u32 { 2 }\n") + self.ci_run(root, env, ["b"]) + + # A later run needs `c`, which now calls the new function. + self.write(root / "c" / "src" / "lib.rs", "pub fn c() -> u32 { a::added() }\n") + self.ci_run(root, env, ["c"]) + + def test_a_build_script_another_run_did_not_rebuild_reruns_when_needed(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.workspace_with_feature(root) + self.write(root / "a" / "build.rs", "fn main() {}\n") + self.write(root / "a" / "src" / "lib.rs", "#[cfg(added)]\npub fn added() -> u32 { 2 }\npub fn base() -> u32 { 1 }\n") + env = dict(os.environ, CARGO_TARGET_DIR=str(root / "target" / "ci-verify")) + self.run_command(root, env, "git", "init", "--quiet") + self.run_command(root, env, "cargo", "generate-lockfile", "--offline") + self.ci_run(root, env, ["b", "c"]) + + self.write(root / "a" / "build.rs", 'fn main() { println!("cargo:rustc-cfg=added"); }\n') + self.ci_run(root, env, ["b"]) + + self.write(root / "c" / "src" / "lib.rs", "pub fn c() -> u32 { a::added() }\n") + self.ci_run(root, env, ["c"]) + + def test_a_build_script_input_another_run_did_not_reread_is_reread_when_needed(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.workspace_with_feature(root) + self.write( + root / "a" / "build.rs", + 'fn main() {\n' + ' println!("cargo:rerun-if-changed=native/flags.txt");\n' + ' if std::fs::read_to_string("native/flags.txt").unwrap().contains("added") {\n' + ' println!("cargo:rustc-cfg=added");\n' + " }\n" + "}\n", + ) + self.write(root / "a" / "native" / "flags.txt", "\n") + self.write(root / "a" / "src" / "lib.rs", "#[cfg(added)]\npub fn added() -> u32 { 2 }\npub fn base() -> u32 { 1 }\n") + env = dict(os.environ, CARGO_TARGET_DIR=str(root / "target" / "ci-verify")) + self.run_command(root, env, "git", "init", "--quiet") + self.run_command(root, env, "cargo", "generate-lockfile", "--offline") + self.ci_run(root, env, ["b", "c"]) + + self.write(root / "a" / "native" / "flags.txt", "added\n") + self.ci_run(root, env, ["b"]) + + self.write(root / "c" / "src" / "lib.rs", "pub fn c() -> u32 { a::added() }\n") + self.ci_run(root, env, ["c"]) + + def workspace_with_feature(self, root): + self.write(root / ".gitignore", "target\n") + self.write(root / "Cargo.toml", '[workspace]\nmembers = ["a", "b", "c"]\nresolver = "2"\n') + self.write( + root / "a" / "Cargo.toml", + '[package]\nname = "a"\nversion = "0.1.0"\nedition = "2021"\n\n[features]\nx = []\n', + ) + for name, dependency in [("b", 'a = { path = "../a" }'), ("c", 'a = { path = "../a", features = ["x"] }')]: + self.write( + root / name / "Cargo.toml", + f'[package]\nname = "{name}"\nversion = "0.1.0"\nedition = "2021"\n\n[dependencies]\n{dependency}\n', + ) + self.write(root / "b" / "src" / "lib.rs", "pub fn b() -> u32 { a::base() }\n") + self.write(root / "c" / "src" / "lib.rs", "pub fn c() -> u32 { a::base() }\n") + + def test_unchanged_crates_stay_cached(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.write(root / "Cargo.toml", '[workspace]\nmembers = ["a", "b"]\nresolver = "2"\n') + self.write(root / "a" / "Cargo.toml", '[package]\nname = "a"\nversion = "0.1.0"\nedition = "2021"\n') + self.write( + root / "b" / "Cargo.toml", + '[package]\nname = "b"\nversion = "0.1.0"\nedition = "2021"\n\n[dependencies]\na = { path = "../a" }\n', + ) + self.write(root / "a" / "src" / "lib.rs", "pub fn base() -> u32 { 1 }\n") + self.write(root / "b" / "src" / "lib.rs", "pub fn b() -> u32 { a::base() }\n") + self.write(root / ".gitignore", "target\n") + env = dict(os.environ, CARGO_TARGET_DIR=str(root / "target" / "ci-verify")) + self.run_command(root, env, "git", "init", "--quiet") + self.run_command(root, env, "cargo", "generate-lockfile", "--offline") + + self.ci_run(root, env, ["b"]) + self.write(root / "b" / "src" / "lib.rs", "pub fn b() -> u32 { a::base() + 1 }\n") + output = self.ci_run(root, env, ["b"]) + self.assertIn("Compiling b ", output) + self.assertNotIn("Compiling a ", output) + + def ci_run(self, root, env, packages): + self.run_command(root, env, "git", "add", "-A") + # A checkout gives every file the current time. + for path in root.rglob("*"): + if path.is_file() and ".git" not in path.parts and "target" not in path.parts: + os.utime(path, None) + self.run_command(root, env, sys.executable, str(SCRIPT), "apply", MANIFEST) + arguments = ["cargo", "build", "--offline"] + for package in packages: + arguments += ["-p", package] + output = self.run_command(root, env, *arguments) + self.run_command(root, env, sys.executable, str(SCRIPT), "record", MANIFEST) + return output + + @staticmethod + def write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + def run_command(self, root, env, *arguments): + result = subprocess.run(arguments, cwd=root, env=env, capture_output=True, text=True) + self.assertEqual( + result.returncode, + 0, + f"{' '.join(arguments)} failed:\n{result.stdout}\n{result.stderr}", + ) + return result.stdout + result.stderr + + +if __name__ == "__main__": + unittest.main()