From aa2d4485b12356d7e4338a77899ac17c695652d4 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Thu, 1 Oct 2026 10:47:45 -0700 Subject: [PATCH 01/12] Add experimental ILLink task caching and cache maintenance Cache eligible whole-task invocations with environment-only opt-in and explicit timestamp-based purging. Bypass caching for extra arguments; runtime-specific argument support is deferred to the next layer. Assisted-by: Copilot:gpt-6-astra Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- docs/design/tools/illink/task-cache.md | 94 ++ docs/tools/illink/README.md | 1 + docs/tools/illink/illink-tasks.md | 110 ++ eng/Subsets.props | 3 + src/tools/illink/illink.slnx | 2 + .../illink/src/ILLink.CacheTool/CachePurge.cs | 59 + .../ILLink.CacheTool/ILLink.CacheTool.csproj | 22 + .../illink/src/ILLink.CacheTool/Program.cs | 54 + .../illink/src/ILLink.CacheTool/README.md | 32 + .../src/ILLink.Tasks/ILLink.Tasks.csproj | 2 + .../illink/src/ILLink.Tasks/ILLinkCache.cs | 355 ++++++ .../src/ILLink.Tasks/ILLinkCacheEntry.cs | 30 + src/tools/illink/src/ILLink.Tasks/LinkTask.cs | 125 +- .../ILLink.CacheTool.Tests/CachePurgeTests.cs | 236 ++++ .../ILLink.CacheTool.Tests.csproj | 5 + .../ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs | 1074 +++++++++++++++++ .../ILLink.Tasks.Tests.csproj | 1 + .../illink/test/ILLink.Tasks.Tests/Mock.cs | 22 +- 18 files changed, 2225 insertions(+), 2 deletions(-) create mode 100644 docs/design/tools/illink/task-cache.md create mode 100644 src/tools/illink/src/ILLink.CacheTool/CachePurge.cs create mode 100644 src/tools/illink/src/ILLink.CacheTool/ILLink.CacheTool.csproj create mode 100644 src/tools/illink/src/ILLink.CacheTool/Program.cs create mode 100644 src/tools/illink/src/ILLink.CacheTool/README.md create mode 100644 src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs create mode 100644 src/tools/illink/src/ILLink.Tasks/ILLinkCacheEntry.cs create mode 100644 src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs create mode 100644 src/tools/illink/test/ILLink.CacheTool.Tests/ILLink.CacheTool.Tests.csproj diff --git a/docs/design/tools/illink/task-cache.md b/docs/design/tools/illink/task-cache.md new file mode 100644 index 00000000000000..6fac3a8e50b428 --- /dev/null +++ b/docs/design/tools/illink/task-cache.md @@ -0,0 +1,94 @@ +# ILLink task cache: v1 decisions + +These are design decisions. Eligible task invocations compute a content-based key, restore a cached output directory on a hit, or run ILLink and store successful results. + +The feature is experimental. Its configuration and behavior may change or be removed without notice; it is not a supported compatibility surface. + +## Scope and configuration + +- Cache complete ILLink invocations in the MSBuild task, not in the linker command-line implementation. +- Keep v1 simple. Reuse results at the same input/output paths; defer cross-worktree reuse and retain absolute paths in cache identity. +- Enable caching only through the `ILLINK_EXPERIMENTAL_CACHE` environment variable set to `true` (case-insensitive). Unset, empty, or `false` disables caching; invalid boolean values disable caching with a diagnostic. Disabled caching performs no cache I/O, even if a cache directory already exists. +- Use the optional `ILLINK_EXPERIMENTAL_CACHE_PATH` environment variable for the directory override. Unset or empty selects the platform default. Selecting a directory does not enable caching, and path values are never interpreted as booleans. +- Use platform-specific default directories: `$XDG_CACHE_HOME/illink` on Linux/other XDG Unix, falling back to `$HOME/.cache/illink`; `$HOME/Library/Caches/illink` on macOS; `LocalApplicationData/illink` on Windows. +- Resolve defaults inside `ILLinkCache.TryCreate` through a private helper, not in targets. Resolve relative overrides against the current working directory. +- Ignore relative `XDG_CACHE_HOME` values. If no usable user location is available, skip caching with a diagnostic rather than fall back to the working directory or shared temporary storage. +- Read configuration from the task process's environment on each invocation. Set it before starting MSBuild; do not change process-wide environment settings during parallel builds. Reused build processes must receive the intended environment. There are no cache task parameters, MSBuild property wiring, or linker command-line switches. +- Do not promise stability or migration support for the on-disk format. + +## API shape + +```csharp +internal sealed class ILLinkCache +{ + internal static ILLinkCache? TryCreate(string? cacheDirectory, TaskLoggingHelper log); + internal bool TryRestore(string inputHash, string outputDirectory, DateTime outputTimestampUtc); + internal void Store(string inputHash, string outputDirectory); +} +``` + +- The task owns input identity and execution. Its private `TryComputeCacheKey` combines command-line and response-file arguments with input-content and toolchain identity; inability to compute a key bypasses caching with a diagnostic. +- `TryCreate` resolves the cache directory and returns null with a diagnostic if initialization cannot proceed. +- `TryRestore` combines lookup and restoration. True means complete restoration. Missing entries and expected I/O, access, or invalid-cache-data failures return false with appropriate logging and fall back to normal linking. Copies overwrite existing files and are not rolled back if a later copy fails. Unexpected exceptions propagate. +- `Store` is best-effort. Lock contention and existing entries are normal skips; expected cache I/O failures are logged without failing a successful link. Unexpected exceptions propagate. +- Expected restore/store failures are `IOException`, `UnauthorizedAccessException`, `InvalidDataException`, and `FormatException`, not every exception. Invalid helper arguments remain programming errors. +- After normal task validation, resolve the cache directory before computing the key so an unusable configuration does not trigger input inspection or hashing. If resolution succeeds, compute the key once and attempt restoration. On a hit, skip linking; otherwise link normally and store only with exit code zero and no logged errors. A failed restore never substitutes for a successful fallback link. + +## Execution and publication + +- Cache operations do not support cancellation in v1, including key computation, validation, restoration, and storage. Assume they are fast relative to actual linker execution, and keep synchronous filesystem operations without cancellation polling or chunked cancellable I/O. A cancellation request may arrive while these operations finish, and a cache hit may return success without observing it. Normal linker execution retains `ToolTask`'s existing subprocess cancellation behavior. +- Check the cache before normal tool execution. On a hit, validate the complete entry, then delete/recreate the destination and copy the cached tree. On any miss or restore failure, the normal execution path deletes/recreates the directory before running ILLink; no separate rollback or failure-cleanup path is needed. +- The task owns its output directory, including when caching is disabled. Normal-execution cleanup failures fail the task. Rely on the supplied `OutputDirectory` parameter without additional task path/input validation; callers must provide a dedicated directory. The command-line linker retains its existing nonempty-directory behavior. +- ILLink writes to its normal output directory. After successful linking, copy results into the cache; do not redirect linker output to a temporary directory. +- Let cache misses compute independently. Scheduling and execution times are nondeterministic, and steps can fail or be aborted; one computation must not block another's progress. +- Use a unique cache staging directory for each store attempt, including attempts with the same input hash. +- Acquire a per-input-hash writer lock, scoped to the cache directory, after linking and before staging copies. +- Acquire the lock nonblocking. If busy, skip storing rather than wait; this avoids duplicate cache-copy work, not duplicate linking. The losing task keeps its own computed outputs and does not wait or retry if the winning writer later fails or is aborted. +- Under the lock, recheck whether the entry already exists. If absent, populate staging and publish with an atomic, same-filesystem directory rename without overwriting an existing entry. +- Rely on the documented `Directory.Move` contract that an existing destination causes `IOException`, even if that destination is empty. The writer lock avoids duplicate staging work; it is not a workaround for publication semantics. Do not add platform-specific interop or require non-empty entries to compensate for runtime/filesystem bugs. Atomic visibility does not imply power-loss durability. +- Create cache directories lazily in `Store`, after acquiring the writer lock and confirming the entry is absent. `TryCreate` only resolves the path; `TryRestore` treats a missing directory as a miss. Directory-creation failures are logged without failing a successful link. +- Keep published outputs and manifests immutable and read them without locks. Write a UTC `last-used` timestamp in staging before publication and refresh it after a successful restore. Replace the marker atomically so parallel restores do not leave partial timestamps; usage-update failures are logged without failing linking. +- Use a named `System.Threading.Mutex` derived from the resolved cache-directory path and input hash. An abandoned mutex grants ownership; recheck the entry and use fresh staging. This coordinates processes on one machine, not writers on different machines sharing a network directory. +- Support concurrent task invocations with distinct output directories sharing a cache directory. Concurrent linking to the same output directory is unsupported. +- Treat missing or unreadable cache data as a failed restore, never a successful hit. +- Start without replaying warnings or other linker diagnostics on cache hits. This is an intentional v1 limitation. + +## Input identity and eligibility + +- Use SHA-256 over a versioned, length-delimited description. Preserve argument ordering and absolute paths, including the working and output directories. +- Hash the contents of all supplied assemblies, reference assemblies, and root descriptors. Include adjacent PDBs, MDBs, configuration files, and satellite assemblies, even when symbol emission is disabled. Follow assembly metadata file entries for linked resources and additional modules. Optional-file additions and removals change the key; timestamps alone do not. +- Hash the entire linker assembly on every invocation, without loading it or using its module version ID (MVID). Content changes invalidate the cache even when the MVID, file length, and timestamp are unchanged. Do not scan its directory. Bundled dependencies and configuration are not fingerprinted: changes to Cecil or deployment configuration require a cleared/isolated cache or disabled caching unless the linker binary also changes. +- Continue hashing the task assembly and selected dotnet host executable. Do not enumerate or hash the host's installed hostfxr/shared-runtime trees: doing so makes lookup cost scale with every installed framework/version, including ones unrelated to the link. Runtime/tool files supplied as linker inputs remain content-hashed. +- Defer identifying the runtime that executes ILLink; do not include a runtime version or framework description in the key. The task runs in MSBuild, so its CoreLib informational version and framework description are not reliable identities for the child linker's runtime. The runtimeconfig describes requested frameworks and roll-forward policy, not the exact resolved CoreLib. Finding that CoreLib would require resolved-runtime information from the caller, host-resolution plumbing, or an additional process; defer that complexity and startup cost. +- This deliberately accepts that runtime installation changes not represented by other keyed inputs may reuse an entry; callers needing invalidation for those changes must clear or isolate the cache. +- Include platform, architecture, and culture. Do not enumerate or hash inherited environment variables, or bypass caching based on them. This deliberately keeps v1 simple rather than maintaining a partial list of runtime/loader settings. Environment changes are reflected only when they change other keyed inputs, such as the selected host path or culture. Callers must disable caching when inherited settings introduce otherwise untracked dependencies, affect outputs, or require tool-execution side effects (for example, startup hooks or profilers). +- Bypass caching for non-whitespace `ExtraArgs`, custom steps/data, dependency-dump options, and explicit `ToolTask.EnvironmentVariables` overrides. These are caller-supplied inputs that v1 does not model in its key and can introduce undeclared dependencies, external outputs, or side effects. These invocations still run the linker normally; no extra-argument parsing or runtime-specific exceptions are supported. +- Log and bypass caching if required files cannot be read or an input assembly cannot be inspected, including PE images without managed metadata. Normal linking determines whether those inputs are acceptable. Input files must remain stable during hashing and linking. + +## Entry layout + +- Store entries under `/v1//`, with a binary `manifest`, an `outputs/` tree, and a `last-used` UTC timestamp in invariant round-trip (`O`) text format. +- The manifest records the format version, output paths, lengths, and SHA-256 content hashes. Validate it and every listed file before starting restoration. +- Copy the complete output tree without special-casing filenames. The MSBuild target owns `.semaphore` (normally `linked.semaphore`) and invalidates it before the incremental skip decision when that directory is missing. There is no special migration or exclusion for old in-directory `Link.semaphore` files; ordinary whole-directory cleanup and caching apply to them. Empty directories are not recorded. +- The restore helper replaces the destination tree after entry validation. Copy failures can leave partial results, which the task's normal execution cleanup removes before fallback linking. Publication is atomic; restoration is not. +- Assume ILLink outputs are not symbolic links; do not check for them. Cache and output directories must not contain each other. +- Stamp restored files with the UTC timestamp captured before input-key inspection. This avoids stale cache timestamps in downstream incremental checks without making outputs appear newer than inputs changed during inspection or restoration. + +## Cache lifetime + +- No automatic eviction or size policy during linking. Users may manually wipe the cache directory or explicitly run the experimental `dotnet-illink-cache purge` tool. +- Do not repair or replace corrupt published entries. Restoration falls back to linking, but storage still skips an existing entry until the user removes it. +- No `created` bookkeeping marker. The `last-used` marker records usage for explicit post-build maintenance. +- Purge requires an explicit cache root and UTC cutoff. Only published SHA-256 entry directories in `v1` are considered; delete entries whose marker is strictly older than the cutoff and retain equality. Leave staging directories and other layouts alone. Report per-entry errors and continue, then return nonzero if maintenance failed. +- Assume old caches are cleared before using markers. Do not infer ages or migrate entries with missing markers; missing, malformed, or unreadable markers are ordinary reported errors. +- Purge must not run during any build using the cache. Do not add purge locks or change reader locking; coordination with active readers/writers remains a separate follow-up, as does protection against manual deletion. +- CI may record build start after restoring a job-private cache, run linking, then purge against that timestamp before uploading the next snapshot. Preserve tool diagnostics but do not fail the build for maintenance errors. Only purge once all users have exited; incremental skips and unreached work in failed builds do not refresh usage. See the [CI recipe](../../../tools/illink/illink-tasks.md#ci-purge-ordering). This does not enable an ILLink cache in CI automatically. +- Attempt best-effort cleanup of the current attempt's unpublished staging directory. Log cleanup I/O/access failures without failing a successful link; do not scan or delete other attempts' staging directories. Correctness must tolerate leftovers after crashes. + +## Open follow-ups + +- Revisit identifying the resolved runtime/CoreLib that executes ILLink without adding substantial cache-lookup overhead. +- Revisit symbolic-link handling when generalizing beyond ILLink outputs. +- Revisit cross-process shared-reader/exclusive-purge coordination if concurrent maintenance becomes necessary. +- Reconsider diagnostic replay, including missing console/binlog warnings and effects on external warning-as-error policies. +- Examine manual wiping racing reads, including safe fallback after partially restoring output files. diff --git a/docs/tools/illink/README.md b/docs/tools/illink/README.md index c837328dbdd73c..864c6fed5e0f68 100644 --- a/docs/tools/illink/README.md +++ b/docs/tools/illink/README.md @@ -14,3 +14,4 @@ View the official trimming docs at [learn.microsoft.com](https://learn.microsoft ## Design Docs - [Reflection Handling](../../design/tools/illink/reflection-flow.md) +- [Task Cache](../../design/tools/illink/task-cache.md) diff --git a/docs/tools/illink/illink-tasks.md b/docs/tools/illink/illink-tasks.md index 94a6034e0ddb93..b13c6168e719d6 100644 --- a/docs/tools/illink/illink-tasks.md +++ b/docs/tools/illink/illink-tasks.md @@ -26,6 +26,116 @@ alternatively you can edit your .csproj file to include The output will include only necessary code to run your application. The framework libraries size will be reduced noticeably. +## Experimental task cache + +> [!WARNING] +> This feature is experimental. Its configuration and behavior may change or be removed without notice. +> Neither the configuration nor the on-disk format is a supported compatibility surface. + +Set `ILLINK_EXPERIMENTAL_CACHE=true` in the environment before starting MSBuild to enable caching. +The boolean value is case-insensitive; unset, empty, or `false` disables caching. Invalid boolean +values disable caching with a diagnostic. Set `ILLINK_EXPERIMENTAL_CACHE_PATH` to override the +cache directory. Setting a path alone does not enable caching, and path values are never +interpreted as booleans. These settings apply only to the ILLink task, not the linker CLI or ILC. + +For example, in a POSIX shell: + +```sh +ILLINK_EXPERIMENTAL_CACHE=true ILLINK_EXPERIMENTAL_CACHE_PATH=/path/to/cache \ + dotnet publish -r -c Release -p:PublishTrimmed=true +``` + +These are environment variables, not MSBuild properties or task parameters. The task reads +its process environment on each invocation; do not change it during parallel builds, and ensure +reused build processes receive the intended environment. + +The cache helper resolves an unset or empty directory override to `$XDG_CACHE_HOME/illink` (or +`$HOME/.cache/illink`) on Linux/other XDG Unix, `$HOME/Library/Caches/illink` on macOS, +and `LocalApplicationData/illink` on Windows. Relative `XDG_CACHE_HOME` values are ignored; +explicit relative cache directories are resolved against the working directory. + +Eligible invocations use a key covering arguments, input-file and sidecar contents, the task +assembly, the selected dotnet host executable, and the entire ILLink assembly. +File contents are hashed on every invocation. Linker changes invalidate the cache even when +the module version ID (MVID), file length, and timestamp are unchanged. +Bundled dependencies and configuration are not fingerprinted. Changes to those files require +a cleared/isolated cache or disabled caching unless the linker binary also changes. + +The key does not cover the host's installed runtime directories or the version of the runtime +executing ILLink. Runtime installation changes alone are not guaranteed to invalidate entries; +clear or isolate the cache when that distinction is required. +Cache hits replace the output directory without running ILLink. +Directories are created in the cache only when storing a successful result. Hits do not replay warnings or other linker diagnostics. + +Caching is bypassed with a diagnostic for non-whitespace `ExtraArgs`, custom steps/data, +dependency-dump options, and explicit task environment overrides. Options supplied through +`ExtraArgs` remain unsupported for caching; these invocations still run the linker normally. +Inherited environment variables are not tracked; disable caching +when they affect outputs or dependencies beyond the keyed inputs, or require tool-execution +side effects such as startup hooks or profiling. Unreadable inputs or linker files also +fall back to normal linking. See the +[cache design](../../design/tools/illink/task-cache.md) for identity and eligibility details. + +### Cache maintenance + +The experimental `dotnet-illink-cache` .NET tool can purge entries that were not used recently: + +```sh +dotnet illink-cache purge --cache-directory /path/to/cache --before 2026-10-01T12:00:00Z +``` + +The directory and UTC cutoff are required; maintenance does not use the task's opt-in or +directory environment variables. Use the tool built from the same revision as the task. +See the [tool README](../../../src/tools/illink/src/ILLink.CacheTool/README.md) for local +packaging and installation. + +Each entry has a `last-used` timestamp, initialized during publication and refreshed after +a successful cache hit. Only entries strictly older than the cutoff are deleted. Staging +directories, unrelated directories, and other layout versions are not purged. There is no +size limit or automatic eviction during linking. + +**Run purge only when no build is using that cache.** Purge and manual deletion are not +coordinated with readers or writers. Clear old caches before adopting usage markers; no +migration or fallback age is provided. Missing, malformed, or unreadable markers are reported +as errors and their entries are retained. The tool reports deleted/kept/error counts and +returns nonzero for argument or maintenance failures. Usage-marker update failures in the +task are logged without failing linking. + +### CI purge ordering + +For a CI cache snapshot, use the build-start timestamp as the cutoff to retain entries used +or created by that build: + +1. Restore the cache into a job-private directory and set `ILLINK_EXPERIMENTAL_CACHE=true` + and `ILLINK_EXPERIMENTAL_CACHE_PATH` to that explicit directory. +2. Record the UTC timestamp before starting any linking. In a POSIX shell: + ```sh + export ILLINK_CACHE_BUILD_START="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + ``` +3. Run the build and wait for all processes using the cache to finish. +4. Run the tool from the same revision, with maintenance failures reported but nonfatal: + ```sh + if ! dotnet illink-cache purge \ + --cache-directory "$ILLINK_EXPERIMENTAL_CACHE_PATH" \ + --before "$ILLINK_CACHE_BUILD_START"; then + echo "Warning: ILLink cache purge failed; see the tool diagnostics above." >&2 + fi + ``` +5. Save/upload the resulting cache directory only after purge finishes. For Azure Pipelines + `Cache@2`, the purge step must precede its post-job save. + +If the build and purge run in different CI steps, persist `ILLINK_CACHE_BUILD_START` as a +pipeline variable; a shell export alone does not carry it across steps. This variable is +only for the CI recipe, not another task configuration setting. + +A failed build can still be followed by purge once all linking has stopped, but entries for +work that was never reached will count as unused. Likewise, incremental builds that skip +linking do not refresh cache usage. Skip purge if the cache may still be in use, including +during cancellation. Do not share a live directory with another job while purging. + +This is usage guidance, not automatic CI enablement. ILLink cache restore/upload wiring +is separate from runtime's Roslyn/csc cache maintenance. + ## ILLink Task Properties ### ExtraArgs diff --git a/eng/Subsets.props b/eng/Subsets.props index a6b04e2beaaa05..9fe02784382057 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -566,6 +566,7 @@ + @@ -574,6 +575,8 @@ + + @@ -20,6 +21,7 @@ + diff --git a/src/tools/illink/src/ILLink.CacheTool/CachePurge.cs b/src/tools/illink/src/ILLink.CacheTool/CachePurge.cs new file mode 100644 index 00000000000000..af2d3e18503a5f --- /dev/null +++ b/src/tools/illink/src/ILLink.CacheTool/CachePurge.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Globalization; +using System.IO; +using ILLink.Tasks; + +namespace ILLink.CacheTool; + +internal static class CachePurge +{ + internal static int Run(string cacheDirectory, DateTimeOffset cutoff, TextWriter output, TextWriter error) + { + int deleted = 0; + int kept = 0; + int errors = 0; + string entriesDirectory = Path.Combine(cacheDirectory, ILLinkCacheEntry.VersionDirectory); + try + { + foreach (string entry in Directory.EnumerateDirectories(entriesDirectory)) + { + if (!ILLinkCacheEntry.IsEntryName(Path.GetFileName(entry))) + continue; + + try + { + DateTimeOffset lastUsed = DateTimeOffset.ParseExact( + File.ReadAllText(Path.Combine(entry, ILLinkCacheEntry.LastUsedFileName)), + "O", CultureInfo.InvariantCulture); + if (lastUsed.Offset != TimeSpan.Zero) + throw new InvalidDataException("The last-used timestamp must be UTC."); + + if (lastUsed >= cutoff) + { + kept++; + continue; + } + + Directory.Delete(entry, recursive: true); + deleted++; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or FormatException or InvalidDataException) + { + errors++; + error.WriteLine($"Could not purge ILLink cache entry '{entry}': {ex.Message}"); + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + errors++; + error.WriteLine($"Could not enumerate ILLink cache '{entriesDirectory}': {ex.Message}"); + } + + output.WriteLine($"ILLink cache purge: Deleted: {deleted}, Kept: {kept}, Errors: {errors}"); + return errors == 0 ? 0 : 1; + } +} diff --git a/src/tools/illink/src/ILLink.CacheTool/ILLink.CacheTool.csproj b/src/tools/illink/src/ILLink.CacheTool/ILLink.CacheTool.csproj new file mode 100644 index 00000000000000..6a6d45539fb368 --- /dev/null +++ b/src/tools/illink/src/ILLink.CacheTool/ILLink.CacheTool.csproj @@ -0,0 +1,22 @@ + + + dotnet-illink-cache + Exe + false + Major + true + false + true + dotnet-illink-cache + dotnet-illink-cache + Experimental ILLink task cache maintenance tool. + README.md + + + + + + + + + diff --git a/src/tools/illink/src/ILLink.CacheTool/Program.cs b/src/tools/illink/src/ILLink.CacheTool/Program.cs new file mode 100644 index 00000000000000..b3fe7a5ba17113 --- /dev/null +++ b/src/tools/illink/src/ILLink.CacheTool/Program.cs @@ -0,0 +1,54 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.CommandLine; +using System.Globalization; +using System.IO; + +namespace ILLink.CacheTool; + +internal static class Program +{ + private static int Main(string[] args) => Run(args, Console.Out, Console.Error); + + internal static int Run(string[] args, TextWriter output, TextWriter error) + { + var command = new RootCommand("Experimental ILLink task cache maintenance. Commands and cache formats may change without notice."); + var directory = new Option("--cache-directory") + { + Description = "Cache root directory. No builds may use this cache while purging.", + Required = true + }.AcceptExistingOnly(); + var before = new Option("--before") + { + Description = "Delete entries last used before this UTC ISO 8601 timestamp (Z or +00:00).", + Required = true, + CustomParser = result => + { + string value = result.Tokens[0].Value; + if (DateTimeOffset.TryParseExact(value, + new[] { "yyyy-MM-dd'T'HH:mm:ss.FFFFFFF'Z'", "yyyy-MM-dd'T'HH:mm:ss.FFFFFFFzzz" }, + CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset cutoff) && + cutoff.Offset == TimeSpan.Zero) + { + return cutoff; + } + + result.AddError("--before must be a UTC ISO 8601 timestamp ending in Z or +00:00."); + return default; + } + }; + var purge = new Command("purge", "Delete unused entries from an idle ILLink task cache."); + purge.Options.Add(directory); + purge.Options.Add(before); + purge.SetAction(result => CachePurge.Run(result.GetValue(directory)!.FullName, result.GetValue(before), output, error)); + command.Subcommands.Add(purge); + return command.Parse(args).Invoke(new() + { + EnableDefaultExceptionHandler = false, + Output = output, + Error = error + }); + } +} diff --git a/src/tools/illink/src/ILLink.CacheTool/README.md b/src/tools/illink/src/ILLink.CacheTool/README.md new file mode 100644 index 00000000000000..08d9f4c43a9566 --- /dev/null +++ b/src/tools/illink/src/ILLink.CacheTool/README.md @@ -0,0 +1,32 @@ +# Experimental ILLink cache maintenance + +`dotnet-illink-cache` is a .NET tool for maintaining the experimental ILLink MSBuild task cache. +Its commands and the cache format may change without notice. Use the tool built from the same +revision as the task. It does not manage Roslyn/csc or native compiler caches. + +```sh +dotnet illink-cache purge --cache-directory /path/to/cache --before 2026-10-01T12:00:00Z +``` + +Both options are required. The cutoff must be an ISO 8601 UTC timestamp, with `Z` or `+00:00`. +Entries last used strictly before the cutoff are deleted; entries at or after it are kept. +The tool prints deleted, kept, and error counts. Argument or maintenance errors return a +nonzero exit code. Missing or invalid usage markers are reported as errors, not assigned an age. + +**No build may use the cache during purging.** The tool does not coordinate with active +readers or writers. Staging directories and unrelated directories are left alone. +Clear caches created before usage markers were introduced; there is no migration support. + +## Building locally + +From the repository root: + +```sh +./dotnet.sh pack src/tools/illink/src/ILLink.CacheTool/ILLink.CacheTool.csproj -c Release +``` + +The package is produced under `artifacts/packages/Release/NonShipping`. Install the exact +locally built version with `dotnet tool install --tool-path +--add-source artifacts/packages/Release/NonShipping dotnet-illink-cache --version `, +then add `` to `PATH` or invoke its `dotnet-illink-cache` executable directly. +Alternatively, run the built `dotnet-illink-cache.dll` with `dotnet` directly. diff --git a/src/tools/illink/src/ILLink.Tasks/ILLink.Tasks.csproj b/src/tools/illink/src/ILLink.Tasks/ILLink.Tasks.csproj index 7dc232e2fd87d3..b6e1f6b1a7e161 100644 --- a/src/tools/illink/src/ILLink.Tasks/ILLink.Tasks.csproj +++ b/src/tools/illink/src/ILLink.Tasks/ILLink.Tasks.csproj @@ -50,6 +50,8 @@ + + diff --git a/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs b/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs new file mode 100644 index 00000000000000..1e302c4c1b5c6e --- /dev/null +++ b/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs @@ -0,0 +1,355 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace ILLink.Tasks; + +internal sealed class ILLinkCache +{ + private const int FormatVersion = 1; + private readonly TaskLoggingHelper _log; + private static readonly StringComparison s_pathComparison = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + + internal string CacheDirectory { get; } + + private ILLinkCache(string cacheDirectory, TaskLoggingHelper log) + { + CacheDirectory = cacheDirectory; + _log = log; + } + + internal static ILLinkCache? TryCreateFromEnvironment(TaskLoggingHelper log) + { + string? setting = Environment.GetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE"); + if (string.IsNullOrEmpty(setting)) + return null; + + if (!bool.TryParse(setting, out bool enabled)) + { + log.LogMessage(MessageImportance.Low, "ILLink caching is disabled because ILLINK_EXPERIMENTAL_CACHE must be 'true' or 'false'."); + return null; + } + + return enabled ? TryCreate(Environment.GetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE_PATH"), log) : null; + } + + internal static ILLinkCache? TryCreate(string? cacheDirectory, TaskLoggingHelper log) + { + if (string.IsNullOrEmpty(cacheDirectory)) + cacheDirectory = GetDefaultCacheDirectory(); + + if (cacheDirectory is null) + { + log.LogMessage(MessageImportance.Low, "ILLink caching is disabled because no default cache directory is available."); + return null; + } + + try + { + return new ILLinkCache(Path.GetFullPath(cacheDirectory), log); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or IOException or UnauthorizedAccessException) + { + log.LogMessage(MessageImportance.Low, $"ILLink caching is disabled because the cache directory '{cacheDirectory}' could not be resolved: {ex.Message}"); + return null; + } + } + + private static string? GetDefaultCacheDirectory() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + string localAppData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData, Environment.SpecialFolderOption.DoNotVerify); + return string.IsNullOrEmpty(localAppData) || !Path.IsPathRooted(localAppData) + ? null + : Path.Combine(localAppData, "illink"); + } + + bool isMacOS = RuntimeInformation.IsOSPlatform(OSPlatform.OSX); + if (!isMacOS) + { + string? cacheHome = Environment.GetEnvironmentVariable("XDG_CACHE_HOME"); + if (!string.IsNullOrEmpty(cacheHome) && Path.IsPathRooted(cacheHome)) + return Path.Combine(cacheHome, "illink"); + } + + string home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile, Environment.SpecialFolderOption.DoNotVerify); + if (string.IsNullOrEmpty(home) || !Path.IsPathRooted(home)) + return null; + + return isMacOS + ? Path.Combine(home, "Library", "Caches", "illink") + : Path.Combine(home, ".cache", "illink"); + } + + internal bool TryRestore(string inputHash, string outputDirectory, DateTime outputTimestampUtc) + { + string entryDirectory = GetEntryDirectory(inputHash); + outputDirectory = ValidateOutputDirectory(outputDirectory); + if (outputTimestampUtc.Kind != DateTimeKind.Utc) + throw new ArgumentException("The output timestamp must be UTC.", nameof(outputTimestampUtc)); + + if (!Directory.Exists(entryDirectory)) + { + _log.LogMessage(MessageImportance.Low, $"ILLink cache miss: {inputHash}"); + return false; + } + + try + { + List files = ReadManifest(entryDirectory); + + // Check the entire entry before modifying outputs. Copies can still fail if the cache is wiped concurrently. + if (Directory.Exists(outputDirectory)) + Directory.Delete(outputDirectory, recursive: true); + + Directory.CreateDirectory(outputDirectory); + foreach (string relativePath in files) + { + string destination = Path.Combine(outputDirectory, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + File.Copy(Path.Combine(entryDirectory, "outputs", relativePath), destination, overwrite: true); + File.SetLastWriteTimeUtc(destination, outputTimestampUtc); + } + + UpdateLastUsed(entryDirectory); + _log.LogMessage(MessageImportance.Low, $"ILLink cache hit: {inputHash}"); + return true; + } + catch (Exception ex) when (IsCacheFailure(ex)) + { + _log.LogMessage(MessageImportance.Low, $"ILLink cache restore failed for '{inputHash}'; running ILLink normally: {ex.Message}"); + return false; + } + } + + internal void Store(string inputHash, string outputDirectory) + { + string entryDirectory = GetEntryDirectory(inputHash); + outputDirectory = ValidateOutputDirectory(outputDirectory); + string? stagingDirectory = null; + + try + { + using var mutex = new Mutex(initiallyOwned: false, GetWriterMutexName(inputHash)); + bool acquired; + try + { + acquired = mutex.WaitOne(0); + } + catch (AbandonedMutexException) + { + acquired = true; + _log.LogMessage(MessageImportance.Low, $"ILLink cache writer lock was abandoned: {inputHash}"); + } + + if (!acquired) + { + _log.LogMessage(MessageImportance.Low, $"ILLink cache store skipped; another writer owns '{inputHash}'."); + return; + } + + try + { + if (Directory.Exists(entryDirectory)) + { + _log.LogMessage(MessageImportance.Low, $"ILLink cache store skipped; entry '{inputHash}' already exists."); + return; + } + + var files = new List(); + CollectOutputFiles(outputDirectory, "", files); + files.Sort(StringComparer.Ordinal); + + string parentDirectory = Path.GetDirectoryName(entryDirectory)!; + Directory.CreateDirectory(parentDirectory); + stagingDirectory = Path.Combine(parentDirectory, inputHash + "." + Guid.NewGuid().ToString("N") + ".tmp"); + Directory.CreateDirectory(stagingDirectory); + + using (var manifest = new BinaryWriter(File.Create(Path.Combine(stagingDirectory, "manifest")))) + { + manifest.Write(FormatVersion); + manifest.Write(files.Count); + foreach (string relativePath in files) + { + string cachedFile = Path.Combine(stagingDirectory, "outputs", relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(cachedFile)!); + File.Copy(Path.Combine(outputDirectory, relativePath), cachedFile); + manifest.Write(relativePath); + manifest.Write(new FileInfo(cachedFile).Length); + manifest.Write(HashFile(cachedFile)); + } + } + + ILLinkCacheEntry.WriteLastUsed(Path.Combine(stagingDirectory, ILLinkCacheEntry.LastUsedFileName)); + Directory.Move(stagingDirectory, entryDirectory); + stagingDirectory = null; + _log.LogMessage(MessageImportance.Low, $"ILLink cache stored: {inputHash}"); + } + finally + { + mutex.ReleaseMutex(); + } + } + catch (Exception ex) when (IsCacheFailure(ex)) + { + _log.LogMessage(MessageImportance.Low, $"ILLink cache store failed for '{inputHash}': {ex.Message}"); + } + finally + { + if (stagingDirectory is not null) + { + try + { + if (Directory.Exists(stagingDirectory)) + Directory.Delete(stagingDirectory, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _log.LogMessage(MessageImportance.Low, $"ILLink cache staging cleanup failed for '{stagingDirectory}': {ex.Message}"); + } + } + } + } + + private void UpdateLastUsed(string entryDirectory) + { + string temporaryFile = Path.Combine(entryDirectory, ILLinkCacheEntry.LastUsedFileName + "." + Guid.NewGuid().ToString("N") + ".tmp"); + try + { + ILLinkCacheEntry.WriteLastUsed(temporaryFile); + // Replace rather than overwrite so parallel restores cannot leave a partially written marker. + File.Replace(temporaryFile, Path.Combine(entryDirectory, ILLinkCacheEntry.LastUsedFileName), null); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _log.LogMessage(MessageImportance.Low, $"ILLink cache last-used update failed for '{entryDirectory}': {ex.Message}"); + } + finally + { + try + { + File.Delete(temporaryFile); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _log.LogMessage(MessageImportance.Low, $"ILLink cache last-used cleanup failed for '{temporaryFile}': {ex.Message}"); + } + } + } + + private string GetEntryDirectory(string inputHash) + { + if (!ILLinkCacheEntry.IsEntryName(inputHash)) + throw new ArgumentException("The input hash must be a lowercase SHA-256 hex string.", nameof(inputHash)); + + return Path.Combine(CacheDirectory, ILLinkCacheEntry.VersionDirectory, inputHash); + } + + private string GetWriterMutexName(string inputHash) + { + string directory = CacheDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + directory = directory.ToUpperInvariant(); + + using var sha256 = SHA256.Create(); + byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(directory + "\n" + inputHash)); + return @"Global\ILLinkCache-" + BitConverter.ToString(hash).Replace("-", ""); + } + + private string ValidateOutputDirectory(string outputDirectory) + { + string outputPath = Path.GetFullPath(outputDirectory); + string cachePrefix = CacheDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; + string outputPrefix = outputPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; + if (cachePrefix.StartsWith(outputPrefix, s_pathComparison) || outputPrefix.StartsWith(cachePrefix, s_pathComparison)) + throw new ArgumentException("The cache and output directories must not contain each other.", nameof(outputDirectory)); + + return outputPath; + } + + private static void CollectOutputFiles(string directory, string relativeDirectory, List files) + { + foreach (string path in Directory.EnumerateFileSystemEntries(directory)) + { + string relativePath = Path.Combine(relativeDirectory, Path.GetFileName(path)); + if (Directory.Exists(path)) + CollectOutputFiles(path, relativePath, files); + else + files.Add(relativePath); + } + } + + private static List ReadManifest(string entryDirectory) + { + using var manifest = new BinaryReader(File.OpenRead(Path.Combine(entryDirectory, "manifest"))); + if (manifest.ReadInt32() != FormatVersion) + throw new InvalidDataException("Unsupported ILLink cache format."); + + int fileCount = manifest.ReadInt32(); + if (fileCount < 0 || fileCount > manifest.BaseStream.Length - manifest.BaseStream.Position) + throw new InvalidDataException("Invalid ILLink cache file count."); + + var files = new List(); + var paths = new HashSet(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); + string outputsDirectory = Path.Combine(entryDirectory, "outputs"); + for (int i = 0; i < fileCount; i++) + { + string relativePath = manifest.ReadString(); + ValidateRelativePath(relativePath); + if (!paths.Add(relativePath)) + throw new InvalidDataException("Invalid ILLink cache output inventory."); + + long length = manifest.ReadInt64(); + byte[] expectedHash = manifest.ReadBytes(32); + string cachedFile = Path.Combine(outputsDirectory, relativePath); + + if (expectedHash.Length != 32 || new FileInfo(cachedFile).Length != length || + !expectedHash.SequenceEqual(HashFile(cachedFile))) + throw new InvalidDataException($"Invalid ILLink cache output: '{relativePath}'."); + + files.Add(relativePath); + } + + if (manifest.BaseStream.Position != manifest.BaseStream.Length) + throw new InvalidDataException("Unexpected data in ILLink cache manifest."); + + return files; + } + + private static void ValidateRelativePath(string path) + { + if (string.IsNullOrEmpty(path) || Path.IsPathRooted(path)) + throw new InvalidDataException("Invalid ILLink cache output path."); + + foreach (string component in path.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) + { + if (component is "" or "." or ".." || component.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || + (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (component.EndsWith(".", StringComparison.Ordinal) || component.EndsWith(" ", StringComparison.Ordinal)))) + throw new InvalidDataException("Invalid ILLink cache output path."); + } + } + + private static byte[] HashFile(string path) + { + using var sha256 = SHA256.Create(); + using var stream = File.OpenRead(path); + return sha256.ComputeHash(stream); + } + + private static bool IsCacheFailure(Exception ex) => ex is IOException or UnauthorizedAccessException or InvalidDataException or FormatException; +} diff --git a/src/tools/illink/src/ILLink.Tasks/ILLinkCacheEntry.cs b/src/tools/illink/src/ILLink.Tasks/ILLinkCacheEntry.cs new file mode 100644 index 00000000000000..155ab5bf993d8c --- /dev/null +++ b/src/tools/illink/src/ILLink.Tasks/ILLinkCacheEntry.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Globalization; +using System.IO; + +namespace ILLink.Tasks; + +internal static class ILLinkCacheEntry +{ + internal const string VersionDirectory = "v1"; + internal const string LastUsedFileName = "last-used"; + + internal static bool IsEntryName(string name) + { + if (name.Length != 64) + return false; + + foreach (char c in name) + { + if (c is not (>= '0' and <= '9') and not (>= 'a' and <= 'f')) + return false; + } + + return true; + } + + internal static void WriteLastUsed(string path) => + File.WriteAllText(path, System.DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture)); +} diff --git a/src/tools/illink/src/ILLink.Tasks/LinkTask.cs b/src/tools/illink/src/ILLink.Tasks/LinkTask.cs index 3994fda22cd4e6..9f2c27e9cf3353 100644 --- a/src/tools/illink/src/ILLink.Tasks/LinkTask.cs +++ b/src/tools/illink/src/ILLink.Tasks/LinkTask.cs @@ -3,8 +3,14 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; +using System.Linq; using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; +using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -264,6 +270,15 @@ private string DotNetPath protected override int ExecuteTool(string pathToTool, string responseFileCommands, string commandLineCommands) { + DateTime outputTimestampUtc = DateTime.UtcNow; + ILLinkCache cache = ILLinkCache.TryCreateFromEnvironment(Log); + string inputHash = null; + if (cache is not null && !TryComputeCacheKey(pathToTool, commandLineCommands, responseFileCommands, out inputHash)) + cache = null; + + if (cache is not null && cache.TryRestore(inputHash, OutputDirectory.ItemSpec, outputTimestampUtc)) + return 0; + try { string outputDirectory = Path.GetFullPath(OutputDirectory.ItemSpec); @@ -278,7 +293,115 @@ protected override int ExecuteTool(string pathToTool, string responseFileCommand return -1; } - return base.ExecuteTool(pathToTool, responseFileCommands, commandLineCommands); + int exitCode = base.ExecuteTool(pathToTool, responseFileCommands, commandLineCommands); + if (exitCode == 0 && !Log.HasLoggedErrors && cache is not null) + cache.Store(inputHash, OutputDirectory.ItemSpec); + + return exitCode; + } + + private bool TryComputeCacheKey(string pathToTool, string commandLineCommands, string responseFileCommands, out string inputHash) + { + inputHash = string.Empty; + if (CustomSteps?.Length > 0 || CustomData?.Length > 0 || DumpDependencies || + !string.IsNullOrEmpty(DependenciesFileFormat) || EnvironmentVariables?.Length > 0) + { + Log.LogMessage(MessageImportance.Low, "ILLink cache bypassed: custom steps/data, dependency dumps, and task environment overrides are not supported."); + return false; + } + + if (!string.IsNullOrWhiteSpace(ExtraArgs)) + { + Log.LogMessage(MessageImportance.Low, "ILLink cache bypassed: extra arguments are not supported."); + return false; + } + + try + { + var files = new SortedSet(StringComparer.Ordinal); + var assemblies = new HashSet(StringComparer.Ordinal); + foreach (ITaskItem assembly in AssemblyPaths) + AddAssembly(assembly.ItemSpec); + foreach (ITaskItem assembly in ReferenceAssemblyPaths ?? Array.Empty()) + AddAssembly(assembly.ItemSpec); + foreach (ITaskItem descriptor in RootDescriptorFiles ?? Array.Empty()) + AddFile(descriptor.ItemSpec); + +#pragma warning disable IL3000 // MSBuild tasks are loaded from assemblies on disk. + AddFile(typeof(ILLink).Assembly.Location); +#pragma warning restore IL3000 + AddFile(pathToTool); + AddFile(ILLinkPath); + + using var sha256 = SHA256.Create(); + using var description = new MemoryStream(); + using var writer = new BinaryWriter(description, Encoding.UTF8, leaveOpen: true); + writer.Write("ILLink task cache key v1"); + writer.Write(commandLineCommands); + writer.Write(responseFileCommands); + writer.Write(Environment.CurrentDirectory); + writer.Write(Path.GetFullPath(OutputDirectory.ItemSpec)); + writer.Write(RuntimeInformation.OSDescription); + writer.Write(RuntimeInformation.ProcessArchitecture.ToString()); + writer.Write(CultureInfo.CurrentCulture.Name); + writer.Write(CultureInfo.CurrentUICulture.Name); + writer.Write(files.Count); + foreach (string file in files) + { + writer.Write(file); + using var input = File.OpenRead(file); + writer.Write(sha256.ComputeHash(input)); + } + + writer.Flush(); + description.Position = 0; + inputHash = BitConverter.ToString(sha256.ComputeHash(description)).Replace("-", "").ToLowerInvariant(); + return true; + + void AddFile(string path) => files.Add(Path.GetFullPath(path)); + + void AddOptionalFile(string path) + { + if (File.Exists(path)) + AddFile(path); + } + + void AddAssembly(string path) + { + path = Path.GetFullPath(path); + if (!assemblies.Add(path)) + return; + + AddFile(path); + AddOptionalFile(path + ".config"); + AddOptionalFile(path + ".mdb"); + AddOptionalFile(Path.ChangeExtension(path, ".pdb")); + string directory = Path.GetDirectoryName(path); + string satelliteName = Path.GetFileNameWithoutExtension(path) + ".resources.dll"; + foreach (string satelliteDirectory in Directory.EnumerateDirectories(directory)) + AddOptionalFile(Path.Combine(satelliteDirectory, satelliteName)); + + using var input = File.OpenRead(path); + using var pe = new PEReader(input); + if (!pe.HasMetadata) + throw new BadImageFormatException("Input PE image has no managed metadata.", path); + MetadataReader metadata = pe.GetMetadataReader(); + foreach (AssemblyFileHandle handle in metadata.AssemblyFiles) + { + AssemblyFile file = metadata.GetAssemblyFile(handle); + string filePath = Path.Combine(directory, metadata.GetString(file.Name)); + if (file.ContainsMetadata) + AddAssembly(filePath); + else + AddFile(filePath); + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException or BadImageFormatException or InvalidDataException) + { + Log.LogMessage(MessageImportance.Low, $"ILLink cache bypassed: input identity could not be computed: {ex.Message}"); + return false; + } } private string _illinkPath = ""; diff --git a/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs b/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs new file mode 100644 index 00000000000000..e515946628b363 --- /dev/null +++ b/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs @@ -0,0 +1,236 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Globalization; +using System.IO; +using ILLink.Tasks; +using Xunit; + +namespace ILLink.CacheTool.Tests; + +public class CachePurgeTests +{ + [Theory] + [InlineData(-1, true)] + [InlineData(0, false)] + [InlineData(1, false)] + public void PurgeUsesLastUsedContentsAndExclusiveCutoff(int ticks, bool deleted) + { + using var cache = new TestCache(); + string entry = cache.AddEntry('a', TestCache.Cutoff.AddTicks(ticks)); + File.SetLastWriteTimeUtc(Path.Combine(entry, ILLinkCacheEntry.LastUsedFileName), DateTime.UtcNow); + + Assert.Equal(0, cache.Run()); + + Assert.Equal(!deleted, Directory.Exists(entry)); + Assert.Equal($"ILLink cache purge: Deleted: {(deleted ? 1 : 0)}, Kept: {(deleted ? 0 : 1)}, Errors: 0{Environment.NewLine}", + cache.Output.ToString()); + Assert.Empty(cache.Error.ToString()); + } + + [Fact] + public void PurgeOnlyVisitsPublishedEntriesInCurrentLayout() + { + using var cache = new TestCache(); + string old = cache.AddEntry('a', TestCache.Cutoff.AddDays(-1)); + string recent = cache.AddEntry('b', TestCache.Cutoff.AddDays(1)); + foreach (string name in new[] { new string('a', 64) + ".unique.tmp", "unrelated", new string('A', 64), new string('c', 63) }) + { + Directory.CreateDirectory(Path.Combine(cache.Entries, name)); + } + string otherVersion = Path.Combine(cache.Root, "v2", new string('d', 64)); + Directory.CreateDirectory(otherVersion); + File.WriteAllText(Path.Combine(cache.Entries, "unrelated-file"), "keep"); + + Assert.Equal(0, cache.Run()); + + Assert.False(Directory.Exists(old)); + Assert.True(Directory.Exists(recent)); + Assert.Equal(5, Directory.GetDirectories(cache.Entries).Length); + Assert.True(Directory.Exists(otherVersion)); + Assert.Equal("keep", File.ReadAllText(Path.Combine(cache.Entries, "unrelated-file"))); + Assert.Contains("Deleted: 1, Kept: 1, Errors: 0", cache.Output.ToString()); + } + + [Theory] + [InlineData("missing")] + [InlineData("malformed")] + [InlineData("directory")] + [InlineData("non-utc")] + public void InvalidMarkerReportsErrorAndDoesNotDeleteEntry(string damage) + { + using var cache = new TestCache(); + string damagedEntry = cache.AddEntry('a', TestCache.Cutoff.AddDays(-1)); + string validEntry = cache.AddEntry('b', TestCache.Cutoff.AddDays(-1)); + string marker = Path.Combine(damagedEntry, ILLinkCacheEntry.LastUsedFileName); + switch (damage) + { + case "missing": + File.Delete(marker); + break; + case "malformed": + File.WriteAllText(marker, "invalid timestamp"); + break; + case "directory": + File.Delete(marker); + Directory.CreateDirectory(marker); + break; + case "non-utc": + File.WriteAllText(marker, TestCache.Cutoff.ToOffset(TimeSpan.FromHours(1)).ToString("O", CultureInfo.InvariantCulture)); + break; + } + + Assert.Equal(1, cache.Run()); + + Assert.True(Directory.Exists(damagedEntry)); + Assert.False(Directory.Exists(validEntry)); + Assert.Contains(damagedEntry, cache.Error.ToString()); + Assert.Contains("Deleted: 1, Kept: 0, Errors: 1", cache.Output.ToString()); + } + + [Fact] + public void MissingVersionDirectoryIsReported() + { + using var cache = new TestCache(); + Directory.Delete(cache.Entries); + + Assert.Equal(1, cache.Run()); + + Assert.Contains(cache.Entries, cache.Error.ToString()); + Assert.Contains("Errors: 1", cache.Output.ToString()); + Assert.False(Directory.Exists(cache.Entries)); + } + + [Fact] + public void EmptyCacheSucceeds() + { + using var cache = new TestCache(); + Assert.Equal(0, cache.Run()); + Assert.Contains("Deleted: 0, Kept: 0, Errors: 0", cache.Output.ToString()); + } + + [Theory] + [InlineData("2025-01-02T03:04:05Z")] + [InlineData("2025-01-02T03:04:05+00:00")] + [InlineData("2025-01-02T03:04:05.0000000+00:00")] + [InlineData("2025-01-02T03:04:05.0000000Z")] + public void CommandAcceptsExplicitUtcCutoff(string cutoff) + { + using var cache = new TestCache(); + string entry = cache.AddEntry('a', TestCache.Cutoff.AddTicks(-1)); + + Assert.Equal(0, Program.Run(new[] { "purge", "--cache-directory", cache.Root, "--before", cutoff }, cache.Output, cache.Error)); + + Assert.False(Directory.Exists(entry)); + Assert.Contains("Deleted: 1", cache.Output.ToString()); + Assert.Empty(cache.Error.ToString()); + } + + [Theory] + [InlineData("invalid")] + [InlineData("")] + [InlineData("2025-01-02")] + [InlineData("2025-01-02T03:04:05")] + [InlineData("2025-01-02T03:04:05+01:00")] + public void CommandRejectsInvalidCutoffWithoutDeleting(string cutoff) + { + using var cache = new TestCache(); + string entry = cache.AddEntry('a', TestCache.Cutoff.AddDays(-1)); + + Assert.NotEqual(0, Program.Run(new[] { "purge", "--cache-directory", cache.Root, "--before", cutoff }, cache.Output, cache.Error)); + + Assert.True(Directory.Exists(entry)); + Assert.Contains("--before", cache.Error.ToString()); + Assert.DoesNotContain("ILLink cache purge:", cache.Output.ToString()); + } + + [Theory] + [InlineData("missing-directory")] + [InlineData("missing-cutoff")] + [InlineData("missing-value")] + [InlineData("unknown-option")] + [InlineData("unknown-command")] + public void CommandRejectsIncompleteOrUnknownArguments(string scenario) + { + using var cache = new TestCache(); + string[] args = scenario switch + { + "missing-directory" => new[] { "purge", "--before", "2025-01-02T03:04:05Z" }, + "missing-cutoff" => new[] { "purge", "--cache-directory", cache.Root }, + "missing-value" => new[] { "purge", "--cache-directory", cache.Root, "--before" }, + "unknown-option" => new[] { "purge", "--cache-directory", cache.Root, "--before", "2025-01-02T03:04:05Z", "--force" }, + _ => new[] { "delete-all" } + }; + string entry = cache.AddEntry('a', TestCache.Cutoff.AddDays(-1)); + + Assert.NotEqual(0, Program.Run(args, cache.Output, cache.Error)); + + Assert.True(Directory.Exists(entry)); + Assert.NotEmpty(cache.Error.ToString()); + } + + [Fact] + public void CommandRejectsNonexistentDirectory() + { + using var cache = new TestCache(); + string missing = Path.Combine(cache.Root, "missing"); + + Assert.NotEqual(0, Program.Run(new[] { "purge", "--cache-directory", missing, "--before", "2025-01-02T03:04:05Z" }, cache.Output, cache.Error)); + + Assert.NotEmpty(cache.Error.ToString()); + Assert.False(Directory.Exists(missing)); + } + + [Fact] + public void CommandPropagatesMaintenanceFailure() + { + using var cache = new TestCache(); + string entry = cache.AddEntry('a', TestCache.Cutoff.AddDays(-1)); + File.Delete(Path.Combine(entry, ILLinkCacheEntry.LastUsedFileName)); + + Assert.Equal(1, Program.Run(new[] { "purge", "--cache-directory", cache.Root, "--before", "2025-01-02T03:04:05Z" }, cache.Output, cache.Error)); + + Assert.True(Directory.Exists(entry)); + Assert.Contains("Errors: 1", cache.Output.ToString()); + } + + [Fact] + public void HelpExplainsIdleCacheRequirement() + { + using var output = new StringWriter(); + using var error = new StringWriter(); + Assert.Equal(0, Program.Run(new[] { "purge", "--help" }, output, error)); + Assert.Contains("No builds may use this cache while purging.", output.ToString()); + Assert.Empty(error.ToString()); + } + + private sealed class TestCache : IDisposable + { + internal static readonly DateTimeOffset Cutoff = new(2025, 1, 2, 3, 4, 5, TimeSpan.Zero); + internal string Root { get; } = Path.Combine(Path.GetTempPath(), "illink-purge-tests-" + Guid.NewGuid().ToString("N")); + internal string Entries => Path.Combine(Root, ILLinkCacheEntry.VersionDirectory); + internal StringWriter Output { get; } = new(); + internal StringWriter Error { get; } = new(); + + internal TestCache() => Directory.CreateDirectory(Entries); + + internal string AddEntry(char key, DateTimeOffset lastUsed) + { + string entry = Path.Combine(Entries, new string(key, 64)); + Directory.CreateDirectory(Path.Combine(entry, "outputs")); + File.WriteAllText(Path.Combine(entry, "outputs", "app.dll"), "cached output"); + File.WriteAllText(Path.Combine(entry, ILLinkCacheEntry.LastUsedFileName), lastUsed.ToString("O", CultureInfo.InvariantCulture)); + return entry; + } + + internal int Run() => CachePurge.Run(Root, Cutoff, Output, Error); + + public void Dispose() + { + Output.Dispose(); + Error.Dispose(); + Directory.Delete(Root, recursive: true); + } + } +} diff --git a/src/tools/illink/test/ILLink.CacheTool.Tests/ILLink.CacheTool.Tests.csproj b/src/tools/illink/test/ILLink.CacheTool.Tests/ILLink.CacheTool.Tests.csproj new file mode 100644 index 00000000000000..8822ac9bdd82f0 --- /dev/null +++ b/src/tools/illink/test/ILLink.CacheTool.Tests/ILLink.CacheTool.Tests.csproj @@ -0,0 +1,5 @@ + + + + + diff --git a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs index 7fd582142e6278..47405f2ae6fbae 100644 --- a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs +++ b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs @@ -3,11 +3,18 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; +using System.Reflection.PortableExecutable; +using System.Runtime.ExceptionServices; +using System.Threading; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; +using Microsoft.DotNet.RemoteExecutor; using Mono.Linker; using Xunit; @@ -1009,6 +1016,1073 @@ public void TestErrorHandling() message.Line.Contains("No input files were specified")); } + [Fact] + public void CacheHasNoTaskParameters() + { + Assert.DoesNotContain(typeof(ILLink).GetProperties(), property => property.Name.Contains("Cache")); + } + + [Theory] + [InlineData(null, false, false)] + [InlineData("", false, false)] + [InlineData("false", false, false)] + [InlineData("FALSE", false, false)] + [InlineData("true", true, false)] + [InlineData("TrUe", true, false)] + [InlineData("1", false, true)] + [InlineData("invalid", false, true)] + public void CacheEnvironmentConfiguration(string setting, bool enabled, bool invalid) + { + RemoteExecutor.Invoke(static (value, expectedEnabled, expectedInvalid) => + { + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", value == "" ? null : value); + string directory = Path.Combine(Path.GetTempPath(), "illink-cache-" + Guid.NewGuid().ToString("N")); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE_PATH", directory); + var buildEngine = new MockBuildEngine(); + var task = new MockTask { BuildEngine = buildEngine }; + ILLinkCache cache = ILLinkCache.TryCreateFromEnvironment(task.Log); + + Assert.Equal(bool.Parse(expectedEnabled), cache is not null); + if (cache is not null) + Assert.Equal(directory, cache.CacheDirectory); + Assert.Equal(bool.Parse(expectedInvalid), buildEngine.Messages.Any(message => + message.Message.Contains("ILLINK_EXPERIMENTAL_CACHE must be 'true' or 'false'"))); + Assert.False(Directory.Exists(directory)); + }, setting ?? "", enabled.ToString(), invalid.ToString()).Dispose(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("true")] + [InlineData("false")] + [InlineData("cache directory")] + public void CacheEnvironmentDirectory(string directory) + { + RemoteExecutor.Invoke(static value => + { + string path = value == "" ? null : value; + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", "true"); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE_PATH", path); + var task = new MockTask { BuildEngine = new MockBuildEngine() }; + + ILLinkCache cache = ILLinkCache.TryCreateFromEnvironment(task.Log); + + Assert.NotNull(cache); + Assert.Equal(ILLinkCache.TryCreate(path, task.Log).CacheDirectory, cache.CacheDirectory); + }, directory ?? "").Dispose(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void CacheUsesPlatformDefaultDirectory(string cacheDirectory) + { + var task = new MockTask { BuildEngine = new MockBuildEngine() }; + string expectedDirectory; + if (OperatingSystem.IsWindows()) + { + expectedDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "illink"); + } + else if (OperatingSystem.IsMacOS()) + { + expectedDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Library", "Caches", "illink"); + } + else + { + string cacheHome = Environment.GetEnvironmentVariable("XDG_CACHE_HOME"); + expectedDirectory = !string.IsNullOrEmpty(cacheHome) && Path.IsPathRooted(cacheHome) + ? Path.Combine(cacheHome, "illink") + : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".cache", "illink"); + } + + ILLinkCache cache = ILLinkCache.TryCreate(cacheDirectory, task.Log); + + Assert.NotNull(cache); + Assert.Equal(Path.GetFullPath(expectedDirectory), cache.CacheDirectory); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CacheDirectoryOverrideDoesNotCreateDirectory(bool absolutePath) + { + string directoryName = "illink-cache-" + Guid.NewGuid().ToString("N"); + string cacheDirectory = absolutePath ? Path.Combine(Path.GetTempPath(), directoryName) : directoryName; + var task = new MockTask { BuildEngine = new MockBuildEngine() }; + + ILLinkCache cache = ILLinkCache.TryCreate(cacheDirectory, task.Log); + + Assert.NotNull(cache); + Assert.Equal(Path.GetFullPath(cacheDirectory), cache.CacheDirectory); + Assert.False(Directory.Exists(cache.CacheDirectory)); + } + + [Fact] + public void InvalidCacheDirectoryBypassesCaching() + { + var buildEngine = new MockBuildEngine(); + var task = new MockTask { BuildEngine = buildEngine }; + + Assert.Null(ILLinkCache.TryCreate("invalid\0path", task.Log)); + Assert.Contains(buildEngine.Messages, message => + message.Message.Contains("could not be resolved")); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CacheRoundTrip(bool includeSymbols) + { + using var test = new CacheTestDirectory(); + var files = new Dictionary + { + ["app.dll"] = "assembly", + ["app.dll.config"] = "configuration", + [Path.Combine("fr", "app.resources.dll")] = "satellite", + ["Link.semaphore"] = "ordinary file" + }; + if (includeSymbols) + files.Add("app.pdb", "symbols"); + + foreach (var file in files) + test.WriteSource(file.Key, file.Value); + + Assert.False(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + Assert.False(Directory.Exists(test.Cache.CacheDirectory)); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + + Directory.CreateDirectory(test.Output); + File.WriteAllText(Path.Combine(test.Output, "app.dll"), "old assembly"); + File.WriteAllText(Path.Combine(test.Output, "Link.semaphore"), "existing semaphore"); + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + + foreach (var file in files) + { + string path = Path.Combine(test.Output, file.Key); + Assert.Equal(file.Value, File.ReadAllText(path)); + Assert.Equal(CacheTestDirectory.Timestamp, File.GetLastWriteTimeUtc(path)); + } + Assert.Equal("ordinary file", File.ReadAllText(Path.Combine(test.Output, "Link.semaphore"))); + Assert.Equal(files.Count, Directory.GetFiles(Path.Combine(test.Entry, "outputs"), "*", SearchOption.AllDirectories).Length); + Assert.Empty(Directory.GetDirectories(Path.GetDirectoryName(test.Entry), "*.tmp")); + + File.WriteAllText(Path.Combine(test.Output, "app.dll"), "modified restored output"); + Assert.Equal("assembly", File.ReadAllText(Path.Combine(test.Entry, "outputs", "app.dll"))); + } + + [Fact] + public void CacheSupportsEmptyOutput() + { + using var test = new CacheTestDirectory(); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + Assert.True(Directory.Exists(test.Output)); + Assert.Empty(Directory.GetFileSystemEntries(test.Output)); + } + + [Fact] + public void CacheRecordsLastUsedOnStoreAndHit() + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "assembly"); + DateTimeOffset beforeStore = DateTimeOffset.UtcNow; + test.Cache.Store(CacheTestDirectory.Key, test.Source); + string marker = Path.Combine(test.Entry, ILLinkCacheEntry.LastUsedFileName); + Assert.InRange(DateTimeOffset.ParseExact(File.ReadAllText(marker), "O", CultureInfo.InvariantCulture), + beforeStore, DateTimeOffset.UtcNow); + File.WriteAllText(marker, DateTimeOffset.MinValue.ToString("O", CultureInfo.InvariantCulture)); + + DateTimeOffset beforeRestore = DateTimeOffset.UtcNow; + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + + Assert.InRange(DateTimeOffset.ParseExact(File.ReadAllText(marker), "O", CultureInfo.InvariantCulture), + beforeRestore, DateTimeOffset.UtcNow); + Assert.False(File.Exists(Path.Combine(test.Output, ILLinkCacheEntry.LastUsedFileName))); + Assert.Empty(Directory.GetFiles(test.Entry, "*.tmp")); + } + + [Fact] + public void CacheMarkerUpdateFailureDoesNotFailRestore() + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "assembly"); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + string marker = Path.Combine(test.Entry, ILLinkCacheEntry.LastUsedFileName); + File.Delete(marker); + Directory.CreateDirectory(marker); + + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + + Assert.Equal("assembly", File.ReadAllText(Path.Combine(test.Output, "app.dll"))); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.Contains("last-used update failed")); + Assert.Empty(Directory.GetFiles(test.Entry, "*.tmp")); + Assert.Empty(test.BuildEngine.Errors); + } + + [Fact] + public void FailedRestoreDoesNotRefreshUsage() + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "assembly"); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + string marker = Path.Combine(test.Entry, ILLinkCacheEntry.LastUsedFileName); + string oldTimestamp = DateTimeOffset.MinValue.ToString("O", CultureInfo.InvariantCulture); + File.WriteAllText(marker, oldTimestamp); + File.WriteAllText(Path.Combine(test.Entry, "outputs", "app.dll"), "corrupt"); + + Assert.False(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + + Assert.Equal(oldTimestamp, File.ReadAllText(marker)); + } + + [Fact] + public void ParallelProcessesRefreshUsageWithoutCorruptingMarker() + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "assembly"); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + DateTimeOffset beforeRestore = DateTimeOffset.UtcNow; + using (StartReader(test, 0)) + using (StartReader(test, 1)) + { + Assert.True(SpinWait.SpinUntil(() => Directory.GetFiles(test.Root, "*.ready").Length == 2, TimeSpan.FromSeconds(30))); + File.WriteAllText(Path.Combine(test.Root, "start"), ""); + } + + string marker = Path.Combine(test.Entry, ILLinkCacheEntry.LastUsedFileName); + Assert.InRange(DateTimeOffset.ParseExact(File.ReadAllText(marker), "O", CultureInfo.InvariantCulture), + beforeRestore, DateTimeOffset.UtcNow); + Assert.Empty(Directory.GetFiles(test.Entry, "*.tmp")); + + static RemoteInvokeHandle StartReader(CacheTestDirectory test, int index) => + RemoteExecutor.Invoke(static (cacheDirectory, output, start) => + { + var engine = new MockBuildEngine(); + var task = new MockTask { BuildEngine = engine }; + ILLinkCache cache = ILLinkCache.TryCreate(cacheDirectory, task.Log); + File.WriteAllText(output + ".ready", ""); + Assert.True(SpinWait.SpinUntil(() => File.Exists(start), TimeSpan.FromSeconds(30))); + for (int attempt = 0; attempt < 20; attempt++) + Assert.True(cache.TryRestore(CacheTestDirectory.Key, output, CacheTestDirectory.Timestamp)); + Assert.Empty(engine.Errors); + Assert.DoesNotContain(engine.Messages, message => message.Message.Contains("last-used update failed")); + }, test.Cache.CacheDirectory, Path.Combine(test.Root, "reader-" + index), Path.Combine(test.Root, "start")); + } + + [Fact] + public void CacheDoesNotReplaceExistingEntry() + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "first"); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + test.WriteSource("app.dll", "second"); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + Assert.Equal("first", File.ReadAllText(Path.Combine(test.Output, "app.dll"))); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.Contains("already exists")); + Assert.Empty(Directory.GetDirectories(Path.GetDirectoryName(test.Entry), "*.tmp")); + } + + [Theory] + [InlineData("missing-manifest")] + [InlineData("version")] + [InlineData("truncated")] + [InlineData("missing-file")] + [InlineData("corrupt-file")] + [InlineData("escaping-path")] + public void CacheRejectsInvalidEntryBeforeCopying(string damage) + { + using var test = new CacheTestDirectory(); + test.WriteSource("a.dll", "first"); + test.WriteSource("b.dll", "second"); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + string manifest = Path.Combine(test.Entry, "manifest"); + string cachedFile = Path.Combine(test.Entry, "outputs", "b.dll"); + switch (damage) + { + case "missing-manifest": + File.Delete(manifest); + break; + case "version": + using (var writer = new BinaryWriter(File.OpenWrite(manifest))) + writer.Write(-1); + break; + case "truncated": + File.WriteAllBytes(manifest, new byte[] { 1 }); + break; + case "missing-file": + File.Delete(cachedFile); + break; + case "corrupt-file": + File.WriteAllText(cachedFile, "broken"); + break; + case "escaping-path": + using (var writer = new BinaryWriter(File.Create(manifest))) + { + writer.Write(1); + writer.Write(1); + writer.Write("../escaped.dll"); + } + break; + } + + Directory.CreateDirectory(test.Output); + string existingOutput = Path.Combine(test.Output, "a.dll"); + File.WriteAllText(existingOutput, "unchanged"); + Assert.False(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + Assert.Equal("unchanged", File.ReadAllText(existingOutput)); + Assert.False(File.Exists(Path.Combine(test.Output, "b.dll"))); + Assert.False(File.Exists(Path.Combine(test.Root, "escaped.dll"))); + Assert.True(Directory.Exists(test.Entry)); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.Contains("restore failed")); + } + + [Fact] + public void CacheRestoreReplacesExistingDirectory() + { + using var test = new CacheTestDirectory(); + test.WriteSource("a.dll", "new assembly"); + test.WriteSource("b.dll", "other assembly"); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + Directory.CreateDirectory(Path.Combine(test.Output, "b.dll")); + File.WriteAllText(Path.Combine(test.Output, "a.dll"), "old assembly"); + + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + Assert.Equal("new assembly", File.ReadAllText(Path.Combine(test.Output, "a.dll"))); + Assert.Equal("other assembly", File.ReadAllText(Path.Combine(test.Output, "b.dll"))); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CacheStoreFailureIsBestEffort(bool failPublication) + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "assembly"); + if (failPublication) + { + Directory.CreateDirectory(Path.GetDirectoryName(test.Entry)); + File.WriteAllText(test.Entry, "blocks publication"); + } + else + { + File.WriteAllText(test.Cache.CacheDirectory, "blocks cache creation"); + } + + test.Cache.Store(CacheTestDirectory.Key, test.Source); + + Assert.Equal("assembly", File.ReadAllText(Path.Combine(test.Source, "app.dll"))); + Assert.False(Directory.Exists(test.Entry)); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.Contains("store failed")); + if (failPublication) + { + Assert.Equal("blocks publication", File.ReadAllText(test.Entry)); + Assert.Empty(Directory.GetDirectories(Path.GetDirectoryName(test.Entry), "*.tmp")); + } + } + + [Fact] + public void CacheSkipsContendedEntryButNotOtherKeys() + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "assembly"); + using var mutex = new Mutex(false, test.MutexName); + Assert.True(mutex.WaitOne(0)); + try + { + RunOnOtherThread(() => test.Cache.Store(CacheTestDirectory.Key, test.Source)); + Assert.False(Directory.Exists(test.Cache.CacheDirectory)); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.Contains("another writer")); + + string otherKey = new string('b', 64); + RunOnOtherThread(() => test.Cache.Store(otherKey, test.Source)); + Assert.True(test.Cache.TryRestore(otherKey, test.Output, CacheTestDirectory.Timestamp)); + } + finally + { + mutex.ReleaseMutex(); + } + + test.Cache.Store(CacheTestDirectory.Key, test.Source); + Assert.True(Directory.Exists(test.Entry)); + } + + [Fact] + public void CacheReadsPublishedEntryWithoutWriterLock() + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "assembly"); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + using var mutex = new Mutex(false, test.MutexName); + Assert.True(mutex.WaitOne(0)); + try + { + RunOnOtherThread(() => + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp))); + } + finally + { + mutex.ReleaseMutex(); + } + } + + [Fact] + public void CacheSupportsConcurrentWritersWithDifferentOutputs() + { + using var test = new CacheTestDirectory(); + string contents = new string('a', 1024 * 1024); + System.Threading.Tasks.Parallel.For(0, 8, index => + { + string source = Path.Combine(test.Root, "source-" + index); + Directory.CreateDirectory(source); + File.WriteAllText(Path.Combine(source, "app.dll"), contents); + File.WriteAllText(Path.Combine(source, "app.pdb"), "symbols"); + var task = new MockTask { BuildEngine = new MockBuildEngine() }; + ILLinkCache cache = ILLinkCache.TryCreate(test.Cache.CacheDirectory, task.Log); + + cache.Store(CacheTestDirectory.Key, source); + }); + + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + Assert.Equal(contents, File.ReadAllText(Path.Combine(test.Output, "app.dll"))); + Assert.Equal("symbols", File.ReadAllText(Path.Combine(test.Output, "app.pdb"))); + Assert.Single(Directory.GetDirectories(Path.GetDirectoryName(test.Entry))); + } + + [Fact] + public void CacheIgnoresOtherWritersStagingDirectories() + { + using var test = new CacheTestDirectory(); + string stagingDirectory = test.Entry + ".orphan.tmp"; + Directory.CreateDirectory(stagingDirectory); + string marker = Path.Combine(stagingDirectory, "partial"); + File.WriteAllText(marker, "other attempt"); + + Assert.False(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + test.Cache.Store(CacheTestDirectory.Key, test.Source); + + Assert.Equal("other attempt", File.ReadAllText(marker)); + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CacheRejectsOverlappingDirectories(bool cacheContainsOutput) + { + using var test = new CacheTestDirectory(); + string output = cacheContainsOutput ? Path.Combine(test.Cache.CacheDirectory, "output") : test.Root; + + Assert.Throws(() => test.Cache.Store(CacheTestDirectory.Key, output)); + Assert.Throws(() => test.Cache.TryRestore(CacheTestDirectory.Key, output, CacheTestDirectory.Timestamp)); + Assert.False(Directory.Exists(test.Cache.CacheDirectory)); + } + + [Fact] + public void CacheRecoversAbandonedWriterLock() + { + using var test = new CacheTestDirectory(); + test.WriteSource("app.dll", "assembly"); + using var mutex = new Mutex(false, test.MutexName); + var owner = new Thread(() => mutex.WaitOne()); + owner.Start(); + owner.Join(); + + test.Cache.Store(CacheTestDirectory.Key, test.Source); + + Assert.True(test.Cache.TryRestore(CacheTestDirectory.Key, test.Output, CacheTestDirectory.Timestamp)); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.Contains("abandoned")); + } + + [Theory] + [InlineData("")] + [InlineData("../entry")] + [InlineData("not-a-hash")] + public void CacheRejectsInvalidInputHash(string inputHash) + { + using var test = new CacheTestDirectory(); + Assert.Throws(() => test.Cache.Store(inputHash, test.Source)); + Assert.Throws(() => test.Cache.TryRestore(inputHash, test.Output, CacheTestDirectory.Timestamp)); + Assert.False(Directory.Exists(test.Cache.CacheDirectory)); + } + + private static void RunOnOtherThread(Action action) + { + ExceptionDispatchInfo exception = null; + var thread = new Thread(() => + { + try + { + action(); + } + catch (Exception ex) + { + exception = ExceptionDispatchInfo.Capture(ex); + } + }); + thread.Start(); + thread.Join(); + exception?.Throw(); + } + + private sealed class CacheTestDirectory : IDisposable + { + internal static readonly string Key = new string('a', 64); + internal static readonly DateTime Timestamp = new DateTime(2025, 1, 2, 3, 4, 5, DateTimeKind.Utc); + internal string Root { get; } = Path.Combine(Path.GetTempPath(), "illink-cache-tests-" + Guid.NewGuid().ToString("N")); + internal string Source => Path.Combine(Root, "source"); + internal string Output => Path.Combine(Root, "output"); + internal string Entry => Path.Combine(Cache.CacheDirectory, "v1", Key); + internal MockBuildEngine BuildEngine { get; } = new(); + internal ILLinkCache Cache { get; } + internal string MutexName => (string)typeof(ILLinkCache).GetMethod("GetWriterMutexName", BindingFlags.NonPublic | BindingFlags.Instance).Invoke(Cache, new object[] { Key }); + + internal CacheTestDirectory() + { + Directory.CreateDirectory(Source); + var task = new MockTask { BuildEngine = BuildEngine }; + Cache = ILLinkCache.TryCreate(Path.Combine(Root, "cache"), task.Log); + } + + internal void WriteSource(string relativePath, string contents) + { + string path = Path.Combine(Source, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + File.WriteAllText(path, contents); + } + + public void Dispose() => Directory.Delete(Root, recursive: true); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CacheOptionsAreNotLinkerArguments(bool enableCache) + { + RemoteExecutor.Invoke(static value => + { + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", null); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE_PATH", null); + string arguments = new MockTask().GetResponseFileCommands(); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", value); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE_PATH", "cache directory"); + + Assert.Equal(arguments, new MockTask().GetResponseFileCommands()); + }, enableCache.ToString()).Dispose(); + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public void TaskDoesNotCacheUnsupportedOrFailedInvocations(bool enableCache, bool succeeds) + { + RemoteExecutor.Invoke(static (enabled, success) => + { + bool succeeds = bool.Parse(success); + using var test = new OutputDirectoryTest(); + string cacheDirectory = Path.Combine(test.Root, "cache"); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", enabled); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE_PATH", cacheDirectory); + var task = test.Task; + task.ExtraArgs = succeeds ? "--help" : null; + Directory.CreateDirectory(test.Output); + string stale = Path.Combine(test.Output, "stale.dll"); + File.WriteAllText(stale, "old output"); + + Assert.Equal(succeeds, task.Execute()); + Assert.False(File.Exists(stale)); + Assert.Equal(succeeds ? 0 : 1, task.ExitCode); + Assert.Contains(task.Messages, message => + message.Line.Contains(succeeds ? "illink [options]" : "No input files were specified")); + Assert.Equal(bool.Parse(enabled) && succeeds, test.BuildEngine.Messages.Any(message => + message.Message.StartsWith("ILLink cache bypassed: extra arguments"))); + Assert.False(Directory.Exists(cacheDirectory)); + }, enableCache.ToString(), succeeds.ToString()).Dispose(); + } + + [Theory] + [InlineData("inputs/Input.dll")] + [InlineData("inputs/Reference.dll")] + [InlineData("roots.xml")] + [InlineData("host/dotnet")] + public void CacheKeyTracksFileContentsNotTimestamps(string relativePath) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + Assert.Matches("^[0-9a-f]{64}$", original); + + string file = Path.Combine(test.Root, relativePath); + DateTime timestamp = File.GetLastWriteTimeUtc(file); + File.SetLastWriteTimeUtc(file, timestamp.AddHours(-1)); + Assert.True(test.Task.TryGetCacheKey(host, out string afterTouch)); + Assert.Equal(original, afterTouch); + + byte[] contents = File.ReadAllBytes(file); + contents[contents.Length - 1] ^= 1; + File.WriteAllBytes(file, contents); + File.SetLastWriteTimeUtc(file, timestamp); + Assert.True(test.Task.TryGetCacheKey(host, out string afterChange)); + Assert.NotEqual(original, afterChange); + } + + [Theory] + [InlineData("--help")] + [InlineData("--ignore-link-attributes true")] + [InlineData("--link-attributes attributes.xml")] + [InlineData("--ignore-link-attributes true --substitutions substitutions.xml")] + [InlineData("--ignore-link-attributes true -d assemblies")] + public void CacheKeyBypassesUnsupportedExtraArgs(string extraArgs) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + test.Task.ExtraArgs = extraArgs; + + Assert.False(test.Task.TryGetCacheKey(host, out string key)); + Assert.Empty(key); + Assert.Contains(test.BuildEngine.Messages, message => + message.Message.StartsWith("ILLink cache bypassed: extra arguments are not supported.")); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("1.0.0+test")] + public void CacheKeyUsesLinkerContents(string informationalVersion) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + Guid mvid = Guid.NewGuid(); + WriteTestAssembly(test.Task.ILLinkPath, informationalVersion, mvid); + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + byte[] originalImage = File.ReadAllBytes(test.Task.ILLinkPath); + + WriteTestAssembly(test.Task.ILLinkPath, "1.0.0+changed", mvid); + Assert.NotEqual(originalImage, File.ReadAllBytes(test.Task.ILLinkPath)); + Assert.True(test.Task.TryGetCacheKey(host, out string sameMvid)); + Assert.NotEqual(original, sameMvid); + + WriteTestAssembly(test.Task.ILLinkPath, "1.0.0+changed", Guid.NewGuid()); + Assert.True(test.Task.TryGetCacheKey(host, out string changedMvid)); + Assert.NotEqual(sameMvid, changedMvid); + + byte[] image = File.ReadAllBytes(test.Task.ILLinkPath); + DateTime timestamp = File.GetLastWriteTimeUtc(test.Task.ILLinkPath); + image[image.Length - 1] ^= 1; + File.WriteAllBytes(test.Task.ILLinkPath, image); + File.SetLastWriteTimeUtc(test.Task.ILLinkPath, timestamp); + Assert.True(test.Task.TryGetCacheKey(host, out string sameLengthAndTimestamp)); + Assert.NotEqual(changedMvid, sameLengthAndTimestamp); + + File.SetLastWriteTimeUtc(test.Task.ILLinkPath, timestamp.AddSeconds(1)); + Assert.True(test.Task.TryGetCacheKey(host, out string timestampOnly)); + Assert.Equal(sameLengthAndTimestamp, timestampOnly); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CacheKeyHashesLinkerWithoutMvid(bool hasMetadata) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + if (hasMetadata) + WriteTestAssembly(test.Task.ILLinkPath, mvid: Guid.Empty); + else + WriteMetadataLessPE(test.Task.ILLinkPath); + + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + File.AppendAllText(test.Task.ILLinkPath, "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(original, changed); + } + + [Theory] + [InlineData("Mono.Cecil.dll")] + [InlineData("illink.runtimeconfig.json")] + [InlineData("illink.deps.json")] + [InlineData("unrelated/generated.xml")] + public void CacheKeyIgnoresToolDirectoryFiles(string relativePath) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + + string file = Path.Combine(Path.GetDirectoryName(test.Task.ILLinkPath), relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(file)); + File.WriteAllText(file, "tool directory contents"); + Assert.True(test.Task.TryGetCacheKey(host, out string added)); + Assert.Equal(original, added); + + File.WriteAllText(file, "changed tool directory contents"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.Equal(original, changed); + + File.Delete(file); + Assert.True(test.Task.TryGetCacheKey(host, out string removed)); + Assert.Equal(original, removed); + } + + [Theory] + [InlineData("inputs/Input.pdb")] + [InlineData("inputs/Input.dll.mdb")] + [InlineData("inputs/Input.dll.config")] + [InlineData("inputs/fr/Input.resources.dll")] + [InlineData("inputs/Reference.pdb")] + public void CacheKeyTracksOptionalFilePresence(string relativePath) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + test.Task.RemoveSymbols = true; + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + + string file = Path.Combine(test.Root, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(file)); + File.WriteAllText(file, "sidecar"); + Assert.True(test.Task.TryGetCacheKey(host, out string added)); + Assert.NotEqual(original, added); + + File.WriteAllText(file, "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(added, changed); + + File.Delete(file); + Assert.True(test.Task.TryGetCacheKey(host, out string removed)); + Assert.Equal(original, removed); + } + + [Theory] + [InlineData("argument")] + [InlineData("metadata")] + [InlineData("output")] + [InlineData("input-path")] + public void CacheKeyTracksInvocation(string change) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + switch (change) + { + case "argument": + test.Task.NoWarn = "2026"; + break; + case "metadata": + test.Task.AssemblyPaths[0].SetMetadata("TrimMode", "copy"); + break; + case "output": + test.Task.OutputDirectory = new TaskItem(test.Output + "-other"); + break; + case "input-path": + string newPath = Path.Combine(test.Root, "Input.dll"); + File.Copy(test.Task.AssemblyPaths[0].ItemSpec, newPath); + test.Task.AssemblyPaths[0] = new TaskItem(newPath); + break; + } + + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(original, changed); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void CacheKeyTracksAssemblyFiles(bool containsMetadata) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string input = test.Task.AssemblyPaths[0].ItemSpec; + string externalFile; + if (containsMetadata) + { + externalFile = Path.Combine(Path.GetDirectoryName(input), "secondary.netmodule"); + using var module = Mono.Cecil.ModuleDefinition.CreateModule("secondary.netmodule", Mono.Cecil.ModuleKind.NetModule); + module.Write(externalFile); + var metadata = new MetadataBuilder(); + metadata.AddModule(0, metadata.GetOrAddString("Input.dll"), metadata.GetOrAddGuid(Guid.NewGuid()), default, default); + metadata.AddAssembly(metadata.GetOrAddString("Input"), new Version(1, 0), default, default, 0, AssemblyHashAlgorithm.Sha256); + metadata.AddAssemblyFile(metadata.GetOrAddString("secondary.netmodule"), default, containsMetadata: true); + var pe = new ManagedPEBuilder(PEHeaderBuilder.CreateLibraryHeader(), new MetadataRootBuilder(metadata), new BlobBuilder()); + var image = new BlobBuilder(); + pe.Serialize(image); + File.WriteAllBytes(input, image.ToArray()); + } + else + { + externalFile = Path.Combine(Path.GetDirectoryName(input), "linked.resources"); + File.WriteAllText(externalFile, "resource contents"); + using var assembly = Mono.Cecil.AssemblyDefinition.CreateAssembly( + new Mono.Cecil.AssemblyNameDefinition("Input", new Version(1, 0)), "Input.dll", Mono.Cecil.ModuleKind.Dll); + assembly.MainModule.Resources.Add(new Mono.Cecil.LinkedResource( + "linked", Mono.Cecil.ManifestResourceAttributes.Public, externalFile)); + assembly.Write(input); + } + + Assert.True(File.Exists(externalFile)); + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + File.AppendAllText(externalFile, "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(original, changed); + } + + [Theory] + [InlineData("host/fxr")] + [InlineData("shared")] + public void CacheKeyIgnoresHostDirectories(string relativePath) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + string directory = Path.Combine(test.Root, "host", relativePath); + Directory.CreateDirectory(Path.Combine(directory, "another-version")); + Assert.True(test.Task.TryGetCacheKey(host, out string addedDirectory)); + Assert.Equal(original, addedDirectory); + + string file = Path.Combine(directory, "another-version", "runtime"); + File.WriteAllText(file, "runtime contents"); + Assert.True(test.Task.TryGetCacheKey(host, out string addedFile)); + Assert.Equal(original, addedFile); + + File.WriteAllText(file, "changed runtime contents"); + Assert.True(test.Task.TryGetCacheKey(host, out string changedFile)); + Assert.Equal(original, changedFile); + + Directory.Delete(directory, recursive: true); + Assert.True(test.Task.TryGetCacheKey(host, out string removedDirectory)); + Assert.Equal(original, removedDirectory); + } + + [Theory] + [InlineData("extra-args")] + [InlineData("custom-step")] + [InlineData("custom-data")] + [InlineData("dump-dependencies")] + [InlineData("dependencies-format")] + [InlineData("environment")] + [InlineData("missing-input")] + [InlineData("invalid-assembly")] + [InlineData("metadata-less-assembly")] + [InlineData("metadata-less-reference")] + [InlineData("missing-linker")] + public void CacheKeyBypassesUnsupportedInputs(string reason) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + switch (reason) + { + case "extra-args": + test.Task.ExtraArgs = "--help"; + break; + case "custom-step": + test.Task.CustomSteps = new ITaskItem[] + { + new TaskItem("step.dll", new Dictionary { { "Type", "Step" } }) + }; + break; + case "custom-data": + test.Task.CustomData = new ITaskItem[] + { + new TaskItem("data", new Dictionary { { "Value", "input" } }) + }; + break; + case "dump-dependencies": + test.Task.DumpDependencies = true; + break; + case "dependencies-format": + test.Task.DependenciesFileFormat = "xml"; + break; + case "environment": + test.Task.EnvironmentVariables = new[] { "CUSTOM_SETTING=value" }; + break; + case "missing-input": + File.Delete(test.Task.AssemblyPaths[0].ItemSpec); + break; + case "invalid-assembly": + File.WriteAllText(test.Task.AssemblyPaths[0].ItemSpec, "invalid"); + break; + case "metadata-less-assembly": + WriteMetadataLessPE(test.Task.AssemblyPaths[0].ItemSpec); + break; + case "metadata-less-reference": + WriteMetadataLessPE(test.Task.ReferenceAssemblyPaths[0].ItemSpec); + break; + case "missing-linker": + File.Delete(test.Task.ILLinkPath); + break; + } + + Assert.False(test.Task.TryGetCacheKey(host, out string key)); + Assert.Empty(key); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.StartsWith("ILLink cache bypassed:")); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void TaskLinksWithUnusedMetadataLessReference(bool enableCache) + { + RemoteExecutor.Invoke(static enabled => + { + using var test = new OutputDirectoryTest(); + string input = Path.Combine(test.Root, "Input.dll"); + string reference = Path.Combine(test.Root, "Native.dll"); + WriteTestAssembly(input); + WriteMetadataLessPE(reference); + var task = test.Task; + task.AssemblyPaths = new ITaskItem[] { new TaskItem(input, new Dictionary { { "TrimMode", "copy" } }) }; + task.RootAssemblyNames = new ITaskItem[] { new TaskItem("Input") }; + task.ReferenceAssemblyPaths = new ITaskItem[] { new TaskItem(reference) }; + task.ExtraArgs = null; + string cacheDirectory = Path.Combine(test.Root, "cache"); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", enabled); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE_PATH", cacheDirectory); + + Directory.CreateDirectory(test.Output); + string stale = Path.Combine(test.Output, "stale.txt"); + File.WriteAllText(stale, "remove before linking"); + + Assert.True(task.Execute(), string.Join(Environment.NewLine, task.Messages.Select(message => message.Line))); + Assert.Empty(test.BuildEngine.Errors); + Assert.False(File.Exists(stale)); + Assert.Equal(File.ReadAllBytes(input), File.ReadAllBytes(Path.Combine(test.Output, "Input.dll"))); + Assert.False(Directory.Exists(cacheDirectory)); + if (bool.Parse(enabled)) + { + Assert.Contains(test.BuildEngine.Messages, message => + message.Message.StartsWith("ILLink cache bypassed: input identity could not be computed:") && + message.Message.Contains("no managed metadata")); + } + else + { + Assert.DoesNotContain(test.BuildEngine.Messages, message => + message.Message.StartsWith("ILLink caching is disabled") || + message.Message.StartsWith("ILLink cache bypassed:")); + } + }, enableCache.ToString()).Dispose(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void TaskCachesAndInvalidatesOutputs(bool corruptEntry) + { + RemoteExecutor.Invoke(static corrupt => + { + bool corruptEntry = bool.Parse(corrupt); + using var test = new OutputDirectoryTest(); + string input = Path.Combine(test.Root, "Input.dll"); + WriteTestAssembly(input); + var task = test.Task; + task.AssemblyPaths = new ITaskItem[] { new TaskItem(input, new Dictionary { { "TrimMode", "copy" } }) }; + task.RootAssemblyNames = new ITaskItem[] { new TaskItem("Input") }; + task.ExtraArgs = null; + string cacheDirectory = Path.Combine(test.Root, "cache"); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", "true"); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE_PATH", cacheDirectory); + + Assert.True(task.Execute()); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.StartsWith("ILLink cache stored:")); + string entry = Assert.Single(Directory.GetDirectories(Path.Combine(cacheDirectory, "v1"))); + string stale = Path.Combine(test.Output, "stale.txt"); + File.WriteAllText(stale, "not part of the cached directory"); + File.Delete(Path.Combine(test.Output, "Input.dll")); + test.BuildEngine.Messages.Clear(); + + Assert.True(task.Execute()); + Assert.Contains(test.BuildEngine.Messages, message => message.Message.StartsWith("ILLink cache hit:")); + Assert.False(File.Exists(stale)); + Assert.Equal(File.ReadAllBytes(input), File.ReadAllBytes(Path.Combine(test.Output, "Input.dll"))); + + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", "false"); + test.BuildEngine.Messages.Clear(); + File.Delete(Path.Combine(test.Output, "Input.dll")); + Assert.True(task.Execute()); + Assert.DoesNotContain(test.BuildEngine.Messages, message => message.Message.StartsWith("ILLink cache")); + Assert.Equal(File.ReadAllBytes(input), File.ReadAllBytes(Path.Combine(test.Output, "Input.dll"))); + Environment.SetEnvironmentVariable("ILLINK_EXPERIMENTAL_CACHE", "true"); + + if (corruptEntry) + File.WriteAllText(Path.Combine(entry, "outputs", "Input.dll"), "corrupt"); + else + File.AppendAllText(input, "changed"); + File.WriteAllText(stale, "remove before fallback execution"); + test.BuildEngine.Messages.Clear(); + + Assert.True(task.Execute()); + Assert.DoesNotContain(test.BuildEngine.Messages, message => message.Message.StartsWith("ILLink cache hit:")); + Assert.Contains(test.BuildEngine.Messages, message => + message.Message.StartsWith(corruptEntry ? "ILLink cache restore failed" : "ILLink cache miss:")); + Assert.False(File.Exists(stale)); + Assert.Equal(File.ReadAllBytes(input), File.ReadAllBytes(Path.Combine(test.Output, "Input.dll"))); + Assert.Equal(corruptEntry ? 1 : 2, Directory.GetDirectories(Path.Combine(cacheDirectory, "v1")).Length); + }, corruptEntry.ToString()).Dispose(); + } + + private static void WriteTestAssembly(string path, string informationalVersion = null, Guid? mvid = null) + { + using var assembly = Mono.Cecil.AssemblyDefinition.CreateAssembly( + new Mono.Cecil.AssemblyNameDefinition(Path.GetFileNameWithoutExtension(path), new Version(1, 0)), + Path.GetFileName(path), Mono.Cecil.ModuleKind.Dll); + if (mvid is Guid moduleVersionId) + assembly.MainModule.Mvid = moduleVersionId; + if (informationalVersion is not null) + { + var constructor = typeof(AssemblyInformationalVersionAttribute).GetConstructor(new[] { typeof(string) }); + var attribute = new Mono.Cecil.CustomAttribute(assembly.MainModule.ImportReference(constructor)); + attribute.ConstructorArguments.Add(new Mono.Cecil.CustomAttributeArgument(assembly.MainModule.TypeSystem.String, informationalVersion)); + assembly.CustomAttributes.Add(attribute); + } + assembly.Write(path); + } + + private static void WriteMetadataLessPE(string path) + { + WriteTestAssembly(path); + byte[] image = File.ReadAllBytes(path); + using (var pe = new PEReader(new MemoryStream(image))) + { + // The CLI header directory precedes the final reserved directory in the PE optional header. + const int DirectoryEntrySize = 2 * sizeof(int); + int cliHeaderDirectoryOffset = pe.PEHeaders.PEHeaderStartOffset + + pe.PEHeaders.CoffHeader.SizeOfOptionalHeader - 2 * DirectoryEntrySize; + Array.Clear(image, cliHeaderDirectoryOffset, DirectoryEntrySize); + } + File.WriteAllBytes(path, image); + using var reader = new PEReader(File.OpenRead(path)); + Assert.False(reader.HasMetadata); + } + + private static string PrepareCacheKeyTest(OutputDirectoryTest test) + { + string inputs = Path.Combine(test.Root, "inputs"); + Directory.CreateDirectory(inputs); + string input = Path.Combine(inputs, "Input.dll"); + string reference = Path.Combine(inputs, "Reference.dll"); + WriteTestAssembly(input); + WriteTestAssembly(reference); + foreach (string path in new[] + { + "roots.xml", "tools/Mono.Cecil.dll", "tools/Mono.Cecil.Mdb.dll", + "tools/Mono.Cecil.Pdb.dll", "tools/Mono.Cecil.Rocks.dll", "tools/illink.runtimeconfig.json", + "host/dotnet", "host/host/fxr/version/hostfxr", "host/shared/Microsoft.NETCore.App/version/runtime" + }) + { + string file = Path.Combine(test.Root, path); + Directory.CreateDirectory(Path.GetDirectoryName(file)); + File.WriteAllText(file, "contents"); + } + + test.Task.AssemblyPaths = new ITaskItem[] { new TaskItem(input) }; + test.Task.ReferenceAssemblyPaths = new ITaskItem[] { new TaskItem(reference) }; + test.Task.RootDescriptorFiles = new ITaskItem[] { new TaskItem(Path.Combine(test.Root, "roots.xml")) }; + test.Task.ILLinkPath = Path.Combine(test.Root, "tools", "illink.dll"); + WriteTestAssembly(test.Task.ILLinkPath, "1.0.0+test"); + test.Task.ExtraArgs = null; + return Path.Combine(test.Root, "host", "dotnet"); + } + private sealed class OutputDirectoryTest : IDisposable { public string Root { get; } = Path.Combine(Path.GetTempPath(), "illink-task-output-" + Guid.NewGuid().ToString("N")); diff --git a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.csproj b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.csproj index 5f3daab5629a06..132b8028e0fe19 100644 --- a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.csproj +++ b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.csproj @@ -4,6 +4,7 @@ + diff --git a/src/tools/illink/test/ILLink.Tasks.Tests/Mock.cs b/src/tools/illink/test/ILLink.Tasks.Tests/Mock.cs index 82cb976ecddf19..ea495b045162dc 100644 --- a/src/tools/illink/test/ILLink.Tasks.Tests/Mock.cs +++ b/src/tools/illink/test/ILLink.Tasks.Tests/Mock.cs @@ -18,6 +18,24 @@ public class MockTask : ILLink public List<(MessageImportance Importance, string Line)> Messages { get; } = new List<(MessageImportance Importance, string Line)>(); + public string GetResponseFileCommands() => GenerateResponseFileCommands(); + + public bool TryGetCacheKey(string pathToTool, out string key) + { + // Match the normalization ToolTask.Execute applies before calling ExecuteTool. + string commandLine = GenerateCommandLineCommands(); + commandLine = string.IsNullOrEmpty(commandLine) ? string.Empty : " " + commandLine; + object[] arguments = { + pathToTool, + AdjustCommandsForOperatingSystem(commandLine), + AdjustCommandsForOperatingSystem(GenerateResponseFileCommands()), + null + }; + bool result = (bool)typeof(ILLink).GetMethod("TryComputeCacheKey", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(this, arguments); + key = (string)arguments[3]; + return result; + } + public MockTask() { // Ensure that [Required] members are non-null @@ -77,9 +95,11 @@ public static IEnumerable GetOptimizationPropertyNames() public class MockBuildEngine : IBuildEngine { public List Errors { get; } = new List(); + public List Messages { get; } = new(); + public void LogErrorEvent(BuildErrorEventArgs e) => Errors.Add(e); public void LogWarningEvent(BuildWarningEventArgs e) { } - public void LogMessageEvent(BuildMessageEventArgs e) { } + public void LogMessageEvent(BuildMessageEventArgs e) => Messages.Add(e); public void LogCustomEvent(CustomBuildEventArgs e) { } public bool BuildProjectFile(string projectFileName, string[] targetNames, IDictionary globalProperties, IDictionary targetOutputs) => false; public bool ContinueOnError => false; From 64b68608f11ca4a7d0297defd4cd860216b7c5d3 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Fri, 2 Oct 2026 16:51:19 -0700 Subject: [PATCH 02/12] Fix ILLink cache analyzer diagnostics Assisted-by: Copilot:gpt-6-astra Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- src/tools/illink/src/ILLink.CacheTool/Program.cs | 5 +++-- src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs | 7 ++++++- src/tools/illink/src/ILLink.Tasks/LinkTask.cs | 4 ++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/tools/illink/src/ILLink.CacheTool/Program.cs b/src/tools/illink/src/ILLink.CacheTool/Program.cs index b3fe7a5ba17113..6cf7a19cb729d4 100644 --- a/src/tools/illink/src/ILLink.CacheTool/Program.cs +++ b/src/tools/illink/src/ILLink.CacheTool/Program.cs @@ -10,6 +10,8 @@ namespace ILLink.CacheTool; internal static class Program { + private static readonly string[] s_cutoffFormats = { "yyyy-MM-dd'T'HH:mm:ss.FFFFFFF'Z'", "yyyy-MM-dd'T'HH:mm:ss.FFFFFFFzzz" }; + private static int Main(string[] args) => Run(args, Console.Out, Console.Error); internal static int Run(string[] args, TextWriter output, TextWriter error) @@ -27,8 +29,7 @@ internal static int Run(string[] args, TextWriter output, TextWriter error) CustomParser = result => { string value = result.Tokens[0].Value; - if (DateTimeOffset.TryParseExact(value, - new[] { "yyyy-MM-dd'T'HH:mm:ss.FFFFFFF'Z'", "yyyy-MM-dd'T'HH:mm:ss.FFFFFFFzzz" }, + if (DateTimeOffset.TryParseExact(value, s_cutoffFormats, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset cutoff) && cutoff.Offset == TimeSpan.Zero) { diff --git a/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs b/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs index 1e302c4c1b5c6e..70770619c2935d 100644 --- a/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs +++ b/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs @@ -266,9 +266,14 @@ private string GetWriterMutexName(string inputHash) if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) directory = directory.ToUpperInvariant(); +#if NET + byte[] hash = SHA256.HashData(Encoding.UTF8.GetBytes(directory + "\n" + inputHash)); + return @"Global\ILLinkCache-" + Convert.ToHexString(hash); +#else using var sha256 = SHA256.Create(); byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(directory + "\n" + inputHash)); return @"Global\ILLinkCache-" + BitConverter.ToString(hash).Replace("-", ""); +#endif } private string ValidateOutputDirectory(string outputDirectory) @@ -339,7 +344,7 @@ private static void ValidateRelativePath(string path) foreach (string component in path.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)) { if (component is "" or "." or ".." || component.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || - (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (component.EndsWith(".", StringComparison.Ordinal) || component.EndsWith(" ", StringComparison.Ordinal)))) + (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && component[component.Length - 1] is '.' or ' ')) throw new InvalidDataException("Invalid ILLink cache output path."); } } diff --git a/src/tools/illink/src/ILLink.Tasks/LinkTask.cs b/src/tools/illink/src/ILLink.Tasks/LinkTask.cs index 9f2c27e9cf3353..900b945591367b 100644 --- a/src/tools/illink/src/ILLink.Tasks/LinkTask.cs +++ b/src/tools/illink/src/ILLink.Tasks/LinkTask.cs @@ -355,7 +355,11 @@ private bool TryComputeCacheKey(string pathToTool, string commandLineCommands, s writer.Flush(); description.Position = 0; +#if NET + inputHash = Convert.ToHexStringLower(sha256.ComputeHash(description)); +#else inputHash = BitConverter.ToString(sha256.ComputeHash(description)).Replace("-", "").ToLowerInvariant(); +#endif return true; void AddFile(string path) => files.Add(Path.GetFullPath(path)); From 37c52190696e10059702e425573f5139a778f915 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Fri, 2 Oct 2026 16:52:05 -0700 Subject: [PATCH 03/12] Fix ILLink cache purge test on case-insensitive filesystems Assisted-by: Copilot:gpt-6-astra Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs b/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs index e515946628b363..7be98b73049b36 100644 --- a/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs +++ b/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs @@ -35,7 +35,7 @@ public void PurgeOnlyVisitsPublishedEntriesInCurrentLayout() using var cache = new TestCache(); string old = cache.AddEntry('a', TestCache.Cutoff.AddDays(-1)); string recent = cache.AddEntry('b', TestCache.Cutoff.AddDays(1)); - foreach (string name in new[] { new string('a', 64) + ".unique.tmp", "unrelated", new string('A', 64), new string('c', 63) }) + foreach (string name in new[] { new string('a', 64) + ".unique.tmp", "unrelated", new string('E', 64), new string('c', 63) }) { Directory.CreateDirectory(Path.Combine(cache.Entries, name)); } From 9f2118f7303ba5aea67874ef8019266a9451887e Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Mon, 5 Oct 2026 09:45:41 -0700 Subject: [PATCH 04/12] Simplify ILLink cache last-used marker update Replace the write-to-temp + File.Replace + retry approach with a plain overwrite. File.Replace's ReplaceFile API isn't meant for concurrent writers to the same destination, which caused a transient Windows CI failure. A plain File.WriteAllText is safe here: FileShare.Read makes the OS serialize conflicting writers, so a losing writer's open fails before touching the file. The marker is advisory bookkeeping, and CachePurge.cs already tolerates malformed content per entry. Assisted-by: Claude:claude-sonnet-5 (GitHub Copilot CLI) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs | 16 +--------------- .../ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs | 1 - 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs b/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs index 70770619c2935d..6932c6a6bfc0e2 100644 --- a/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs +++ b/src/tools/illink/src/ILLink.Tasks/ILLinkCache.cs @@ -228,28 +228,14 @@ internal void Store(string inputHash, string outputDirectory) private void UpdateLastUsed(string entryDirectory) { - string temporaryFile = Path.Combine(entryDirectory, ILLinkCacheEntry.LastUsedFileName + "." + Guid.NewGuid().ToString("N") + ".tmp"); try { - ILLinkCacheEntry.WriteLastUsed(temporaryFile); - // Replace rather than overwrite so parallel restores cannot leave a partially written marker. - File.Replace(temporaryFile, Path.Combine(entryDirectory, ILLinkCacheEntry.LastUsedFileName), null); + ILLinkCacheEntry.WriteLastUsed(Path.Combine(entryDirectory, ILLinkCacheEntry.LastUsedFileName)); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { _log.LogMessage(MessageImportance.Low, $"ILLink cache last-used update failed for '{entryDirectory}': {ex.Message}"); } - finally - { - try - { - File.Delete(temporaryFile); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - _log.LogMessage(MessageImportance.Low, $"ILLink cache last-used cleanup failed for '{temporaryFile}': {ex.Message}"); - } - } } private string GetEntryDirectory(string inputHash) diff --git a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs index 47405f2ae6fbae..58dad011b7cc07 100644 --- a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs +++ b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs @@ -1266,7 +1266,6 @@ static RemoteInvokeHandle StartReader(CacheTestDirectory test, int index) => for (int attempt = 0; attempt < 20; attempt++) Assert.True(cache.TryRestore(CacheTestDirectory.Key, output, CacheTestDirectory.Timestamp)); Assert.Empty(engine.Errors); - Assert.DoesNotContain(engine.Messages, message => message.Message.Contains("last-used update failed")); }, test.Cache.CacheDirectory, Path.Combine(test.Root, "reader-" + index), Path.Combine(test.Root, "start")); } From 72f0f94d975ccff3730cb19c338bee6d5c7fac8d Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Tue, 6 Oct 2026 14:44:16 -0700 Subject: [PATCH 05/12] Fall back to directory creation time for invalid ILLink cache markers Keep purge usable if a best-effort last-used refresh leaves an invalid marker. Report the fallback while preserving errors for actual maintenance failures. Assisted-by: GitHub Copilot CLI:gpt-6-sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- docs/design/tools/illink/task-cache.md | 6 ++-- docs/tools/illink/illink-tasks.md | 11 ++++--- .../illink/src/ILLink.CacheTool/CachePurge.cs | 29 +++++++++++++--- .../illink/src/ILLink.CacheTool/README.md | 5 +-- .../ILLink.CacheTool.Tests/CachePurgeTests.cs | 33 ++++++++++++------- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/docs/design/tools/illink/task-cache.md b/docs/design/tools/illink/task-cache.md index 6fac3a8e50b428..21ca8a50e85aa5 100644 --- a/docs/design/tools/illink/task-cache.md +++ b/docs/design/tools/illink/task-cache.md @@ -47,7 +47,7 @@ internal sealed class ILLinkCache - Under the lock, recheck whether the entry already exists. If absent, populate staging and publish with an atomic, same-filesystem directory rename without overwriting an existing entry. - Rely on the documented `Directory.Move` contract that an existing destination causes `IOException`, even if that destination is empty. The writer lock avoids duplicate staging work; it is not a workaround for publication semantics. Do not add platform-specific interop or require non-empty entries to compensate for runtime/filesystem bugs. Atomic visibility does not imply power-loss durability. - Create cache directories lazily in `Store`, after acquiring the writer lock and confirming the entry is absent. `TryCreate` only resolves the path; `TryRestore` treats a missing directory as a miss. Directory-creation failures are logged without failing a successful link. -- Keep published outputs and manifests immutable and read them without locks. Write a UTC `last-used` timestamp in staging before publication and refresh it after a successful restore. Replace the marker atomically so parallel restores do not leave partial timestamps; usage-update failures are logged without failing linking. +- Keep published outputs and manifests immutable and read them without locks. Write a UTC `last-used` timestamp in staging before publication and refresh it after a successful restore. Marker refreshes are best-effort overwrites; usage-update failures are logged without failing linking. An interrupted write may leave an invalid marker, which purge handles using the entry directory's creation time. - Use a named `System.Threading.Mutex` derived from the resolved cache-directory path and input hash. An abandoned mutex grants ownership; recheck the entry and use fresh staging. This coordinates processes on one machine, not writers on different machines sharing a network directory. - Support concurrent task invocations with distinct output directories sharing a cache directory. Concurrent linking to the same output directory is unsupported. - Treat missing or unreadable cache data as a failed restore, never a successful hit. @@ -79,8 +79,8 @@ internal sealed class ILLinkCache - No automatic eviction or size policy during linking. Users may manually wipe the cache directory or explicitly run the experimental `dotnet-illink-cache purge` tool. - Do not repair or replace corrupt published entries. Restoration falls back to linking, but storage still skips an existing entry until the user removes it. - No `created` bookkeeping marker. The `last-used` marker records usage for explicit post-build maintenance. -- Purge requires an explicit cache root and UTC cutoff. Only published SHA-256 entry directories in `v1` are considered; delete entries whose marker is strictly older than the cutoff and retain equality. Leave staging directories and other layouts alone. Report per-entry errors and continue, then return nonzero if maintenance failed. -- Assume old caches are cleared before using markers. Do not infer ages or migrate entries with missing markers; missing, malformed, or unreadable markers are ordinary reported errors. +- Purge requires an explicit cache root and UTC cutoff. Only published SHA-256 entry directories in `v1` are considered; delete entries whose last-used time is strictly older than the cutoff and retain equality. Leave staging directories and other layouts alone. Report per-entry maintenance errors and continue, then return nonzero if maintenance failed. +- For a missing, malformed, non-UTC, or unreadable marker, report the fallback and use the entry directory's creation time instead. This may purge a recently used entry or retain a stale one if restoring a cache snapshot resets directory creation times. Do not migrate or repair the marker. - Purge must not run during any build using the cache. Do not add purge locks or change reader locking; coordination with active readers/writers remains a separate follow-up, as does protection against manual deletion. - CI may record build start after restoring a job-private cache, run linking, then purge against that timestamp before uploading the next snapshot. Preserve tool diagnostics but do not fail the build for maintenance errors. Only purge once all users have exited; incremental skips and unreached work in failed builds do not refresh usage. See the [CI recipe](../../../tools/illink/illink-tasks.md#ci-purge-ordering). This does not enable an ILLink cache in CI automatically. - Attempt best-effort cleanup of the current attempt's unpublished staging directory. Log cleanup I/O/access failures without failing a successful link; do not scan or delete other attempts' staging directories. Correctness must tolerate leftovers after crashes. diff --git a/docs/tools/illink/illink-tasks.md b/docs/tools/illink/illink-tasks.md index b13c6168e719d6..209a3f8b7a631b 100644 --- a/docs/tools/illink/illink-tasks.md +++ b/docs/tools/illink/illink-tasks.md @@ -95,11 +95,12 @@ directories, unrelated directories, and other layout versions are not purged. Th size limit or automatic eviction during linking. **Run purge only when no build is using that cache.** Purge and manual deletion are not -coordinated with readers or writers. Clear old caches before adopting usage markers; no -migration or fallback age is provided. Missing, malformed, or unreadable markers are reported -as errors and their entries are retained. The tool reports deleted/kept/error counts and -returns nonzero for argument or maintenance failures. Usage-marker update failures in the -task are logged without failing linking. +coordinated with readers or writers. Missing, malformed, non-UTC, or unreadable markers use +the entry directory's creation time as a fallback; the tool reports this on stderr without +counting it as an error. Such entries may be deleted despite recent use, or retained if +restoring a cache snapshot reset directory creation times. The tool reports +deleted/kept/error counts and returns nonzero for argument or maintenance failures. +Usage-marker update failures in the task are logged without failing linking. ### CI purge ordering diff --git a/src/tools/illink/src/ILLink.CacheTool/CachePurge.cs b/src/tools/illink/src/ILLink.CacheTool/CachePurge.cs index af2d3e18503a5f..a7e4ebfc21b21b 100644 --- a/src/tools/illink/src/ILLink.CacheTool/CachePurge.cs +++ b/src/tools/illink/src/ILLink.CacheTool/CachePurge.cs @@ -25,11 +25,7 @@ internal static int Run(string cacheDirectory, DateTimeOffset cutoff, TextWriter try { - DateTimeOffset lastUsed = DateTimeOffset.ParseExact( - File.ReadAllText(Path.Combine(entry, ILLinkCacheEntry.LastUsedFileName)), - "O", CultureInfo.InvariantCulture); - if (lastUsed.Offset != TimeSpan.Zero) - throw new InvalidDataException("The last-used timestamp must be UTC."); + DateTimeOffset lastUsed = GetLastUsed(entry, error); if (lastUsed >= cutoff) { @@ -56,4 +52,27 @@ internal static int Run(string cacheDirectory, DateTimeOffset cutoff, TextWriter output.WriteLine($"ILLink cache purge: Deleted: {deleted}, Kept: {kept}, Errors: {errors}"); return errors == 0 ? 0 : 1; } + + private static DateTimeOffset GetLastUsed(string entry, TextWriter error) + { + string marker = Path.Combine(entry, ILLinkCacheEntry.LastUsedFileName); + string reason = "missing or invalid"; + try + { + if (File.Exists(marker) && + DateTimeOffset.TryParseExact(File.ReadAllText(marker), "O", CultureInfo.InvariantCulture, + DateTimeStyles.None, out DateTimeOffset lastUsed) && + lastUsed.Offset == TimeSpan.Zero) + { + return lastUsed; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + reason = ex.Message; + } + + error.WriteLine($"Could not read a valid last-used marker for ILLink cache entry '{entry}' ({reason}); using directory creation time."); + return new DateTimeOffset(Directory.GetCreationTimeUtc(entry), TimeSpan.Zero); + } } diff --git a/src/tools/illink/src/ILLink.CacheTool/README.md b/src/tools/illink/src/ILLink.CacheTool/README.md index 08d9f4c43a9566..e8dfd8b679f94d 100644 --- a/src/tools/illink/src/ILLink.CacheTool/README.md +++ b/src/tools/illink/src/ILLink.CacheTool/README.md @@ -11,11 +11,12 @@ dotnet illink-cache purge --cache-directory /path/to/cache --before 2026-10-01T1 Both options are required. The cutoff must be an ISO 8601 UTC timestamp, with `Z` or `+00:00`. Entries last used strictly before the cutoff are deleted; entries at or after it are kept. The tool prints deleted, kept, and error counts. Argument or maintenance errors return a -nonzero exit code. Missing or invalid usage markers are reported as errors, not assigned an age. +nonzero exit code. Missing, invalid, or unreadable usage markers use the entry directory's +creation time instead; this fallback is reported on stderr but does not count as an error. **No build may use the cache during purging.** The tool does not coordinate with active readers or writers. Staging directories and unrelated directories are left alone. -Clear caches created before usage markers were introduced; there is no migration support. +There is no migration of entries created before usage markers were introduced. ## Building locally diff --git a/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs b/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs index 7be98b73049b36..14f15cfe20c7ac 100644 --- a/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs +++ b/src/tools/illink/test/ILLink.CacheTool.Tests/CachePurgeTests.cs @@ -54,11 +54,17 @@ public void PurgeOnlyVisitsPublishedEntriesInCurrentLayout() } [Theory] - [InlineData("missing")] - [InlineData("malformed")] - [InlineData("directory")] - [InlineData("non-utc")] - public void InvalidMarkerReportsErrorAndDoesNotDeleteEntry(string damage) + [InlineData("missing", false)] + [InlineData("malformed", false)] + [InlineData("empty", false)] + [InlineData("directory", false)] + [InlineData("non-utc", false)] + [InlineData("missing", true)] + [InlineData("malformed", true)] + [InlineData("empty", true)] + [InlineData("directory", true)] + [InlineData("non-utc", true)] + public void InvalidMarkerUsesEntryCreationTime(string damage, bool delete) { using var cache = new TestCache(); string damagedEntry = cache.AddEntry('a', TestCache.Cutoff.AddDays(-1)); @@ -72,6 +78,9 @@ public void InvalidMarkerReportsErrorAndDoesNotDeleteEntry(string damage) case "malformed": File.WriteAllText(marker, "invalid timestamp"); break; + case "empty": + File.WriteAllText(marker, ""); + break; case "directory": File.Delete(marker); Directory.CreateDirectory(marker); @@ -81,12 +90,14 @@ public void InvalidMarkerReportsErrorAndDoesNotDeleteEntry(string damage) break; } - Assert.Equal(1, cache.Run()); + Assert.Equal(0, cache.Run(delete ? DateTimeOffset.UtcNow.AddMinutes(1) : TestCache.Cutoff)); - Assert.True(Directory.Exists(damagedEntry)); + Assert.Equal(!delete, Directory.Exists(damagedEntry)); Assert.False(Directory.Exists(validEntry)); Assert.Contains(damagedEntry, cache.Error.ToString()); - Assert.Contains("Deleted: 1, Kept: 0, Errors: 1", cache.Output.ToString()); + Assert.Contains("using directory creation time", cache.Error.ToString()); + Assert.Equal($"ILLink cache purge: Deleted: {(delete ? 2 : 1)}, Kept: {(delete ? 0 : 1)}, Errors: 0{Environment.NewLine}", + cache.Output.ToString()); } [Fact] @@ -186,12 +197,10 @@ public void CommandRejectsNonexistentDirectory() public void CommandPropagatesMaintenanceFailure() { using var cache = new TestCache(); - string entry = cache.AddEntry('a', TestCache.Cutoff.AddDays(-1)); - File.Delete(Path.Combine(entry, ILLinkCacheEntry.LastUsedFileName)); + Directory.Delete(cache.Entries); Assert.Equal(1, Program.Run(new[] { "purge", "--cache-directory", cache.Root, "--before", "2025-01-02T03:04:05Z" }, cache.Output, cache.Error)); - Assert.True(Directory.Exists(entry)); Assert.Contains("Errors: 1", cache.Output.ToString()); } @@ -224,7 +233,7 @@ internal string AddEntry(char key, DateTimeOffset lastUsed) return entry; } - internal int Run() => CachePurge.Run(Root, Cutoff, Output, Error); + internal int Run(DateTimeOffset? cutoff = null) => CachePurge.Run(Root, cutoff ?? Cutoff, Output, Error); public void Dispose() { From 0932cf7e480801311454e13f3d1c6c9b615f6677 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Tue, 13 Oct 2026 14:11:55 -0700 Subject: [PATCH 06/12] Remove redundant ILLink ExtraArgs cache-key test case The task execution test already verifies that unsupported ExtraArgs bypass caching. Drop the overlapping key test case and its unused switch arm. Assisted-by: Copilot:gpt-6-sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- .../illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs index 58dad011b7cc07..90cc266ac8eac1 100644 --- a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs +++ b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs @@ -1854,7 +1854,6 @@ public void CacheKeyIgnoresHostDirectories(string relativePath) } [Theory] - [InlineData("extra-args")] [InlineData("custom-step")] [InlineData("custom-data")] [InlineData("dump-dependencies")] @@ -1871,9 +1870,6 @@ public void CacheKeyBypassesUnsupportedInputs(string reason) string host = PrepareCacheKeyTest(test); switch (reason) { - case "extra-args": - test.Task.ExtraArgs = "--help"; - break; case "custom-step": test.Task.CustomSteps = new ITaskItem[] { From 896eb0b18c7a7fae523fbb7a63d4c7ae747f3b60 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Thu, 1 Oct 2026 10:49:54 -0700 Subject: [PATCH 07/12] Support ILLink caching in runtime library builds Use structured task parameters for runtime trimming and model the narrow extra-argument shapes needed by shared-framework, out-of-band, and per-library builds. Preserve custom argument precedence. Assisted-by: Copilot:gpt-6-astra Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- docs/design/tools/illink/task-cache.md | 2 +- docs/tools/illink/illink-tasks.md | 8 +- eng/illink.targets | 52 ++++-- src/libraries/oob-trim.proj | 5 +- src/libraries/sfx-finish.proj | 3 +- src/tools/illink/src/ILLink.Tasks/LinkTask.cs | 103 +++++++++++- .../ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs | 157 +++++++++++++++++- 7 files changed, 301 insertions(+), 29 deletions(-) diff --git a/docs/design/tools/illink/task-cache.md b/docs/design/tools/illink/task-cache.md index 21ca8a50e85aa5..55662b0ce75432 100644 --- a/docs/design/tools/illink/task-cache.md +++ b/docs/design/tools/illink/task-cache.md @@ -62,7 +62,7 @@ internal sealed class ILLinkCache - Defer identifying the runtime that executes ILLink; do not include a runtime version or framework description in the key. The task runs in MSBuild, so its CoreLib informational version and framework description are not reliable identities for the child linker's runtime. The runtimeconfig describes requested frameworks and roll-forward policy, not the exact resolved CoreLib. Finding that CoreLib would require resolved-runtime information from the caller, host-resolution plumbing, or an additional process; defer that complexity and startup cost. - This deliberately accepts that runtime installation changes not represented by other keyed inputs may reuse an entry; callers needing invalidation for those changes must clear or isolate the cache. - Include platform, architecture, and culture. Do not enumerate or hash inherited environment variables, or bypass caching based on them. This deliberately keeps v1 simple rather than maintaining a partial list of runtime/loader settings. Environment changes are reflected only when they change other keyed inputs, such as the selected host path or culture. Callers must disable caching when inherited settings introduce otherwise untracked dependencies, affect outputs, or require tool-execution side effects (for example, startup hooks or profilers). -- Bypass caching for non-whitespace `ExtraArgs`, custom steps/data, dependency-dump options, and explicit `ToolTask.EnvironmentVariables` overrides. These are caller-supplied inputs that v1 does not model in its key and can introduce undeclared dependencies, external outputs, or side effects. These invocations still run the linker normally; no extra-argument parsing or runtime-specific exceptions are supported. +- Bypass caching for arbitrary `ExtraArgs`, custom steps/data, dependency-dump options, and explicit `ToolTask.EnvironmentVariables` overrides. These are caller-supplied inputs that v1 does not model in its key and can introduce undeclared dependencies, external outputs, or side effects. The supported `ExtraArgs` shape for runtime's library builds is `--ignore-link-attributes true` followed by zero or more `--link-attributes FILE`, `--substitutions FILE`, or `-d DIRECTORY` pairs. Hash every referenced XML file and every top-level `.dll`, `.exe`, and `.winmd` candidate in each search directory, including assembly sidecars/modules/resources. Preserve directory order in the arguments and leave linker resolution unchanged; adding, changing, or removing candidates invalidates the key, including shadowed candidates. An unreadable directory or uninspectable candidate bypasses caching. All other SDK options passed through `ExtraArgs` remain subject to the bypass. - Log and bypass caching if required files cannot be read or an input assembly cannot be inspected, including PE images without managed metadata. Normal linking determines whether those inputs are acceptable. Input files must remain stable during hashing and linking. ## Entry layout diff --git a/docs/tools/illink/illink-tasks.md b/docs/tools/illink/illink-tasks.md index 209a3f8b7a631b..c6f092e7fca48e 100644 --- a/docs/tools/illink/illink-tasks.md +++ b/docs/tools/illink/illink-tasks.md @@ -67,9 +67,11 @@ clear or isolate the cache when that distinction is required. Cache hits replace the output directory without running ILLink. Directories are created in the cache only when storing a successful result. Hits do not replay warnings or other linker diagnostics. -Caching is bypassed with a diagnostic for non-whitespace `ExtraArgs`, custom steps/data, -dependency-dump options, and explicit task environment overrides. Options supplied through -`ExtraArgs` remain unsupported for caching; these invocations still run the linker normally. +Caching is bypassed with a diagnostic for arbitrary `ExtraArgs`, custom steps/data, +dependency-dump options, and explicit task environment overrides. Runtime's library-build +arguments for link attributes, substitutions, and assembly search directories are supported. +Their XML files and search-directory assembly candidates (including sidecars) are content-hashed, +without changing assembly resolution. Other SDK options supplied through `ExtraArgs` remain unsupported. Inherited environment variables are not tracked; disable caching when they affect outputs or dependencies beyond the keyed inputs, or require tool-execution side effects such as startup hooks or profiling. Unreadable inputs or linker files also diff --git a/eng/illink.targets b/eng/illink.targets index 248e5f6a34e50a..714ac859810554 100644 --- a/eng/illink.targets +++ b/eng/illink.targets @@ -178,10 +178,9 @@ + <_ILLinkHasCustomArgs Condition="'$(ILLinkArgs)' != ''">true $(ILLinkArgs) --ignore-link-attributes true - - $(ILLinkArgs) --skip-unresolved true + $(LinkerNoWarn);IL2008;IL2009;IL2121;IL2025;IL2035 + + + + $(ILLinkArgs) --trim-mode skip @@ -215,18 +226,30 @@ $(ILLinkArgs) -x "@(ILLinkDescriptorsLibraryBuildXml->'%(FullPath)', '" -x "')" $(ILLinkArgs) --substitutions "$(ILLinkSubstitutionsLibraryBuildXml)" $(ILLinkArgs) --link-attributes "@(ILLinkSuppressionsLibraryBuildXml->'%(FullPath)', '" --link-attributes "')" - - $(LinkerNoWarn);IL2008;IL2009;IL2121;IL2025;IL2035 $(ILLinkArgs) --nowarn $(LinkerNoWarn) $(ILLinkArgs) --disable-opt ipconstprop + + <_ILLinkTrimMode>skip + <_ILLinkDefaultAction>skip + <_ILLinkNoWarn>$(LinkerNoWarn) + <_ILLinkRemoveSymbols>true + <_ILLinkRemoveSymbols Condition="'$(ILLinkRewritePDBs)' == 'true' and Exists('$(ILLinkTrimAssemblySymbols)')">false + <_ILLinkPreserveSymbolPaths Condition="'$(_ILLinkRemoveSymbols)' == 'false' and '$(DeterministicSourcePaths)' == 'true'">true + <_ILLinkIPConstProp Condition="'$(ILLinkDisableIPConstProp)' == 'true'">false + $(ILLinkArgs) --substitutions "$(ILLinkSubstitutionsLibraryBuildXml)" + $(ILLinkArgs) --link-attributes "@(ILLinkSuppressionsLibraryBuildXml->'%(FullPath)', '" --link-attributes "')" + + + + <_ILLinkAssemblyToTrim Include="$(ILLinkTrimInputPath)$(TargetName)$(TargetExt)"> + link + + <_ILLinkRootDescriptors Include="@(ILLinkDescriptorsLibraryBuildXml->'%(FullPath)')" + Condition="'$(_ILLinkHasCustomArgs)' != 'true'" /> + + @@ -264,8 +287,15 @@ $(ILLinkArgs) -d @(_DependencyDirectories->'"%(Identity)"', ' -d ') - $(NetCoreAppCurrent)-$(TargetOS) true + + IL2121 @@ -37,8 +39,6 @@ $(ILLinkArgs) - - $(ILLinkArgs) --nowarn IL2121 $(OOBILLinkArgs) --link-attributes "@(OOBLibrarySuppressionsXml->'%(FullPath)', '" --link-attributes "')" @@ -52,6 +52,7 @@ ReferenceAssemblyPaths="@(OOBAssemblyReference)" OutputDirectory="$(OOBAssembliesTrimDir)" ExtraArgs="$(OOBILLinkArgs)" + NoWarn="$(OOBILLinkNoWarn)" ToolExe="$(_DotNetHostFileName)" ToolPath="$(_DotNetHostDirectory)" /> diff --git a/src/libraries/sfx-finish.proj b/src/libraries/sfx-finish.proj index 47e300d3adc289..a0af3730354192 100644 --- a/src/libraries/sfx-finish.proj +++ b/src/libraries/sfx-finish.proj @@ -124,8 +124,6 @@ $(ILLinkArgs) - - $(SharedFrameworkILLinkArgs) -b true $(SharedFrameworkILLinkArgs) --link-attributes "@(SharedFrameworkSuppressionsXml, '" --link-attributes "')" @@ -141,6 +139,7 @@ RootAssemblyNames="@(_SharedFrameworkRootAssemblyName)" OutputDirectory="$(SharedFrameworkAssembliesTrimDir)" ExtraArgs="$(SharedFrameworkILLinkArgs)" + RemoveSymbols="false" ToolExe="$(_DotNetHostFileName)" ToolPath="$(_DotNetHostDirectory)" /> diff --git a/src/tools/illink/src/ILLink.Tasks/LinkTask.cs b/src/tools/illink/src/ILLink.Tasks/LinkTask.cs index 900b945591367b..0f43e75a857db0 100644 --- a/src/tools/illink/src/ILLink.Tasks/LinkTask.cs +++ b/src/tools/illink/src/ILLink.Tasks/LinkTask.cs @@ -310,15 +310,16 @@ private bool TryComputeCacheKey(string pathToTool, string commandLineCommands, s return false; } - if (!string.IsNullOrWhiteSpace(ExtraArgs)) - { - Log.LogMessage(MessageImportance.Low, "ILLink cache bypassed: extra arguments are not supported."); - return false; - } - try { var files = new SortedSet(StringComparer.Ordinal); + var directories = new SortedSet(StringComparer.Ordinal); + if (!TryAddCacheableExtraArgsInputs(files, directories)) + { + Log.LogMessage(MessageImportance.Low, "ILLink cache bypassed: extra arguments are not supported."); + return false; + } + var assemblies = new HashSet(StringComparer.Ordinal); foreach (ITaskItem assembly in AssemblyPaths) AddAssembly(assembly.ItemSpec); @@ -327,6 +328,22 @@ private bool TryComputeCacheKey(string pathToTool, string commandLineCommands, s foreach (ITaskItem descriptor in RootDescriptorFiles ?? Array.Empty()) AddFile(descriptor.ItemSpec); + // Preserve AssemblyResolver's search behavior instead of replacing -d with + // explicit references. Include every candidate so additions and shadowing invalidate. + foreach (string directory in directories) + { + foreach (string file in Directory.EnumerateFiles(directory)) + { + string extension = Path.GetExtension(file); + if (extension.Equals(".dll", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".exe", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".winmd", StringComparison.OrdinalIgnoreCase)) + { + AddAssembly(file); + } + } + } + #pragma warning disable IL3000 // MSBuild tasks are loaded from assemblies on disk. AddFile(typeof(ILLink).Assembly.Location); #pragma warning restore IL3000 @@ -408,6 +425,80 @@ void AddAssembly(string path) } } + private bool TryAddCacheableExtraArgsInputs(SortedSet files, SortedSet directories) + { + ReadOnlySpan arguments = ExtraArgs.AsSpan().Trim(); + if (arguments.IsEmpty) + return true; + + // The runtime library builds pass this policy through ExtraArgs. Keep the + // supported shape narrow so arbitrary linker arguments continue to bypass caching. + if (!TryReadArgument(ref arguments, out string option) || + option != "--ignore-link-attributes" || + !TryReadArgument(ref arguments, out string value) || + value != "true") + { + return false; + } + + while (!arguments.TrimStart().IsEmpty) + { + if (!TryReadArgument(ref arguments, out option) || + option is not ("--link-attributes" or "--substitutions" or "-d") || + !TryReadArgument(ref arguments, out string path) || + string.IsNullOrEmpty(path)) + { + return false; + } + + if (option == "-d") + directories.Add(Path.GetFullPath(path)); + else + files.Add(Path.GetFullPath(path)); + } + + return true; + + static bool TryReadArgument(ref ReadOnlySpan arguments, out string argument) + { + arguments = arguments.TrimStart(); + if (arguments.IsEmpty) + { + argument = string.Empty; + return false; + } + + if (arguments[0] == '"') + { + int closingQuote = arguments.Slice(1).IndexOf('"'); + if (closingQuote < 0) + { + argument = string.Empty; + return false; + } + + argument = arguments.Slice(1, closingQuote).ToString(); + arguments = arguments.Slice(closingQuote + 2); + return arguments.IsEmpty || char.IsWhiteSpace(arguments[0]); + } + + int end = 0; + while (end < arguments.Length && !char.IsWhiteSpace(arguments[end])) + end++; + + ReadOnlySpan token = arguments.Slice(0, end); + if (token.IndexOf('"') >= 0) + { + argument = string.Empty; + return false; + } + + argument = token.ToString(); + arguments = arguments.Slice(end); + return true; + } + } + private string _illinkPath = ""; public string ILLinkPath diff --git a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs index 90cc266ac8eac1..84135d1a807f87 100644 --- a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs +++ b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs @@ -1629,11 +1629,144 @@ public void CacheKeyTracksFileContentsNotTimestamps(string relativePath) } [Theory] - [InlineData("--help")] - [InlineData("--ignore-link-attributes true")] + [InlineData("--link-attributes", 0)] + [InlineData("--link-attributes", 1)] + [InlineData("--substitutions", 0)] + [InlineData("--substitutions", 1)] + public void CacheKeyTracksRuntimeXmlFiles(string option, int changedFile) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string[] linkAttributesFiles = + { + Path.Combine(test.Root, "first link attributes.xml"), + Path.Combine(test.Root, "second link attributes.xml") + }; + foreach (string file in linkAttributesFiles) + File.WriteAllText(file, "contents"); + + test.Task.ExtraArgs = $" --ignore-link-attributes true {option} \"{linkAttributesFiles[0]}\" {option} \"{linkAttributesFiles[1]}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + + File.AppendAllText(linkAttributesFiles[changedFile], "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(original, changed); + } + + [Theory] + [InlineData(".dll")] + [InlineData(".exe")] + [InlineData(".winmd")] + public void CacheKeyTracksRuntimeSearchDirectoryCandidates(string extension) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string first = Path.Combine(test.Root, "first search directory"); + string second = Path.Combine(test.Root, "second search directory"); + Directory.CreateDirectory(first); + Directory.CreateDirectory(second); + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{first}\" -d \"{second}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string empty)); + + string candidate = Path.Combine(second, "Dependency" + extension); + WriteTestAssembly(candidate); + Assert.True(test.Task.TryGetCacheKey(host, out string added)); + Assert.NotEqual(empty, added); + WriteTestAssembly(candidate, "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(added, changed); + + string earlierCandidate = Path.Combine(first, Path.GetFileName(candidate)); + File.Copy(candidate, earlierCandidate); + Assert.True(test.Task.TryGetCacheKey(host, out string shadowed)); + Assert.NotEqual(changed, shadowed); + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{second}\" -d \"{first}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string reordered)); + Assert.NotEqual(shadowed, reordered); + + File.Delete(candidate); + File.Delete(earlierCandidate); + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{first}\" -d \"{second}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string removed)); + Assert.Equal(empty, removed); + } + + [Theory] + [InlineData("Dependency.pdb")] + [InlineData("Dependency.dll.mdb")] + [InlineData("Dependency.dll.config")] + [InlineData("fr/Dependency.resources.dll")] + public void CacheKeyTracksRuntimeSearchDirectorySidecars(string relativePath) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string directory = Path.Combine(test.Root, "search"); + Directory.CreateDirectory(directory); + WriteTestAssembly(Path.Combine(directory, "Dependency.dll")); + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{directory}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + + string sidecar = Path.Combine(directory, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(sidecar)); + File.WriteAllText(sidecar, "contents"); + Assert.True(test.Task.TryGetCacheKey(host, out string added)); + Assert.NotEqual(original, added); + File.AppendAllText(sidecar, "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(added, changed); + File.Delete(sidecar); + Assert.True(test.Task.TryGetCacheKey(host, out string removed)); + Assert.Equal(original, removed); + } + + [Theory] + [InlineData("missing-directory")] + [InlineData("invalid-candidate")] + [InlineData("metadata-less-candidate")] + public void CacheKeyBypassesUnmodeledSearchDirectoryInputs(string reason) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string directory = Path.Combine(test.Root, "search"); + if (reason != "missing-directory") + { + Directory.CreateDirectory(directory); + string candidate = Path.Combine(directory, "Dependency.dll"); + if (reason == "invalid-candidate") + File.WriteAllText(candidate, "invalid"); + else + WriteMetadataLessPE(candidate); + } + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{directory}\""; + + Assert.False(test.Task.TryGetCacheKey(host, out string key)); + Assert.Empty(key); + Assert.Contains(test.BuildEngine.Messages, message => + message.Message.StartsWith("ILLink cache bypassed: input identity could not be computed:")); + } + + [Fact] + public void CacheKeyAllowsRuntimeIgnoreLinkAttributesArgument() + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + test.Task.ExtraArgs = "--ignore-link-attributes true"; + + Assert.True(test.Task.TryGetCacheKey(host, out string key)); + Assert.Matches("^[0-9a-f]{64}$", key); + } + + [Theory] + [InlineData("--ignore-link-attributes false")] [InlineData("--link-attributes attributes.xml")] - [InlineData("--ignore-link-attributes true --substitutions substitutions.xml")] - [InlineData("--ignore-link-attributes true -d assemblies")] + [InlineData("--ignore-link-attributes true --help")] + [InlineData("--ignore-link-attributes true --link-attributes")] + [InlineData("--ignore-link-attributes true --link-attributes \"\"")] + [InlineData("--ignore-link-attributes true --link-attributes \"unterminated")] + [InlineData("--ignore-link-attributes true --substitutions")] + [InlineData("--ignore-link-attributes true -d")] + [InlineData("--ignore-link-attributes true -d \"\"")] + [InlineData("--ignore-link-attributes true -d \"unterminated")] public void CacheKeyBypassesUnsupportedExtraArgs(string extraArgs) { using var test = new OutputDirectoryTest(); @@ -1646,6 +1779,22 @@ public void CacheKeyBypassesUnsupportedExtraArgs(string extraArgs) message.Message.StartsWith("ILLink cache bypassed: extra arguments are not supported.")); } + [Theory] + [InlineData("--link-attributes")] + [InlineData("--substitutions")] + public void CacheKeyBypassesMissingRuntimeXmlFile(string option) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string missing = Path.Combine(test.Root, "missing link attributes.xml"); + test.Task.ExtraArgs = $"--ignore-link-attributes true {option} \"{missing}\""; + + Assert.False(test.Task.TryGetCacheKey(host, out string key)); + Assert.Empty(key); + Assert.Contains(test.BuildEngine.Messages, message => + message.Message.StartsWith("ILLink cache bypassed: input identity could not be computed:")); + } + [Theory] [InlineData(null)] [InlineData("")] From 2edf5de69f5756ce1ca1134e8552f2d572d7fbb5 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Thu, 1 Oct 2026 10:51:06 -0700 Subject: [PATCH 08/12] Enable ILLink caching and post-build purging in CI Reuse the existing compiler-cache job selection and purge only after all post-build work has finished, before saving the next cache snapshot. Assisted-by: Copilot:gpt-6-astra Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- docs/design/tools/illink/task-cache.md | 2 +- docs/tools/illink/illink-tasks.md | 13 +++++- eng/pipelines/common/global-build-job.yml | 4 ++ .../coreclr/templates/purge-illink-cache.yml | 44 +++++++++++++++++++ .../coreclr/templates/setup-illink-cache.yml | 27 ++++++++++++ .../coreclr/templates/setup-sccache.yml | 7 +++ 6 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 eng/pipelines/coreclr/templates/purge-illink-cache.yml create mode 100644 eng/pipelines/coreclr/templates/setup-illink-cache.yml diff --git a/docs/design/tools/illink/task-cache.md b/docs/design/tools/illink/task-cache.md index 55662b0ce75432..aba63753f62711 100644 --- a/docs/design/tools/illink/task-cache.md +++ b/docs/design/tools/illink/task-cache.md @@ -82,7 +82,7 @@ internal sealed class ILLinkCache - Purge requires an explicit cache root and UTC cutoff. Only published SHA-256 entry directories in `v1` are considered; delete entries whose last-used time is strictly older than the cutoff and retain equality. Leave staging directories and other layouts alone. Report per-entry maintenance errors and continue, then return nonzero if maintenance failed. - For a missing, malformed, non-UTC, or unreadable marker, report the fallback and use the entry directory's creation time instead. This may purge a recently used entry or retain a stale one if restoring a cache snapshot resets directory creation times. Do not migrate or repair the marker. - Purge must not run during any build using the cache. Do not add purge locks or change reader locking; coordination with active readers/writers remains a separate follow-up, as does protection against manual deletion. -- CI may record build start after restoring a job-private cache, run linking, then purge against that timestamp before uploading the next snapshot. Preserve tool diagnostics but do not fail the build for maintenance errors. Only purge once all users have exited; incremental skips and unreached work in failed builds do not refresh usage. See the [CI recipe](../../../tools/illink/illink-tasks.md#ci-purge-ordering). This does not enable an ILLink cache in CI automatically. +- CI jobs using `setup-sccache.yml` also enable ILLink caching under its existing Unix x64/arm64 platform gate. Restore a job-private Azure cache, record build start, run linking and all post-build steps, then purge against that timestamp before uploading the next snapshot. Build the maintenance tool from the checkout if it was not part of the main build. Skip maintenance when no versioned cache directory exists. Preserve tool diagnostics but do not fail the build for maintenance errors, and skip canceled jobs or jobs where ILLink caching was not enabled. Only purge once all users have exited; incremental skips and unreached work in failed builds do not refresh usage. See the [CI recipe and wiring](../../../tools/illink/illink-tasks.md#ci-purge-ordering). - Attempt best-effort cleanup of the current attempt's unpublished staging directory. Log cleanup I/O/access failures without failing a successful link; do not scan or delete other attempts' staging directories. Correctness must tolerate leftovers after crashes. ## Open follow-ups diff --git a/docs/tools/illink/illink-tasks.md b/docs/tools/illink/illink-tasks.md index c6f092e7fca48e..689c6e5f4c6eb5 100644 --- a/docs/tools/illink/illink-tasks.md +++ b/docs/tools/illink/illink-tasks.md @@ -136,8 +136,17 @@ work that was never reached will count as unused. Likewise, incremental builds t linking do not refresh cache usage. Skip purge if the cache may still be in use, including during cancellation. Do not share a live directory with another job while purging. -This is usage guidance, not automatic CI enablement. ILLink cache restore/upload wiring -is separate from runtime's Roslyn/csc cache maintenance. +Runtime's jobs that use `setup-sccache.yml` also enable ILLink caching under its existing +Unix x64/arm64 platform gate. `setup-illink-cache.yml` restores a branch-scoped Azure +Pipelines snapshot into `$(Pipeline.Workspace)/.illink-cache`, enables the experiment, and +records build start. `global-build-job.yml` runs `purge-illink-cache.yml` after all post-build +steps and before the `Cache@2` post-job save. The purge step builds the tool from the checkout +because library-only builds need not include the tool project. It reports missing +prerequisites and maintenance failures as warnings, and runs only when ILLink caching +was enabled and the job was not canceled. When no versioned cache directory exists, +as on a cold native-only build, it skips the maintenance tool build and purge. + +This configuration is separate from runtime's Roslyn/csc and native compiler caches. ## ILLink Task Properties diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index c29b03465a49c0..64c70b994b92b7 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -273,3 +273,7 @@ jobs: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log displayName: Collect vslogs on exit condition: always() + + # Some post-build steps compile additional tests. Purge only after those + # steps have completed, before Cache@2 saves its post-job snapshot. + - template: /eng/pipelines/coreclr/templates/purge-illink-cache.yml diff --git a/eng/pipelines/coreclr/templates/purge-illink-cache.yml b/eng/pipelines/coreclr/templates/purge-illink-cache.yml new file mode 100644 index 00000000000000..c6d4c9e6d1585a --- /dev/null +++ b/eng/pipelines/coreclr/templates/purge-illink-cache.yml @@ -0,0 +1,44 @@ +steps: + - script: | + set -euo pipefail + + if [[ -z "${ILLINK_EXPERIMENTAL_CACHE_PATH:-}" || + -z "${ILLINK_CACHE_BUILD_START:-}" ]]; then + echo "##vso[task.logissue type=warning]ILLink cache configuration or build-start timestamp is unavailable; skipping purge." + exit 0 + fi + + if [[ ! -e "$ILLINK_EXPERIMENTAL_CACHE_PATH/v1" ]]; then + echo "No ILLink cache entries were created; skipping purge." + exit 0 + fi + + dotnet="$(Build.SourcesDirectory)/.dotnet/dotnet" + if [[ ! -x "$dotnet" ]]; then + echo "##vso[task.logissue type=warning]The build SDK is unavailable; skipping ILLink cache purge." + exit 0 + fi + + # Library-only builds need not build the tools.illink subset. Build the + # maintenance tool from this checkout into a known host-tool directory. + toolDirectory="$(Build.SourcesDirectory)/artifacts/bin/ILLink.CacheTool/ci-purge" + if ! "$dotnet" build \ + "$(Build.SourcesDirectory)/src/tools/illink/src/ILLink.CacheTool/ILLink.CacheTool.csproj" \ + --configuration Release --output "$toolDirectory" \ + -m:1 -nodeReuse:false -p:BuildInParallel=false -p:UseSharedCompilation=false; then + echo "##vso[task.logissue type=warning]Could not build the ILLink cache maintenance tool; skipping purge." + exit 0 + fi + + if ! "$dotnet" "$toolDirectory/dotnet-illink-cache.dll" purge \ + --cache-directory "$ILLINK_EXPERIMENTAL_CACHE_PATH" \ + --before "$ILLINK_CACHE_BUILD_START"; then + echo "##vso[task.logissue type=warning]ILLink cache purge failed; see the tool diagnostics above." + fi + displayName: Purge unused ILLink cache entries + condition: and(succeededOrFailed(), eq(variables['ILLINK_EXPERIMENTAL_CACHE'], 'true')) + continueOnError: true + env: + DOTNET_PROCESSOR_COUNT: '2' + DOTNET_CLI_USE_MSBUILD_SERVER: '0' + MSBUILDDISABLENODEREUSE: '1' diff --git a/eng/pipelines/coreclr/templates/setup-illink-cache.yml b/eng/pipelines/coreclr/templates/setup-illink-cache.yml new file mode 100644 index 00000000000000..250c7627fbd41a --- /dev/null +++ b/eng/pipelines/coreclr/templates/setup-illink-cache.yml @@ -0,0 +1,27 @@ +parameters: + archType: 'x64' + osGroup: 'linux' + nameSuffix: '' + osSubgroup: '' + +steps: + # Use the same branch-scoped restore policy as the compiler caches. Each job + # works on its own restored copy, so post-build purge cannot affect another job. + - task: Cache@2 + displayName: ILLink task cache + continueOnError: true + inputs: + key: illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} | "$(Build.BuildId)" + path: $(Pipeline.Workspace)/.illink-cache + restoreKeys: | + illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} + + - script: | + set -euo pipefail + mkdir -p "$(Pipeline.Workspace)/.illink-cache" + buildStart="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + echo "##vso[task.setvariable variable=ILLINK_EXPERIMENTAL_CACHE_PATH]$(Pipeline.Workspace)/.illink-cache" + echo "##vso[task.setvariable variable=ILLINK_CACHE_BUILD_START]$buildStart" + echo "##vso[task.setvariable variable=ILLINK_EXPERIMENTAL_CACHE]true" + displayName: Configure experimental ILLink task cache + continueOnError: true diff --git a/eng/pipelines/coreclr/templates/setup-sccache.yml b/eng/pipelines/coreclr/templates/setup-sccache.yml index cf703d5da6501b..ebb2f64f98e882 100644 --- a/eng/pipelines/coreclr/templates/setup-sccache.yml +++ b/eng/pipelines/coreclr/templates/setup-sccache.yml @@ -24,6 +24,13 @@ steps: archType: ${{ parameters.archType }} nameSuffix: ${{ parameters.nameSuffix }} + - template: /eng/pipelines/coreclr/templates/setup-illink-cache.yml + parameters: + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + archType: ${{ parameters.archType }} + nameSuffix: ${{ parameters.nameSuffix }} + # Set up the Azure Pipeline Cache for sccache's local cache directory. # Use a rolling key so each build can update the cache; restoreKeys # falls back to the most recent saved entry. From 118290360b8ab13376315144165c8eabc2d12dee Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Thu, 1 Oct 2026 10:49:54 -0700 Subject: [PATCH 09/12] Support ILLink caching in runtime library builds Use structured task parameters for runtime trimming and model the narrow extra-argument shapes needed by shared-framework, out-of-band, and per-library builds. Preserve custom argument precedence. Assisted-by: Copilot:gpt-6-astra Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- docs/design/tools/illink/task-cache.md | 2 +- docs/tools/illink/illink-tasks.md | 8 +- eng/illink.targets | 52 ++++-- src/libraries/oob-trim.proj | 5 +- src/libraries/sfx-finish.proj | 3 +- src/tools/illink/src/ILLink.Tasks/LinkTask.cs | 103 +++++++++++- .../ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs | 157 +++++++++++++++++- 7 files changed, 301 insertions(+), 29 deletions(-) diff --git a/docs/design/tools/illink/task-cache.md b/docs/design/tools/illink/task-cache.md index 21ca8a50e85aa5..55662b0ce75432 100644 --- a/docs/design/tools/illink/task-cache.md +++ b/docs/design/tools/illink/task-cache.md @@ -62,7 +62,7 @@ internal sealed class ILLinkCache - Defer identifying the runtime that executes ILLink; do not include a runtime version or framework description in the key. The task runs in MSBuild, so its CoreLib informational version and framework description are not reliable identities for the child linker's runtime. The runtimeconfig describes requested frameworks and roll-forward policy, not the exact resolved CoreLib. Finding that CoreLib would require resolved-runtime information from the caller, host-resolution plumbing, or an additional process; defer that complexity and startup cost. - This deliberately accepts that runtime installation changes not represented by other keyed inputs may reuse an entry; callers needing invalidation for those changes must clear or isolate the cache. - Include platform, architecture, and culture. Do not enumerate or hash inherited environment variables, or bypass caching based on them. This deliberately keeps v1 simple rather than maintaining a partial list of runtime/loader settings. Environment changes are reflected only when they change other keyed inputs, such as the selected host path or culture. Callers must disable caching when inherited settings introduce otherwise untracked dependencies, affect outputs, or require tool-execution side effects (for example, startup hooks or profilers). -- Bypass caching for non-whitespace `ExtraArgs`, custom steps/data, dependency-dump options, and explicit `ToolTask.EnvironmentVariables` overrides. These are caller-supplied inputs that v1 does not model in its key and can introduce undeclared dependencies, external outputs, or side effects. These invocations still run the linker normally; no extra-argument parsing or runtime-specific exceptions are supported. +- Bypass caching for arbitrary `ExtraArgs`, custom steps/data, dependency-dump options, and explicit `ToolTask.EnvironmentVariables` overrides. These are caller-supplied inputs that v1 does not model in its key and can introduce undeclared dependencies, external outputs, or side effects. The supported `ExtraArgs` shape for runtime's library builds is `--ignore-link-attributes true` followed by zero or more `--link-attributes FILE`, `--substitutions FILE`, or `-d DIRECTORY` pairs. Hash every referenced XML file and every top-level `.dll`, `.exe`, and `.winmd` candidate in each search directory, including assembly sidecars/modules/resources. Preserve directory order in the arguments and leave linker resolution unchanged; adding, changing, or removing candidates invalidates the key, including shadowed candidates. An unreadable directory or uninspectable candidate bypasses caching. All other SDK options passed through `ExtraArgs` remain subject to the bypass. - Log and bypass caching if required files cannot be read or an input assembly cannot be inspected, including PE images without managed metadata. Normal linking determines whether those inputs are acceptable. Input files must remain stable during hashing and linking. ## Entry layout diff --git a/docs/tools/illink/illink-tasks.md b/docs/tools/illink/illink-tasks.md index 209a3f8b7a631b..c6f092e7fca48e 100644 --- a/docs/tools/illink/illink-tasks.md +++ b/docs/tools/illink/illink-tasks.md @@ -67,9 +67,11 @@ clear or isolate the cache when that distinction is required. Cache hits replace the output directory without running ILLink. Directories are created in the cache only when storing a successful result. Hits do not replay warnings or other linker diagnostics. -Caching is bypassed with a diagnostic for non-whitespace `ExtraArgs`, custom steps/data, -dependency-dump options, and explicit task environment overrides. Options supplied through -`ExtraArgs` remain unsupported for caching; these invocations still run the linker normally. +Caching is bypassed with a diagnostic for arbitrary `ExtraArgs`, custom steps/data, +dependency-dump options, and explicit task environment overrides. Runtime's library-build +arguments for link attributes, substitutions, and assembly search directories are supported. +Their XML files and search-directory assembly candidates (including sidecars) are content-hashed, +without changing assembly resolution. Other SDK options supplied through `ExtraArgs` remain unsupported. Inherited environment variables are not tracked; disable caching when they affect outputs or dependencies beyond the keyed inputs, or require tool-execution side effects such as startup hooks or profiling. Unreadable inputs or linker files also diff --git a/eng/illink.targets b/eng/illink.targets index 248e5f6a34e50a..714ac859810554 100644 --- a/eng/illink.targets +++ b/eng/illink.targets @@ -178,10 +178,9 @@ + <_ILLinkHasCustomArgs Condition="'$(ILLinkArgs)' != ''">true $(ILLinkArgs) --ignore-link-attributes true - - $(ILLinkArgs) --skip-unresolved true + $(LinkerNoWarn);IL2008;IL2009;IL2121;IL2025;IL2035 + + + + $(ILLinkArgs) --trim-mode skip @@ -215,18 +226,30 @@ $(ILLinkArgs) -x "@(ILLinkDescriptorsLibraryBuildXml->'%(FullPath)', '" -x "')" $(ILLinkArgs) --substitutions "$(ILLinkSubstitutionsLibraryBuildXml)" $(ILLinkArgs) --link-attributes "@(ILLinkSuppressionsLibraryBuildXml->'%(FullPath)', '" --link-attributes "')" - - $(LinkerNoWarn);IL2008;IL2009;IL2121;IL2025;IL2035 $(ILLinkArgs) --nowarn $(LinkerNoWarn) $(ILLinkArgs) --disable-opt ipconstprop + + <_ILLinkTrimMode>skip + <_ILLinkDefaultAction>skip + <_ILLinkNoWarn>$(LinkerNoWarn) + <_ILLinkRemoveSymbols>true + <_ILLinkRemoveSymbols Condition="'$(ILLinkRewritePDBs)' == 'true' and Exists('$(ILLinkTrimAssemblySymbols)')">false + <_ILLinkPreserveSymbolPaths Condition="'$(_ILLinkRemoveSymbols)' == 'false' and '$(DeterministicSourcePaths)' == 'true'">true + <_ILLinkIPConstProp Condition="'$(ILLinkDisableIPConstProp)' == 'true'">false + $(ILLinkArgs) --substitutions "$(ILLinkSubstitutionsLibraryBuildXml)" + $(ILLinkArgs) --link-attributes "@(ILLinkSuppressionsLibraryBuildXml->'%(FullPath)', '" --link-attributes "')" + + + + <_ILLinkAssemblyToTrim Include="$(ILLinkTrimInputPath)$(TargetName)$(TargetExt)"> + link + + <_ILLinkRootDescriptors Include="@(ILLinkDescriptorsLibraryBuildXml->'%(FullPath)')" + Condition="'$(_ILLinkHasCustomArgs)' != 'true'" /> + + @@ -264,8 +287,15 @@ $(ILLinkArgs) -d @(_DependencyDirectories->'"%(Identity)"', ' -d ') - $(NetCoreAppCurrent)-$(TargetOS) true + + IL2121 @@ -37,8 +39,6 @@ $(ILLinkArgs) - - $(ILLinkArgs) --nowarn IL2121 $(OOBILLinkArgs) --link-attributes "@(OOBLibrarySuppressionsXml->'%(FullPath)', '" --link-attributes "')" @@ -52,6 +52,7 @@ ReferenceAssemblyPaths="@(OOBAssemblyReference)" OutputDirectory="$(OOBAssembliesTrimDir)" ExtraArgs="$(OOBILLinkArgs)" + NoWarn="$(OOBILLinkNoWarn)" ToolExe="$(_DotNetHostFileName)" ToolPath="$(_DotNetHostDirectory)" /> diff --git a/src/libraries/sfx-finish.proj b/src/libraries/sfx-finish.proj index 47e300d3adc289..a0af3730354192 100644 --- a/src/libraries/sfx-finish.proj +++ b/src/libraries/sfx-finish.proj @@ -124,8 +124,6 @@ $(ILLinkArgs) - - $(SharedFrameworkILLinkArgs) -b true $(SharedFrameworkILLinkArgs) --link-attributes "@(SharedFrameworkSuppressionsXml, '" --link-attributes "')" @@ -141,6 +139,7 @@ RootAssemblyNames="@(_SharedFrameworkRootAssemblyName)" OutputDirectory="$(SharedFrameworkAssembliesTrimDir)" ExtraArgs="$(SharedFrameworkILLinkArgs)" + RemoveSymbols="false" ToolExe="$(_DotNetHostFileName)" ToolPath="$(_DotNetHostDirectory)" /> diff --git a/src/tools/illink/src/ILLink.Tasks/LinkTask.cs b/src/tools/illink/src/ILLink.Tasks/LinkTask.cs index 900b945591367b..0f43e75a857db0 100644 --- a/src/tools/illink/src/ILLink.Tasks/LinkTask.cs +++ b/src/tools/illink/src/ILLink.Tasks/LinkTask.cs @@ -310,15 +310,16 @@ private bool TryComputeCacheKey(string pathToTool, string commandLineCommands, s return false; } - if (!string.IsNullOrWhiteSpace(ExtraArgs)) - { - Log.LogMessage(MessageImportance.Low, "ILLink cache bypassed: extra arguments are not supported."); - return false; - } - try { var files = new SortedSet(StringComparer.Ordinal); + var directories = new SortedSet(StringComparer.Ordinal); + if (!TryAddCacheableExtraArgsInputs(files, directories)) + { + Log.LogMessage(MessageImportance.Low, "ILLink cache bypassed: extra arguments are not supported."); + return false; + } + var assemblies = new HashSet(StringComparer.Ordinal); foreach (ITaskItem assembly in AssemblyPaths) AddAssembly(assembly.ItemSpec); @@ -327,6 +328,22 @@ private bool TryComputeCacheKey(string pathToTool, string commandLineCommands, s foreach (ITaskItem descriptor in RootDescriptorFiles ?? Array.Empty()) AddFile(descriptor.ItemSpec); + // Preserve AssemblyResolver's search behavior instead of replacing -d with + // explicit references. Include every candidate so additions and shadowing invalidate. + foreach (string directory in directories) + { + foreach (string file in Directory.EnumerateFiles(directory)) + { + string extension = Path.GetExtension(file); + if (extension.Equals(".dll", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".exe", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".winmd", StringComparison.OrdinalIgnoreCase)) + { + AddAssembly(file); + } + } + } + #pragma warning disable IL3000 // MSBuild tasks are loaded from assemblies on disk. AddFile(typeof(ILLink).Assembly.Location); #pragma warning restore IL3000 @@ -408,6 +425,80 @@ void AddAssembly(string path) } } + private bool TryAddCacheableExtraArgsInputs(SortedSet files, SortedSet directories) + { + ReadOnlySpan arguments = ExtraArgs.AsSpan().Trim(); + if (arguments.IsEmpty) + return true; + + // The runtime library builds pass this policy through ExtraArgs. Keep the + // supported shape narrow so arbitrary linker arguments continue to bypass caching. + if (!TryReadArgument(ref arguments, out string option) || + option != "--ignore-link-attributes" || + !TryReadArgument(ref arguments, out string value) || + value != "true") + { + return false; + } + + while (!arguments.TrimStart().IsEmpty) + { + if (!TryReadArgument(ref arguments, out option) || + option is not ("--link-attributes" or "--substitutions" or "-d") || + !TryReadArgument(ref arguments, out string path) || + string.IsNullOrEmpty(path)) + { + return false; + } + + if (option == "-d") + directories.Add(Path.GetFullPath(path)); + else + files.Add(Path.GetFullPath(path)); + } + + return true; + + static bool TryReadArgument(ref ReadOnlySpan arguments, out string argument) + { + arguments = arguments.TrimStart(); + if (arguments.IsEmpty) + { + argument = string.Empty; + return false; + } + + if (arguments[0] == '"') + { + int closingQuote = arguments.Slice(1).IndexOf('"'); + if (closingQuote < 0) + { + argument = string.Empty; + return false; + } + + argument = arguments.Slice(1, closingQuote).ToString(); + arguments = arguments.Slice(closingQuote + 2); + return arguments.IsEmpty || char.IsWhiteSpace(arguments[0]); + } + + int end = 0; + while (end < arguments.Length && !char.IsWhiteSpace(arguments[end])) + end++; + + ReadOnlySpan token = arguments.Slice(0, end); + if (token.IndexOf('"') >= 0) + { + argument = string.Empty; + return false; + } + + argument = token.ToString(); + arguments = arguments.Slice(end); + return true; + } + } + private string _illinkPath = ""; public string ILLinkPath diff --git a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs index 90cc266ac8eac1..84135d1a807f87 100644 --- a/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs +++ b/src/tools/illink/test/ILLink.Tasks.Tests/ILLink.Tasks.Tests.cs @@ -1629,11 +1629,144 @@ public void CacheKeyTracksFileContentsNotTimestamps(string relativePath) } [Theory] - [InlineData("--help")] - [InlineData("--ignore-link-attributes true")] + [InlineData("--link-attributes", 0)] + [InlineData("--link-attributes", 1)] + [InlineData("--substitutions", 0)] + [InlineData("--substitutions", 1)] + public void CacheKeyTracksRuntimeXmlFiles(string option, int changedFile) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string[] linkAttributesFiles = + { + Path.Combine(test.Root, "first link attributes.xml"), + Path.Combine(test.Root, "second link attributes.xml") + }; + foreach (string file in linkAttributesFiles) + File.WriteAllText(file, "contents"); + + test.Task.ExtraArgs = $" --ignore-link-attributes true {option} \"{linkAttributesFiles[0]}\" {option} \"{linkAttributesFiles[1]}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + + File.AppendAllText(linkAttributesFiles[changedFile], "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(original, changed); + } + + [Theory] + [InlineData(".dll")] + [InlineData(".exe")] + [InlineData(".winmd")] + public void CacheKeyTracksRuntimeSearchDirectoryCandidates(string extension) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string first = Path.Combine(test.Root, "first search directory"); + string second = Path.Combine(test.Root, "second search directory"); + Directory.CreateDirectory(first); + Directory.CreateDirectory(second); + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{first}\" -d \"{second}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string empty)); + + string candidate = Path.Combine(second, "Dependency" + extension); + WriteTestAssembly(candidate); + Assert.True(test.Task.TryGetCacheKey(host, out string added)); + Assert.NotEqual(empty, added); + WriteTestAssembly(candidate, "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(added, changed); + + string earlierCandidate = Path.Combine(first, Path.GetFileName(candidate)); + File.Copy(candidate, earlierCandidate); + Assert.True(test.Task.TryGetCacheKey(host, out string shadowed)); + Assert.NotEqual(changed, shadowed); + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{second}\" -d \"{first}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string reordered)); + Assert.NotEqual(shadowed, reordered); + + File.Delete(candidate); + File.Delete(earlierCandidate); + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{first}\" -d \"{second}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string removed)); + Assert.Equal(empty, removed); + } + + [Theory] + [InlineData("Dependency.pdb")] + [InlineData("Dependency.dll.mdb")] + [InlineData("Dependency.dll.config")] + [InlineData("fr/Dependency.resources.dll")] + public void CacheKeyTracksRuntimeSearchDirectorySidecars(string relativePath) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string directory = Path.Combine(test.Root, "search"); + Directory.CreateDirectory(directory); + WriteTestAssembly(Path.Combine(directory, "Dependency.dll")); + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{directory}\""; + Assert.True(test.Task.TryGetCacheKey(host, out string original)); + + string sidecar = Path.Combine(directory, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(sidecar)); + File.WriteAllText(sidecar, "contents"); + Assert.True(test.Task.TryGetCacheKey(host, out string added)); + Assert.NotEqual(original, added); + File.AppendAllText(sidecar, "changed"); + Assert.True(test.Task.TryGetCacheKey(host, out string changed)); + Assert.NotEqual(added, changed); + File.Delete(sidecar); + Assert.True(test.Task.TryGetCacheKey(host, out string removed)); + Assert.Equal(original, removed); + } + + [Theory] + [InlineData("missing-directory")] + [InlineData("invalid-candidate")] + [InlineData("metadata-less-candidate")] + public void CacheKeyBypassesUnmodeledSearchDirectoryInputs(string reason) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string directory = Path.Combine(test.Root, "search"); + if (reason != "missing-directory") + { + Directory.CreateDirectory(directory); + string candidate = Path.Combine(directory, "Dependency.dll"); + if (reason == "invalid-candidate") + File.WriteAllText(candidate, "invalid"); + else + WriteMetadataLessPE(candidate); + } + test.Task.ExtraArgs = $"--ignore-link-attributes true -d \"{directory}\""; + + Assert.False(test.Task.TryGetCacheKey(host, out string key)); + Assert.Empty(key); + Assert.Contains(test.BuildEngine.Messages, message => + message.Message.StartsWith("ILLink cache bypassed: input identity could not be computed:")); + } + + [Fact] + public void CacheKeyAllowsRuntimeIgnoreLinkAttributesArgument() + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + test.Task.ExtraArgs = "--ignore-link-attributes true"; + + Assert.True(test.Task.TryGetCacheKey(host, out string key)); + Assert.Matches("^[0-9a-f]{64}$", key); + } + + [Theory] + [InlineData("--ignore-link-attributes false")] [InlineData("--link-attributes attributes.xml")] - [InlineData("--ignore-link-attributes true --substitutions substitutions.xml")] - [InlineData("--ignore-link-attributes true -d assemblies")] + [InlineData("--ignore-link-attributes true --help")] + [InlineData("--ignore-link-attributes true --link-attributes")] + [InlineData("--ignore-link-attributes true --link-attributes \"\"")] + [InlineData("--ignore-link-attributes true --link-attributes \"unterminated")] + [InlineData("--ignore-link-attributes true --substitutions")] + [InlineData("--ignore-link-attributes true -d")] + [InlineData("--ignore-link-attributes true -d \"\"")] + [InlineData("--ignore-link-attributes true -d \"unterminated")] public void CacheKeyBypassesUnsupportedExtraArgs(string extraArgs) { using var test = new OutputDirectoryTest(); @@ -1646,6 +1779,22 @@ public void CacheKeyBypassesUnsupportedExtraArgs(string extraArgs) message.Message.StartsWith("ILLink cache bypassed: extra arguments are not supported.")); } + [Theory] + [InlineData("--link-attributes")] + [InlineData("--substitutions")] + public void CacheKeyBypassesMissingRuntimeXmlFile(string option) + { + using var test = new OutputDirectoryTest(); + string host = PrepareCacheKeyTest(test); + string missing = Path.Combine(test.Root, "missing link attributes.xml"); + test.Task.ExtraArgs = $"--ignore-link-attributes true {option} \"{missing}\""; + + Assert.False(test.Task.TryGetCacheKey(host, out string key)); + Assert.Empty(key); + Assert.Contains(test.BuildEngine.Messages, message => + message.Message.StartsWith("ILLink cache bypassed: input identity could not be computed:")); + } + [Theory] [InlineData(null)] [InlineData("")] From 645e0a7eb9bf0673ecde655c5d0ac78b20f9dc6f Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Thu, 15 Oct 2026 12:01:41 -0700 Subject: [PATCH 10/12] Separate Debug and Release ILLink CI cache snapshots Prevent rolling Debug and Release legs from sharing an immutable Azure cache key by including build configuration in both save and restore keys. Assisted-by: Copilot:gpt-6-sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- eng/pipelines/coreclr/templates/setup-illink-cache.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/setup-illink-cache.yml b/eng/pipelines/coreclr/templates/setup-illink-cache.yml index 250c7627fbd41a..7d8f094cd02c8e 100644 --- a/eng/pipelines/coreclr/templates/setup-illink-cache.yml +++ b/eng/pipelines/coreclr/templates/setup-illink-cache.yml @@ -7,14 +7,15 @@ parameters: steps: # Use the same branch-scoped restore policy as the compiler caches. Each job # works on its own restored copy, so post-build purge cannot affect another job. + # Keep Debug and Release snapshots separate in rolling builds. - task: Cache@2 displayName: ILLink task cache continueOnError: true inputs: - key: illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} | "$(Build.BuildId)" + key: illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} | "$(buildConfigUpper)" | "$(Build.BuildId)" path: $(Pipeline.Workspace)/.illink-cache restoreKeys: | - illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} + illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} | "$(buildConfigUpper)" - script: | set -euo pipefail From 66a06f194139bfef7b1faf32a4ed57d2c158df97 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Thu, 15 Oct 2026 16:40:00 -0700 Subject: [PATCH 11/12] Scope ILLink CI cache to linker build and reuse purge tool Restore the ILLink cache in the analyzer/linker job alongside Roslyn and sccache, including Windows. Build the maintenance tool through the explicit tools.illink subset and purge only after successful jobs without an end-of-job restore. Assisted-by: Copilot:gpt-6-sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- docs/design/tools/illink/task-cache.md | 2 +- docs/tools/illink/illink-tasks.md | 34 ++--- eng/pipelines/common/global-build-job.yml | 2 + .../coreclr/templates/purge-illink-cache.yml | 130 ++++++++++++------ .../coreclr/templates/setup-build-caches.yml | 6 +- .../coreclr/templates/setup-illink-cache.yml | 54 +++++--- eng/pipelines/global-build.yml | 9 +- 7 files changed, 154 insertions(+), 83 deletions(-) diff --git a/docs/design/tools/illink/task-cache.md b/docs/design/tools/illink/task-cache.md index 0ca0e8fa95bf84..54c0a8c59e2fa9 100644 --- a/docs/design/tools/illink/task-cache.md +++ b/docs/design/tools/illink/task-cache.md @@ -82,7 +82,7 @@ internal sealed class ILLinkCache - Purge requires an explicit cache root and UTC cutoff. Only published SHA-256 entry directories in `v1` are considered; delete entries whose last-used time is strictly older than the cutoff and retain equality. Leave staging directories and other layouts alone. Report per-entry maintenance errors and continue, then return nonzero if maintenance failed. - For a missing, malformed, non-UTC, or unreadable marker, report the fallback and use the entry directory's creation time instead. This may purge a recently used entry or retain a stale one if restoring a cache snapshot resets directory creation times. Do not migrate or repair the marker. - Purge must not run during any build using the cache. Do not add purge locks or change reader locking; coordination with active readers/writers remains a separate follow-up, as does protection against manual deletion. -- CI jobs using `setup-build-caches.yml` also enable ILLink caching under its Unix x64/arm64 platform gate. Restore a job-private Azure cache, record build start, run linking and all post-build steps, then purge against that timestamp before uploading the next snapshot. Build the maintenance tool from the checkout if it was not part of the main build. Skip maintenance when no versioned cache directory exists. Preserve tool diagnostics but do not fail the build for maintenance errors, and skip canceled jobs or jobs where ILLink caching was not enabled. Only purge once all users have exited; incremental skips and unreached work in failed builds do not refresh usage. See the [CI recipe and wiring](../../../tools/illink/illink-tasks.md#ci-purge-ordering). +- Only the `CoreLib_Libraries_AnalyzersLinker` CI job opts into ILLink caching through `setup-build-caches.yml`, on Windows and Unix x64/arm64. Roslyn and sccache remain gated by platform. Restore a job-private Azure cache, record build start, run linking and all post-build steps, then purge against that timestamp before uploading the next snapshot. This job explicitly enables `RunILLinkInBuild` and builds the `tools.illink` subset for the maintenance tool; building `ILLink.Tasks` through a library project reference does not build it. Purge only after a successful build; skip maintenance when no versioned cache directory exists and report a missing tool. Preserve tool diagnostics but do not fail the build for maintenance errors. Only purge once all users have exited; incremental skips and unreached work in failed builds do not refresh usage. See the [CI recipe and wiring](../../../tools/illink/illink-tasks.md#ci-purge-ordering). - Attempt best-effort cleanup of the current attempt's unpublished staging directory. Log cleanup I/O/access failures without failing a successful link; do not scan or delete other attempts' staging directories. Correctness must tolerate leftovers after crashes. ## Open follow-ups diff --git a/docs/tools/illink/illink-tasks.md b/docs/tools/illink/illink-tasks.md index c95f37c16b450f..7e11f70f63b0f0 100644 --- a/docs/tools/illink/illink-tasks.md +++ b/docs/tools/illink/illink-tasks.md @@ -115,8 +115,8 @@ or created by that build: ```sh export ILLINK_CACHE_BUILD_START="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" ``` -3. Run the build and wait for all processes using the cache to finish. -4. Run the tool from the same revision, with maintenance failures reported but nonfatal: +3. Run the build successfully and wait for all processes using the cache to finish. +4. Run the maintenance tool from the same checkout, with maintenance failures reported but nonfatal: ```sh if ! dotnet illink-cache purge \ --cache-directory "$ILLINK_EXPERIMENTAL_CACHE_PATH" \ @@ -131,20 +131,22 @@ If the build and purge run in different CI steps, persist `ILLINK_CACHE_BUILD_ST pipeline variable; a shell export alone does not carry it across steps. This variable is only for the CI recipe, not another task configuration setting. -A failed build can still be followed by purge once all linking has stopped, but entries for -work that was never reached will count as unused. Likewise, incremental builds that skip -linking do not refresh cache usage. Skip purge if the cache may still be in use, including -during cancellation. Do not share a live directory with another job while purging. - -Runtime's jobs that use `setup-build-caches.yml` also enable ILLink caching under its -Unix x64/arm64 platform gate. `setup-illink-cache.yml` restores a branch-scoped Azure -Pipelines snapshot into `$(Pipeline.Workspace)/.illink-cache`, enables the experiment, and -records build start. `global-build-job.yml` runs `purge-illink-cache.yml` after all post-build -steps and before the `Cache@2` post-job save. The purge step builds the tool from the checkout -because library-only builds need not include the tool project. It reports missing -prerequisites and maintenance failures as warnings, and runs only when ILLink caching -was enabled and the job was not canceled. When no versioned cache directory exists, -as on a cold native-only build, it skips the maintenance tool build and purge. +Skip purge after a failed or canceled build. Incremental builds that skip linking do not +refresh cache usage. Do not share a live directory with another job while purging. + +The `CoreLib_Libraries_AnalyzersLinker` job in `global-build.yml` opts into ILLink caching +through `setup-build-caches.yml` on Windows and Unix x64/arm64 and sets `RunILLinkInBuild=true`. +The shared setup also configures Roslyn caching on Windows and Unix x64/arm64, and sccache +on Unix x64/arm64; other jobs retain those platform gates without restoring the ILLink cache. +`setup-illink-cache.yml` restores a branch-scoped Azure +Pipelines snapshot into `$(Pipeline.Workspace)/.illink-cache`, enables the experiment, +and records build start. `global-build-job.yml` runs `purge-illink-cache.yml` after all +post-build steps and before the `Cache@2` post-job save. The linker job explicitly +builds the `tools.illink` subset, which includes the purge tool; building `ILLink.Tasks` +through a library project reference alone does not build it. The purge step reports +missing prerequisites and maintenance failures as warnings, and runs only when ILLink +caching was enabled and the job succeeded. When no versioned cache directory exists, +it skips purge. This configuration is separate from runtime's Roslyn/csc and native compiler caches. diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 64c70b994b92b7..2242691041c9a4 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -277,3 +277,5 @@ jobs: # Some post-build steps compile additional tests. Purge only after those # steps have completed, before Cache@2 saves its post-job snapshot. - template: /eng/pipelines/coreclr/templates/purge-illink-cache.yml + parameters: + osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/coreclr/templates/purge-illink-cache.yml b/eng/pipelines/coreclr/templates/purge-illink-cache.yml index c6d4c9e6d1585a..b2a566635a0599 100644 --- a/eng/pipelines/coreclr/templates/purge-illink-cache.yml +++ b/eng/pipelines/coreclr/templates/purge-illink-cache.yml @@ -1,44 +1,88 @@ +parameters: + osGroup: 'linux' + steps: - - script: | - set -euo pipefail - - if [[ -z "${ILLINK_EXPERIMENTAL_CACHE_PATH:-}" || - -z "${ILLINK_CACHE_BUILD_START:-}" ]]; then - echo "##vso[task.logissue type=warning]ILLink cache configuration or build-start timestamp is unavailable; skipping purge." - exit 0 - fi - - if [[ ! -e "$ILLINK_EXPERIMENTAL_CACHE_PATH/v1" ]]; then - echo "No ILLink cache entries were created; skipping purge." - exit 0 - fi - - dotnet="$(Build.SourcesDirectory)/.dotnet/dotnet" - if [[ ! -x "$dotnet" ]]; then - echo "##vso[task.logissue type=warning]The build SDK is unavailable; skipping ILLink cache purge." - exit 0 - fi - - # Library-only builds need not build the tools.illink subset. Build the - # maintenance tool from this checkout into a known host-tool directory. - toolDirectory="$(Build.SourcesDirectory)/artifacts/bin/ILLink.CacheTool/ci-purge" - if ! "$dotnet" build \ - "$(Build.SourcesDirectory)/src/tools/illink/src/ILLink.CacheTool/ILLink.CacheTool.csproj" \ - --configuration Release --output "$toolDirectory" \ - -m:1 -nodeReuse:false -p:BuildInParallel=false -p:UseSharedCompilation=false; then - echo "##vso[task.logissue type=warning]Could not build the ILLink cache maintenance tool; skipping purge." - exit 0 - fi - - if ! "$dotnet" "$toolDirectory/dotnet-illink-cache.dll" purge \ - --cache-directory "$ILLINK_EXPERIMENTAL_CACHE_PATH" \ - --before "$ILLINK_CACHE_BUILD_START"; then - echo "##vso[task.logissue type=warning]ILLink cache purge failed; see the tool diagnostics above." - fi - displayName: Purge unused ILLink cache entries - condition: and(succeededOrFailed(), eq(variables['ILLINK_EXPERIMENTAL_CACHE'], 'true')) - continueOnError: true - env: - DOTNET_PROCESSOR_COUNT: '2' - DOTNET_CLI_USE_MSBUILD_SERVER: '0' - MSBUILDDISABLENODEREUSE: '1' + - ${{ if ne(parameters.osGroup, 'windows') }}: + - script: | + set -euo pipefail + + if [[ -z "${ILLINK_EXPERIMENTAL_CACHE_PATH:-}" || + -z "${ILLINK_CACHE_BUILD_START:-}" ]]; then + echo "##vso[task.logissue type=warning]ILLink cache configuration or build-start timestamp is unavailable; skipping purge." + exit 0 + fi + + if [[ ! -e "$ILLINK_EXPERIMENTAL_CACHE_PATH/v1" ]]; then + echo "No ILLink cache entries were created; skipping purge." + exit 0 + fi + + dotnet="$(Build.SourcesDirectory)/.dotnet/dotnet" + if [[ ! -x "$dotnet" ]]; then + echo "##vso[task.logissue type=warning]The build SDK is unavailable; skipping ILLink cache purge." + exit 0 + fi + + tool="" + for candidate in "$(Build.SourcesDirectory)"/artifacts/bin/ILLink.CacheTool/*/net*/dotnet-illink-cache.dll; do + if [[ -f "$candidate" ]]; then + tool="$candidate" + break + fi + done + + if [[ -z "$tool" ]]; then + echo "##vso[task.logissue type=warning]The ILLink cache maintenance tool was not built; skipping purge." + exit 1 + fi + + if ! "$dotnet" "$tool" purge \ + --cache-directory "$ILLINK_EXPERIMENTAL_CACHE_PATH" \ + --before "$ILLINK_CACHE_BUILD_START"; then + echo "##vso[task.logissue type=warning]ILLink cache purge failed; see the tool diagnostics above." + fi + displayName: Purge unused ILLink cache entries + condition: and(succeeded(), eq(variables['ILLINK_EXPERIMENTAL_CACHE'], 'true')) + continueOnError: true + + - ${{ if eq(parameters.osGroup, 'windows') }}: + - pwsh: | + if ([string]::IsNullOrEmpty($env:ILLINK_EXPERIMENTAL_CACHE_PATH) -or + [string]::IsNullOrEmpty($env:ILLINK_CACHE_BUILD_START)) { + Write-Host "##vso[task.logissue type=warning]ILLink cache configuration or build-start timestamp is unavailable; skipping purge." + exit 0 + } + + if (-not (Test-Path -LiteralPath (Join-Path $env:ILLINK_EXPERIMENTAL_CACHE_PATH "v1") -PathType Container)) { + Write-Host "No ILLink cache entries were created; skipping purge." + exit 0 + } + + $dotnet = Join-Path "$(Build.SourcesDirectory)" ".dotnet/dotnet.exe" + if (-not (Test-Path -LiteralPath $dotnet -PathType Leaf)) { + Write-Host "##vso[task.logissue type=warning]The build SDK is unavailable; skipping ILLink cache purge." + exit 0 + } + + $toolRoot = Join-Path "$(Build.SourcesDirectory)" "artifacts/bin/ILLink.CacheTool" + if (-not (Test-Path -LiteralPath $toolRoot -PathType Container)) { + Write-Host "##vso[task.logissue type=warning]The ILLink cache maintenance tool was not built; skipping purge." + exit 1 + } + + $tool = Get-ChildItem -LiteralPath $toolRoot -Filter "dotnet-illink-cache.dll" -File -Recurse | + Sort-Object FullName | Select-Object -First 1 + if ($null -eq $tool) { + Write-Host "##vso[task.logissue type=warning]The ILLink cache maintenance tool was not built; skipping purge." + exit 1 + } + + & $dotnet $tool.FullName purge ` + --cache-directory $env:ILLINK_EXPERIMENTAL_CACHE_PATH ` + --before $env:ILLINK_CACHE_BUILD_START + if ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=warning]ILLink cache purge failed; see the tool diagnostics above." + } + displayName: Purge unused ILLink cache entries + condition: and(succeeded(), eq(variables['ILLINK_EXPERIMENTAL_CACHE'], 'true')) + continueOnError: true diff --git a/eng/pipelines/coreclr/templates/setup-build-caches.yml b/eng/pipelines/coreclr/templates/setup-build-caches.yml index 8404bf3203968b..36de64a7632368 100644 --- a/eng/pipelines/coreclr/templates/setup-build-caches.yml +++ b/eng/pipelines/coreclr/templates/setup-build-caches.yml @@ -10,6 +10,7 @@ parameters: platform: '' shouldContinueOnError: false osSubgroup: '' + enableILLinkCache: false # sccache version. Keep the version in sync across platforms so all jobs # cache against the same compiler-launcher behavior. @@ -52,8 +53,7 @@ steps: Write-Host "##vso[task.setvariable variable=ROSLYN_CACHE_BUILD_START]$buildStart" displayName: Configure Roslyn compiler cache - # sccache native compiler cache - - ${{ if and(or(eq(parameters.osGroup, 'linux'), eq(parameters.osGroup, 'freebsd'), eq(parameters.osGroup, 'openbsd'), eq(parameters.osGroup, 'osx')), or(eq(parameters.archType, 'x64'), eq(parameters.archType, 'arm64'))) }}: + - ${{ if eq(parameters.enableILLinkCache, true) }}: - template: /eng/pipelines/coreclr/templates/setup-illink-cache.yml parameters: osGroup: ${{ parameters.osGroup }} @@ -61,6 +61,8 @@ steps: archType: ${{ parameters.archType }} nameSuffix: ${{ parameters.nameSuffix }} + # sccache native compiler cache + - ${{ if and(or(eq(parameters.osGroup, 'linux'), eq(parameters.osGroup, 'freebsd'), eq(parameters.osGroup, 'openbsd'), eq(parameters.osGroup, 'osx')), or(eq(parameters.archType, 'x64'), eq(parameters.archType, 'arm64'))) }}: # Set up the Azure Pipeline Cache for sccache's local cache directory. # Use a rolling key so each build can update the cache; restoreKeys # falls back to the most recent saved entry. diff --git a/eng/pipelines/coreclr/templates/setup-illink-cache.yml b/eng/pipelines/coreclr/templates/setup-illink-cache.yml index 7d8f094cd02c8e..a18c73527ec0d2 100644 --- a/eng/pipelines/coreclr/templates/setup-illink-cache.yml +++ b/eng/pipelines/coreclr/templates/setup-illink-cache.yml @@ -5,24 +5,38 @@ parameters: osSubgroup: '' steps: - # Use the same branch-scoped restore policy as the compiler caches. Each job - # works on its own restored copy, so post-build purge cannot affect another job. - # Keep Debug and Release snapshots separate in rolling builds. - - task: Cache@2 - displayName: ILLink task cache - continueOnError: true - inputs: - key: illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} | "$(buildConfigUpper)" | "$(Build.BuildId)" - path: $(Pipeline.Workspace)/.illink-cache - restoreKeys: | - illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} | "$(buildConfigUpper)" + - ${{ if or(eq(parameters.osGroup, 'windows'), and(in(parameters.osGroup, 'linux', 'freebsd', 'openbsd', 'osx'), in(parameters.archType, 'x64', 'arm64'))) }}: + # Use the same branch-scoped restore policy as the compiler caches. Each job + # works on its own restored copy, so post-build purge cannot affect another job. + # Keep Debug and Release snapshots separate in rolling builds. + - task: Cache@2 + displayName: ILLink task cache + continueOnError: true + inputs: + key: illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} | "$(buildConfigUpper)" | "$(Build.BuildId)" + path: $(Pipeline.Workspace)/.illink-cache + restoreKeys: | + illink | v1 | ${{ parameters.osGroup }}${{ parameters.osSubgroup }} | ${{ parameters.archType }} | ${{ parameters.nameSuffix }} | "$(buildConfigUpper)" - - script: | - set -euo pipefail - mkdir -p "$(Pipeline.Workspace)/.illink-cache" - buildStart="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" - echo "##vso[task.setvariable variable=ILLINK_EXPERIMENTAL_CACHE_PATH]$(Pipeline.Workspace)/.illink-cache" - echo "##vso[task.setvariable variable=ILLINK_CACHE_BUILD_START]$buildStart" - echo "##vso[task.setvariable variable=ILLINK_EXPERIMENTAL_CACHE]true" - displayName: Configure experimental ILLink task cache - continueOnError: true + - ${{ if ne(parameters.osGroup, 'windows') }}: + - script: | + set -euo pipefail + mkdir -p "$(Pipeline.Workspace)/.illink-cache" + buildStart="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + echo "##vso[task.setvariable variable=ILLINK_EXPERIMENTAL_CACHE_PATH]$(Pipeline.Workspace)/.illink-cache" + echo "##vso[task.setvariable variable=ILLINK_CACHE_BUILD_START]$buildStart" + echo "##vso[task.setvariable variable=ILLINK_EXPERIMENTAL_CACHE]true" + displayName: Configure experimental ILLink task cache + continueOnError: true + + - ${{ if eq(parameters.osGroup, 'windows') }}: + - pwsh: | + $cachePath = Join-Path "$(Pipeline.Workspace)" ".illink-cache" + New-Item -ItemType Directory -Path $cachePath -Force | Out-Null + $buildStart = [DateTime]::UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ") + + Write-Host "##vso[task.setvariable variable=ILLINK_EXPERIMENTAL_CACHE_PATH]$cachePath" + Write-Host "##vso[task.setvariable variable=ILLINK_CACHE_BUILD_START]$buildStart" + Write-Host "##vso[task.setvariable variable=ILLINK_EXPERIMENTAL_CACHE]true" + displayName: Configure experimental ILLink task cache + continueOnError: true diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index f40cd1e3ceabee..5db24664ad6c48 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -93,11 +93,18 @@ extends: - osx_arm64 jobParameters: nameSuffix: CoreLib_Libraries_AnalyzersLinker - buildArgs: -subset clr.corelib+libs -restore -build -pack -c $(_BuildConfig) /p:RunAnalyzersInBuild=true /p:RunILLinkInBuild=true + buildArgs: -subset clr.corelib+libs+tools.illink -restore -build -pack -c $(_BuildConfig) /p:RunAnalyzersInBuild=true /p:RunILLinkInBuild=true timeoutInMinutes: 120 + preBuildSteps: + - template: /eng/pipelines/coreclr/templates/setup-build-caches.yml + parameters: + enableILLinkCache: true + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-cache-stats.yml condition: or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_tools_illink.containsChange'], true), eq(variables['isRollingBuild'], true)) # From 04b3e806b60c9ccd8e38d6e5cc7146368c56d747 Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Tue, 20 Oct 2026 11:04:05 -0700 Subject: [PATCH 12/12] Remove obsolete cache purge ordering comment Assisted-by: Copilot:gpt-6.1-sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 561d1c87-e14b-46f8-be7f-a08f544ec537 --- eng/pipelines/common/global-build-job.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index c11ebcda340e26..bdf49787d55b4f 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -274,8 +274,6 @@ jobs: displayName: Collect vslogs on exit condition: always() - # Some post-build steps compile additional tests. Purge only after those - # steps have completed, before Cache@2 saves its post-job snapshot. - template: /eng/pipelines/coreclr/templates/prune-roslyn-cache.yml parameters: osGroup: ${{ parameters.osGroup }}