diff --git a/src/ImageBuilder.Tests/Build/BuildPlannerTests.cs b/src/ImageBuilder.Tests/Build/BuildPlannerTests.cs new file mode 100644 index 000000000..6093354f1 --- /dev/null +++ b/src/ImageBuilder.Tests/Build/BuildPlannerTests.cs @@ -0,0 +1,477 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.ImageBuilder.Build; +using Microsoft.DotNet.ImageBuilder.Commands; +using Microsoft.DotNet.ImageBuilder.Models.Image; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.ImageBuilder.Tests.Helpers; +using Microsoft.DotNet.ImageBuilder.ViewModel; +using Moq; +using Newtonsoft.Json; +using Shouldly; +using static Microsoft.DotNet.ImageBuilder.Tests.Helpers.DockerfileHelper; +using static Microsoft.DotNet.ImageBuilder.Tests.Helpers.ManifestHelper; + +namespace Microsoft.DotNet.ImageBuilder.Tests.Build; + +[TestClass] +public class BuildPlannerTests +{ + [TestMethod] + public async Task ChangedBaseImageExplainsFullDependencyChain() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + ManifestInfo manifest = LoadManifest( + tempFolder, + CreateManifest( + CreateRepo("root", CreateImage(CreatePlatform( + CreateDockerfile("root", tempFolder, "base:tag"), ["tag"]))), + CreateRepo("middle", CreateImage(CreatePlatform( + CreateDockerfile("middle", tempFolder, "root:tag"), ["tag"]))), + CreateRepo("support", CreateImage(CreatePlatform( + CreateDockerfile("support", tempFolder, "support-base:tag"), ["tag"]))), + CreateRepo("leaf", CreateImage(CreatePlatform( + CreateDockerfile("leaf", tempFolder, "support:tag", "middle:tag"), ["tag"]))))); + ImageArtifactDetails imageInfo = CreatePublishedImages( + manifest, + new Dictionary + { + ["root"] = "base@sha256:old", + ["middle"] = "root@sha256:root", + ["support"] = "support-base@sha256:support-base", + ["leaf"] = "middle@sha256:middle", + }); + BuildGraph graph = BuildGraph.Create(manifest); + Mock manifestService = new(); + manifestService + .Setup(service => service.GetManifestDigestShaAsync( + It.Is(image => image.Contains("support-base", StringComparison.Ordinal)), + false)) + .ReturnsAsync("sha256:support-base"); + manifestService + .Setup(service => service.GetManifestDigestShaAsync( + It.Is(image => !image.Contains("support-base", StringComparison.Ordinal)), + false)) + .ReturnsAsync("sha256:new"); + + BuildPlanItem[] plan = await CreatePlanner().CreatePlanAsync( + graph, + imageInfo, + new CompositeBuildPolicy( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("All checks passed, so no work is required.")), + Mock.Of>(), + new MissingPublishedImagePolicy(), + CreateBaseImageRule(manifest, manifestService.Object), + new TagSetChangedPolicy())); + + BuildPlanItem root = GetItem(plan, "root"); + BuildPlanItem middle = GetItem(plan, "middle"); + BuildPlanItem support = GetItem(plan, "support"); + BuildPlanItem leaf = GetItem(plan, "leaf"); + + root.Decision.Action.ShouldBe(BuildAction.BuildImage); + middle.Decision.Action.ShouldBe(BuildAction.BuildImage); + leaf.Decision.Action.ShouldBe(BuildAction.BuildImage); + support.Decision.Action.ShouldBe(BuildAction.UsePublishedImage); + + BuildReason leafReason = leaf.Decision.Reason; + leafReason.Message.ShouldStartWith("Dependency"); + BuildReason middleReason = leafReason.CausedBy.ShouldNotBeNull(); + middleReason.Message.ShouldStartWith("Dependency"); + BuildReason rootReason = middleReason.CausedBy.ShouldNotBeNull(); + rootReason.Message.ShouldContain("changed from 'sha256:old' to 'sha256:new'"); + + BuildReason supportReason = support.Decision.Reason; + supportReason.Message.ShouldContain("leaf/Dockerfile"); + supportReason.CausedBy.ShouldBe(leafReason); + } + + [TestMethod] + public async Task ChangedTagSetsRequireReuse() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + ManifestInfo manifest = LoadManifest( + tempFolder, + CreateManifest( + CreateRepo("runtime", CreateImage( + ["old-shared", "new-shared"], + CreatePlatform( + CreateDockerfile("runtime", tempFolder, "base:tag"), + ["old", "new"]))))); + ImageArtifactDetails imageInfo = CreatePublishedImages( + manifest, + new Dictionary { ["runtime"] = "base@sha256:base" }); + ImageData image = imageInfo.Repos.Single().Images.Single(); + image.Platforms.Single().SimpleTags = ["old", "removed"]; + image.Manifest!.SharedTags = ["old-shared"]; + BuildGraph graph = BuildGraph.Create(manifest); + Mock manifestService = CreateDigestService("sha256:base"); + + BuildPlanItem[] plan = await CreatePlanner().CreatePlanAsync( + graph, + imageInfo, + new CompositeBuildPolicy( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("All checks passed, so no work is required.")), + Mock.Of>(), + new MissingPublishedImagePolicy(), + CreateBaseImageRule(manifest, manifestService.Object), + new TagSetChangedPolicy())); + + BuildPlanItem item = plan.ShouldHaveSingleItem(); + item.Decision.Action.ShouldBe(BuildAction.PublishExistingImage); + BuildReason reason = item.Decision.Reason; + reason.Message.ShouldContain("[old, removed]"); + reason.Message.ShouldContain("[old, new]"); + reason.Message.ShouldContain("[old-shared, new-shared]"); + } + + [TestMethod] + public async Task InvalidatedSharedBuildForcesEveryTargetToBuild() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + string dockerfile = CreateDockerfile("shared", tempFolder, "base:tag"); + ManifestInfo manifest = LoadManifest( + tempFolder, + CreateManifest( + CreateRepo("first", CreateImage(CreatePlatform(dockerfile, ["tag"]))), + CreateRepo("second", CreateImage(CreatePlatform(dockerfile, ["tag"]))))); + ImageArtifactDetails imageInfo = CreatePublishedImages( + manifest, + new Dictionary + { + ["first"] = "base@sha256:old", + ["second"] = "base@sha256:new", + }); + imageInfo.Repos.Single(repo => repo.Repo == "second") + .Images.Single().Platforms.Single().IsUnchanged = true; + BuildGraph graph = BuildGraph.Create(manifest); + Mock manifestService = CreateDigestService("sha256:new"); + + BuildPlanItem[] plan = await CreatePlanner().CreatePlanAsync( + graph, + imageInfo, + new CompositeBuildPolicy( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("All checks passed, so no work is required.")), + Mock.Of>(), + new MissingPublishedImagePolicy(), + CreateBaseImageRule(manifest, manifestService.Object), + new TagSetChangedPolicy())); + + BuildPlanItem first = GetItem(plan, "first"); + BuildPlanItem second = GetItem(plan, "second"); + first.Decision.Action.ShouldBe(BuildAction.BuildImage); + second.Decision.Action.ShouldBe(BuildAction.BuildImage); + BuildReason reason = second.Decision.Reason; + reason.Message.ShouldContain("first"); + reason.CausedBy.ShouldNotBeNull().Message.ShouldContain("changed from"); + } + + [TestMethod] + public async Task SharedTagCreatesDependencyEdge() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + ManifestInfo manifest = LoadManifest( + tempFolder, + CreateManifest( + CreateRepo("parent", CreateImage( + ["shared"], + CreatePlatform( + CreateDockerfile("parent", tempFolder, "base:tag"), + ["specific"]))), + CreateRepo("child", CreateImage(CreatePlatform( + CreateDockerfile("child", tempFolder, "parent:shared"), ["tag"]))))); + ImageArtifactDetails imageInfo = CreatePublishedImages( + manifest, + new Dictionary + { + ["parent"] = "base@sha256:old", + ["child"] = "parent@sha256:parent", + }); + BuildGraph graph = BuildGraph.Create(manifest); + Mock manifestService = CreateDigestService("sha256:new"); + + BuildPlanItem[] plan = await CreatePlanner().CreatePlanAsync( + graph, + imageInfo, + new CompositeBuildPolicy( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("All checks passed, so no work is required.")), + Mock.Of>(), + new MissingPublishedImagePolicy(), + CreateBaseImageRule(manifest, manifestService.Object), + new TagSetChangedPolicy())); + + BuildPlanItem child = GetItem(plan, "child"); + child.Decision.Action.ShouldBe(BuildAction.BuildImage); + child.Decision.Reason.Message.ShouldStartWith("Dependency"); + } + + [TestMethod] + public void StructurallyDifferentBuildArgumentsDoNotShareImages() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + string dockerfile = CreateDockerfile("shared", tempFolder, "base:tag"); + Platform first = CreatePlatform(dockerfile, ["tag"]); + first.BuildArgs = new Dictionary { ["A"] = "x|B=y" }; + Platform second = CreatePlatform(dockerfile, ["tag"]); + second.BuildArgs = new Dictionary { ["A"] = "x", ["B"] = "y" }; + ManifestInfo manifest = LoadManifest( + tempFolder, + CreateManifest( + CreateRepo("first", CreateImage(first)), + CreateRepo("second", CreateImage(second)))); + BuildGraph graph = BuildGraph.Create(manifest); + + foreach (BuildTarget target in graph.Targets) + { + graph.SharedBuildTargets[target].ShouldHaveSingleItem(); + } + } + + [TestMethod] + public void DuplicateFromOverridesProduceOneTargetOverride() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + const string FromImage = "mcr.microsoft.com/app:base"; + string dockerfile = CreateDockerfile("app", tempFolder, FromImage, FromImage); + Manifest model = CreateManifest( + CreateRepo("app", CreateImage(CreatePlatform(dockerfile, ["base"])))); + model.Registry = "mcr.microsoft.com"; + ManifestInfo manifest = LoadManifest(tempFolder, model, repoPrefix: "prefix/"); + BuildGraph graph = BuildGraph.Create(manifest); + + KeyValuePair imageOverride = graph.Targets.ShouldHaveSingleItem() + .FromImageOverrides.ShouldHaveSingleItem(); + imageOverride.Key.ShouldBe(FromImage); + imageOverride.Value.ShouldBe("mcr.microsoft.com/prefix/app:base"); + } + + [TestMethod] + public async Task CachedSiblingIsNotIncludedWhenAnotherChildBuilds() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + ManifestInfo manifest = LoadManifest( + tempFolder, + CreateManifest( + CreateRepo("parent", CreateImage(CreatePlatform( + CreateDockerfile("parent", tempFolder, "base:tag"), ["tag"]))), + CreateRepo("first", CreateImage(CreatePlatform( + CreateDockerfile("first", tempFolder, "parent:tag"), ["tag"]))), + CreateRepo("second", CreateImage(CreatePlatform( + CreateDockerfile("second", tempFolder, "parent:tag"), ["tag"]))))); + ImageArtifactDetails imageInfo = CreatePublishedImages( + manifest, + new Dictionary + { + ["parent"] = "base@sha256:base", + ["first"] = "parent@sha256:parent", + ["second"] = "parent@sha256:parent", + }); + imageInfo.Repos.RemoveAll(repo => repo.Repo == "first"); + BuildGraph graph = BuildGraph.Create(manifest); + Mock manifestService = CreateDigestService("sha256:base"); + + BuildPlanItem[] plan = await CreatePlanner().CreatePlanAsync( + graph, + imageInfo, + new CompositeBuildPolicy( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("All checks passed, so no work is required.")), + Mock.Of>(), + new MissingPublishedImagePolicy(), + CreateBaseImageRule(manifest, manifestService.Object), + new TagSetChangedPolicy())); + + GetItem(plan, "parent").Decision.Action.ShouldBe(BuildAction.UsePublishedImage); + GetItem(plan, "first").Decision.Action.ShouldBe(BuildAction.BuildImage); + GetItem(plan, "second").Decision.Action.ShouldBe(BuildAction.NoAction); + } + + [TestMethod] + public async Task CustomRuleMethodCanAddAPlanningDecision() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + ManifestInfo manifest = LoadManifest( + tempFolder, + CreateManifest( + CreateRepo("runtime", CreateImage(CreatePlatform( + CreateDockerfile("runtime", tempFolder, "base:tag"), ["tag"]))))); + BuildGraph graph = BuildGraph.Create(manifest); + BuildPlanItem[] plan = await CreatePlanner().CreatePlanAsync( + graph, + imageInfo: null, + new CompositeBuildPolicy( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("No package changed.")), + Mock.Of>(), + new PackageVersionChangedPolicy())); + + BuildPlanItem item = plan.ShouldHaveSingleItem(); + item.Decision.Action.ShouldBe(BuildAction.BuildImage); + item.Decision.Reason.Message.ShouldContain("openssl"); + } + + [TestMethod] + public async Task CompositePolicyAppliesEveryChildAndChoosesStrongestAction() + { + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + ManifestInfo manifest = LoadManifest( + tempFolder, + CreateManifest( + CreateRepo("runtime", CreateImage(CreatePlatform( + CreateDockerfile("runtime", tempFolder, "base:tag"), ["tag"]))))); + BuildGraph graph = BuildGraph.Create(manifest); + List appliedPolicies = []; + IBuildPolicy policy = new CompositeBuildPolicy( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("No checks selected work.")), + Mock.Of>(), + new TestPolicy( + appliedPolicies, + "use", + new BuildPolicyResult( + BuildAction.UsePublishedImage, + new BuildReason("Use the published image."))), + new TestPolicy( + appliedPolicies, + "build", + new BuildPolicyResult( + BuildAction.BuildImage, + new BuildReason("Build the image.")))); + + BuildPlanItem[] plan = await CreatePlanner().CreatePlanAsync( + graph, + CreatePublishedImages( + manifest, + new Dictionary { ["runtime"] = "base@sha256:base" }), + policy); + + appliedPolicies.ShouldBe(["use", "build"]); + BuildPlanItem item = plan.ShouldHaveSingleItem(); + item.Decision.Action.ShouldBe(BuildAction.BuildImage); + item.Decision.Reason.Message.ShouldBe("Build the image."); + } + + private static BuildPlanner CreatePlanner() => + new(Mock.Of>()); + + private static IBuildPolicy CreateBaseImageRule( + ManifestInfo manifest, + IManifestService manifestService) => + BaseImageChangedPolicy.FromRegistry( + new ImageDigestCache(new Lazy(() => manifestService)), + new ImageNameResolverForMatrix(new(), manifest, null, null), + isDryRun: false); + + private sealed class PackageVersionChangedPolicy : IBuildPolicy + { + public Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default) => + Task.FromResult( + new BuildPolicyResult( + BuildAction.BuildImage, + new BuildReason("Package 'openssl' changed from '1.0' to '1.1'."))); + } + + private sealed class TestPolicy( + ICollection appliedPolicies, + string name, + BuildPolicyResult result) : IBuildPolicy + { + public Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default) + { + appliedPolicies.Add(name); + return Task.FromResult(result); + } + } + + private static Mock CreateDigestService(string digest) + { + Mock manifestService = new(); + manifestService + .Setup(service => service.GetManifestDigestShaAsync(It.IsAny(), false)) + .ReturnsAsync(digest); + return manifestService; + } + + private static BuildPlanItem GetItem( + IEnumerable plan, + string repoName) => + plan.Single(item => item.Target.Repo.Name == repoName); + + private static ManifestInfo LoadManifest( + TempFolderContext tempFolder, + Manifest manifest, + string? repoPrefix = null) + { + string manifestPath = Path.Combine(tempFolder.Path, "manifest.json"); + File.WriteAllText(manifestPath, JsonConvert.SerializeObject(manifest)); + IManifestOptionsInfo options = GetManifestOptions(manifestPath); + Mock.Get(options) + .SetupGet(manifestOptions => manifestOptions.RepoPrefix) + .Returns(repoPrefix); + return TestHelper.CreateManifestJsonService().Load(options); + } + + private static ImageArtifactDetails CreatePublishedImages( + ManifestInfo manifest, + IReadOnlyDictionary baseImageDigests) + { + ImageArtifactDetails details = new(); + + foreach (RepoInfo repo in manifest.AllRepos) + { + RepoData repoData = new() { Repo = repo.Name }; + details.Repos.Add(repoData); + + foreach (ImageInfo image in repo.AllImages) + { + ImageData imageData = new() + { + Manifest = image.SharedTags.Any() + ? new ManifestData + { + SharedTags = image.SharedTags.Select(tag => tag.Name).ToList() + } + : null + }; + repoData.Images.Add(imageData); + + foreach (PlatformInfo platform in image.AllPlatforms) + { + imageData.Platforms.Add(new PlatformData(image, platform) + { + Dockerfile = platform.DockerfilePathRelativeToManifest, + Digest = $"{repo.Name}@sha256:{repo.Name}", + BaseImageDigest = baseImageDigests[repo.Name], + CommitUrl = $"https://example.test/{platform.DockerfilePathRelativeToManifest}", + SimpleTags = platform.Tags.Select(tag => tag.Name).ToList(), + }); + } + } + } + + return details; + } +} diff --git a/src/ImageBuilder.Tests/BuildCommandTests.cs b/src/ImageBuilder.Tests/BuildCommandTests.cs index c34c17119..ba048eb66 100644 --- a/src/ImageBuilder.Tests/BuildCommandTests.cs +++ b/src/ImageBuilder.Tests/BuildCommandTests.cs @@ -1,4 +1,4 @@ -#nullable disable +#nullable disable // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Azure.ResourceManager.ContainerRegistry.Models; using FluentAssertions; +using Microsoft.DotNet.ImageBuilder.Build; using Microsoft.DotNet.ImageBuilder.Commands; using Microsoft.DotNet.ImageBuilder.Configuration; using Microsoft.DotNet.ImageBuilder.Models.Image; @@ -136,7 +137,7 @@ public async Task BuildCommand_ImageInfoOutput_Basic() "1.0/runtime-deps/os", tempFolderContext, baseImageTag); string runtimeDockerfileRelativePath = DockerfileHelper.CreateDockerfile( - "1.0/runtime/os", tempFolderContext, $"{runtimeDepsRepo}:{tag}"); + "1.0/runtime/os", tempFolderContext, $"{runtimeDepsRepo}:shared"); string aspnetDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/aspnet/os", tempFolderContext, $"{runtimeRepo}:{tag}"); @@ -159,7 +160,7 @@ public async Task BuildCommand_ImageInfoOutput_Basic() processService: processServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: manifestServiceFactoryMock.Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); command.Options.IsPushEnabled = true; @@ -355,7 +356,7 @@ public async Task BuildCommand_ImageInfoOutput_DuplicatedPlatform() gitService: gitServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: manifestServiceFactoryMock.Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); @@ -480,7 +481,7 @@ public async Task BuildCommand_Publish() dockerService: dockerServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: CreateManifestServiceFactoryMock().Object, - imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.IsPushEnabled = true; @@ -554,7 +555,7 @@ public async Task BuildCommand_VerifyOnBaseImageArchMismatch() dockerService: dockerServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock().Object, - imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); const string runtimeRelativeDir = "1.0/runtime/os"; @@ -608,7 +609,7 @@ public async Task BuildCommand_ArmVariantCompatibility(string manifestVariant, s dockerService: dockerServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock().Object, - imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); const string runtimeRelativeDir = "1.0/runtime/os"; @@ -654,7 +655,7 @@ public async Task BuildCommand_BuildArgs() dockerService: dockerServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock().Object, - imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.BuildArgs.Add("arg1", "val1"); command.Options.BuildArgs.Add("arg2", "val2a"); @@ -712,7 +713,7 @@ public async Task BuildCommand_DockerBuildOptions() dockerService: dockerServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock().Object, - imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.DockerBuildOptions = ["--ulimit nofile=65536:65536", "--network host"]; @@ -764,7 +765,7 @@ public async Task BuildCommand_NoBaseImage_Build() BuildCommand command = CreateBuildCommand( dockerService: dockerServiceMock.Object, copyImageService: Mock.Of(), - imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.IsPushEnabled = true; @@ -861,7 +862,7 @@ public async Task BuildCommand_NoBaseImage_Cached() gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: manifestServiceFactoryMock.Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "dest-image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -1045,7 +1046,7 @@ public async Task BuildCommand_ImageInfoOutput_CustomDockerfile() gitService: gitServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: manifestServiceFactoryMock.Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); command.Options.SourceRepoUrl = "https://source"; @@ -1089,7 +1090,7 @@ public async Task BuildCommand_ThrowsIfImageIsPulled(bool isSkipPullingEnabled) dockerService: dockerServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock().Object, - imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.IsSkipPullingEnabled = isSkipPullingEnabled; @@ -1301,7 +1302,7 @@ public async Task BuildCommand_Caching( gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: manifestServiceFactoryMock.Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "dest-image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -1511,7 +1512,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_MissingSourceImageInfoEn gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: manifestServiceFactoryMock.Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "dest-image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -1822,7 +1823,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_MissingSourceImageInfoEn gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: manifestServiceFactoryMock.Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "dest-image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -2110,7 +2111,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_NoExistingImageInfoEntri gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "dest-image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -2316,7 +2317,7 @@ public async Task BuildCommand_SharedDockerfile() gitService: gitServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "dest-image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -2561,7 +2562,7 @@ public async Task BuildCommand_Caching_TagUpdate() gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "dest-image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -2783,7 +2784,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_TagUpdate() gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "dest-image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -3090,7 +3091,7 @@ public async Task BuildCommand_MirroredImages(bool hasCachedImage, string srcBas gitService: gitServiceMock.Object, copyImageService: copyImageServiceMock.Object, manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); command.Options.ImageInfoSourcePath = Path.Combine(tempFolderContext.Path, "src-image-info.json"); @@ -3433,7 +3434,7 @@ public async Task BuildCommand_MirroredImages_External(string baseImageRegistry, gitService: gitServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); command.Options.IsPushEnabled = true; @@ -3529,7 +3530,7 @@ public async Task BuildCommand_MirroredImages_BaseImageTagOverride() gitService: gitServiceMock.Object, copyImageService: Mock.Of(), manifestServiceFactory: CreateManifestServiceFactoryMock(manifestServiceMock).Object, - imageCacheService: new ImageCacheService(Mock.Of>(), gitServiceMock.Object)); + buildPlanner: CreateBuildPlanner()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.ImageInfoOutputPath = Path.Combine(tempFolderContext.Path, "image-info.json"); command.Options.IsPushEnabled = true; @@ -3641,7 +3642,7 @@ private static BuildCommand CreateBuildCommand( IManifestServiceFactory? manifestServiceFactory = null, IRegistryCredentialsProvider? registryCredentialsProvider = null, IAzureTokenCredentialProvider? azureTokenCredentialProvider = null, - IImageCacheService? imageCacheService = null) + BuildPlanner? buildPlanner = null) { BuildCommand command = new( manifestJsonService ?? TestHelper.CreateManifestJsonService(), @@ -3653,10 +3654,14 @@ private static BuildCommand CreateBuildCommand( manifestServiceFactory ?? Mock.Of(), registryCredentialsProvider ?? Mock.Of(), azureTokenCredentialProvider ?? Mock.Of(), - imageCacheService ?? Mock.Of()); + buildPlanner ?? CreateBuildPlanner()); return command; } + + private static BuildPlanner CreateBuildPlanner() => + new(Mock.Of>()); + #nullable disable private static Mock CreateDockerServiceMock(string buildOutput = null) diff --git a/src/ImageBuilder.Tests/GenerateBuildMatrixCommandTests.cs b/src/ImageBuilder.Tests/GenerateBuildMatrixCommandTests.cs index 68f1c0130..98d20d26f 100644 --- a/src/ImageBuilder.Tests/GenerateBuildMatrixCommandTests.cs +++ b/src/ImageBuilder.Tests/GenerateBuildMatrixCommandTests.cs @@ -8,7 +8,9 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.ImageBuilder.Build; using Microsoft.DotNet.ImageBuilder.Commands; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.Models.Manifest; @@ -159,56 +161,82 @@ public async Task GenerateBuildMatrixCommand_PlatformDependencyGraph(string filt } } - private static void SetCacheResult(Mock imageCacheServiceMock, string dockerfilePath, ImageCacheState cacheState) + [TestMethod] + public async Task PlatformDependencyGraph_SharedTagDependency() { - imageCacheServiceMock - .Setup(o => o.CheckForCachedImageAsync( - It.IsAny(), - It.Is(platform => platform.Dockerfile == dockerfilePath), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny())) - .ReturnsAsync(new ImageCacheResult(cacheState, false, null)); + using TempFolderContext tempFolder = TestHelper.UseTempFolder(); + string parentDockerfile = CreateDockerfile( + "parent", + tempFolder, + "base:tag"); + string childDockerfile = CreateDockerfile( + "child", + tempFolder, + "parent:shared"); + Manifest manifest = CreateManifest( + CreateRepo( + "parent", + CreateImage( + ["shared"], + CreatePlatform(parentDockerfile, ["specific"]))), + CreateRepo( + "child", + CreateImage(CreatePlatform(childDockerfile, ["tag"])))); + GenerateBuildMatrixCommand command = CreateCommand(); + command.Options.Manifest = Path.Combine(tempFolder.Path, "manifest.json"); + command.Options.MatrixType = MatrixType.PlatformDependencyGraph; + File.WriteAllText( + command.Options.Manifest, + JsonConvert.SerializeObject(manifest)); + + command.LoadManifest(); + BuildMatrixInfo matrix = (await command.GenerateMatrixInfoAsync()) + .ShouldHaveSingleItem(); + BuildLegInfo leg = matrix.Legs.ShouldHaveSingleItem(); + + leg.Variables + .Single(variable => variable.Name == "imageBuilderPaths") + .Value + .ShouldBe("--path parent/Dockerfile --path child/Dockerfile"); } [TestMethod] [DataRow( - ImageCacheState.NotCached, - ImageCacheState.NotCached, + BuildAction.BuildImage, + BuildAction.BuildImage, "--path 1.0/runtime/os/amd64/Dockerfile --path 1.0/sdk/os/amd64/Dockerfile", "--path 2.0/runtime/os/amd64/Dockerfile --path 2.0/sdk/os/amd64/Dockerfile")] [DataRow( - ImageCacheState.Cached, - ImageCacheState.Cached, + BuildAction.NoAction, + BuildAction.NoAction, "--path 2.0/runtime/os/amd64/Dockerfile --path 2.0/sdk/os/amd64/Dockerfile")] [DataRow( - ImageCacheState.Cached, - ImageCacheState.Cached, + BuildAction.NoAction, + BuildAction.NoAction, "--path 1.0/standalone/os/amd64/Dockerfile", "--path 2.0/standalone/os/amd64/Dockerfile", null, "*standalone*")] [DataRow( - ImageCacheState.CachedWithMissingTags, - ImageCacheState.Cached, + BuildAction.UsePublishedImage, + BuildAction.NoAction, + "--path 1.0/runtime/os/amd64/Dockerfile", "--path 2.0/runtime/os/amd64/Dockerfile --path 2.0/sdk/os/amd64/Dockerfile")] [DataRow( - ImageCacheState.Cached, - ImageCacheState.NotCached, + BuildAction.UsePublishedImage, + BuildAction.BuildImage, "--path 1.0/runtime/os/amd64/Dockerfile --path 1.0/sdk/os/amd64/Dockerfile", "--path 2.0/runtime/os/amd64/Dockerfile --path 2.0/sdk/os/amd64/Dockerfile")] [DataRow( - ImageCacheState.NotCached, - ImageCacheState.NotCached, + BuildAction.BuildImage, + BuildAction.BuildImage, "--path 1.0/runtime/os/amd64/Dockerfile --path 1.0/sdk/os/amd64/Dockerfile", "--path 1.0/standalone/os/amd64/Dockerfile", "--path 2.0/runtime/os/amd64/Dockerfile --path 2.0/sdk/os/amd64/Dockerfile", "")] // Clear out the path filters to ensure all images are included public async Task FilterOutCachedImages( - ImageCacheState runtime1CacheState, - ImageCacheState sdk1CacheState, + BuildAction runtime1Action, + BuildAction sdk1Action, string leg1ExpectedPaths, string leg2ExpectedPaths = null, string leg3ExpectedPaths = null, @@ -251,14 +279,41 @@ public async Task FilterOutCachedImages( CreatePlatform(dockerfileSdk2Path = CreateDockerfile(Sdk2RelativeDir, tempFolderContext, "runtime:2.0"), ["2.0"]))) ); - Mock imageCacheServiceMock = new(); - SetCacheResult(imageCacheServiceMock, dockerfileStandalone1Path, ImageCacheState.NotCached); - SetCacheResult(imageCacheServiceMock, dockerfileRuntime1Path, runtime1CacheState); - SetCacheResult(imageCacheServiceMock, dockerfileSdk1Path, sdk1CacheState); - SetCacheResult(imageCacheServiceMock, dockerfileRuntime2Path, ImageCacheState.NotCached); - SetCacheResult(imageCacheServiceMock, dockerfileSdk2Path, ImageCacheState.NotCached); + Dictionary actions = new() + { + [dockerfileRuntime1Path] = runtime1Action, + [dockerfileSdk1Path] = sdk1Action, + }; + Mock buildPlannerMock = new( + Mock.Of>()); + buildPlannerMock + .Setup(planner => planner.CreatePlanAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(( + BuildGraph graph, + ImageArtifactDetails imageInfo, + IBuildPolicy policy, + CancellationToken _) => Task.FromResult( + graph.Targets.Select(target => + new BuildPlanItem( + target, + new BuildPolicyResult( + actions.GetValueOrDefault( + target.Platform.DockerfilePathRelativeToManifest, + BuildAction.BuildImage), + new BuildReason("Test-selected action.")), + PublishedImage: null)) + .ToArray())); - GenerateBuildMatrixCommand command = new(TestHelper.CreateManifestJsonService(), imageCacheServiceMock.Object, Mock.Of(), Mock.Of>()); + GenerateBuildMatrixCommand command = new( + TestHelper.CreateManifestJsonService(), + buildPlannerMock.Object, + Mock.Of(), + Mock.Of(), + Mock.Of>()); command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); command.Options.MatrixType = MatrixType.PlatformDependencyGraph; command.Options.ImageInfoPath = Path.Combine(tempFolderContext.Path, "imageinfo.json"); @@ -1707,13 +1762,12 @@ private static GenerateBuildMatrixCommand SetupTrimCacheTest( externalImageDigestResults: externalImageDigestResults ?? []); } - ImageCacheService imageCacheService = new( - Mock.Of>(), - gitServiceMock.Object); + BuildPlanner buildPlanner = new(Mock.Of>()); GenerateBuildMatrixCommand command = new( TestHelper.CreateManifestJsonService(), - imageCacheService, + buildPlanner, + gitServiceMock.Object, manifestServiceFactoryMock.Object, Mock.Of>()); @@ -1758,6 +1812,11 @@ private static GenerateBuildMatrixCommand SetupTrimCacheTest( } private static GenerateBuildMatrixCommand CreateCommand() => - new(TestHelper.CreateManifestJsonService(), Mock.Of(), Mock.Of(), Mock.Of>()); + new( + TestHelper.CreateManifestJsonService(), + new BuildPlanner(Mock.Of>()), + Mock.Of(), + Mock.Of(), + Mock.Of>()); } } diff --git a/src/ImageBuilder.Tests/GetStaleImagesCommandTests.cs b/src/ImageBuilder.Tests/GetStaleImagesCommandTests.cs index 67836b594..1096e2e43 100644 --- a/src/ImageBuilder.Tests/GetStaleImagesCommandTests.cs +++ b/src/ImageBuilder.Tests/GetStaleImagesCommandTests.cs @@ -1,4 +1,4 @@ -#nullable disable +#nullable disable // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -12,6 +12,7 @@ using System.Text; using System.Threading.Tasks; using LibGit2Sharp; +using Microsoft.DotNet.ImageBuilder.Build; using Microsoft.DotNet.ImageBuilder.Commands; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.Models.Manifest; @@ -1373,6 +1374,7 @@ public async Task GetStaleImagesCommand_InternalFromOnly() const string repo1 = "test-repo"; const string dockerfile1Path = "dockerfile1/Dockerfile"; const string dockerfile2Path = "dockerfile2/Dockerfile"; + string parentDigest = $"sha256:{new string('0', 64)}"; SubscriptionInfo[] subscriptionInfos = new SubscriptionInfo[] { @@ -1398,11 +1400,15 @@ public async Task GetStaleImagesCommand_InternalFromOnly() { Platforms = { - CreatePlatform(dockerfile1Path), + CreatePlatform( + dockerfile1Path, + baseImageDigest: $"{repo1}@{parentDigest}", + simpleTags: new List { "tag1" }), CreatePlatform( dockerfile2Path, + digest: parentDigest, baseImageDigest: "base1@base1digest", - simpleTags: new List { "tag1" }) + simpleTags: new List { "tag2" }) } } } @@ -1754,7 +1760,7 @@ public void Verify(IDictionary> expectedPathsBySubsc string[] actualPaths = pathsBySubscription .First(imagePaths => imagePaths.SubscriptionId == kvp.Key.Id).ImagePaths; - actualPaths.ShouldBe(kvp.Value); + actualPaths.ShouldBe(kvp.Value, ignoreOrder: true); } } @@ -1769,7 +1775,12 @@ private string SerializeJsonObjectToTempFile(object jsonObject) private GetStaleImagesCommand CreateCommand() { GetStaleImagesCommand command = new( - this.ManifestServiceFactoryMock.Object, TestHelper.CreateManifestJsonService(), this.loggerServiceMock.Object, this.octokitClientFactory, this.gitService); + this.ManifestServiceFactoryMock.Object, + TestHelper.CreateManifestJsonService(), + this.loggerServiceMock.Object, + this.octokitClientFactory, + this.gitService, + new BuildPlanner(Mock.Of>())); command.Options.SubscriptionOptions.SubscriptionsPath = this.subscriptionsPath; command.Options.VariableName = VariableName; command.Options.FilterOptions.Platform.OsType = this.osType; diff --git a/src/ImageBuilder/Build/BuildGraph.cs b/src/ImageBuilder/Build/BuildGraph.cs new file mode 100644 index 000000000..c33541b4d --- /dev/null +++ b/src/ImageBuilder/Build/BuildGraph.cs @@ -0,0 +1,199 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.DotNet.ImageBuilder.Models.Image; +using Microsoft.DotNet.ImageBuilder.ViewModel; + +namespace Microsoft.DotNet.ImageBuilder.Build; + +/// +/// A current image definition that ImageBuilder can build. +/// +public sealed record BuildTarget( + RepoInfo Repo, + ImageInfo Image, + PlatformInfo Platform, + IReadOnlyDictionary FromImageOverrides) +{ + public string DisplayName => + $"{Repo.Name} ({Platform.DockerfilePathRelativeToManifest})"; +} + +/// +/// Dependency, shared-build, and published-image data for build targets. +/// +public sealed record BuildGraph( + IReadOnlyList Targets, + IReadOnlyDictionary> Parents, + IReadOnlyDictionary> Children, + IReadOnlyDictionary> SharedBuildTargets) +{ + /// + /// Creates a graph for every platform in the manifest. + /// + public static BuildGraph Create(ManifestInfo manifest) => + CreateForPlatforms(manifest, manifest.GetAllPlatforms()); + + /// + /// Creates a graph for the manifest platforms selected by its command-line filters. + /// + public static BuildGraph CreateFiltered(ManifestInfo manifest) => + CreateForPlatforms(manifest, manifest.GetFilteredPlatforms()); + + /// + /// Creates a graph for selected platforms after applying the manifest's command-line filters. + /// + public static BuildGraph CreateFiltered( + ManifestInfo manifest, + Func include) => + CreateForPlatforms(manifest, manifest.GetFilteredPlatforms().Where(include)); + + private static BuildGraph CreateForPlatforms( + ManifestInfo manifest, + IEnumerable platforms) + { + // Create build targets. These contain all the information needed to build an image. + HashSet platformSet = platforms.ToHashSet(); + BuildTarget[] targets = manifest.AllRepos + .SelectMany(repo => repo.AllImages.SelectMany(image => + image.AllPlatforms + .Where(platformSet.Contains) + .Select(platform => new BuildTarget( + repo, + image, + platform, + GetFromImageOverrides(manifest, platform))))) + .ToArray(); + + // Index every platform and shared tag so internal FROM references can be resolved. + Dictionary> targetsByTag = []; + foreach (BuildTarget target in targets) + { + foreach (string tag in GetTags(target)) + { + if (!targetsByTag.TryGetValue(tag, out List? taggedTargets)) + { + taggedTargets = []; + targetsByTag.Add(tag, taggedTargets); + } + + taggedTargets.Add(target); + } + } + + // Create the dependency graph from internal FROM references. + Dictionary> parents = targets.ToDictionary( + target => target, + target => (IReadOnlyList)target.Platform.InternalFromImages + .Select(fromImage => ResolveParent(target, fromImage, targetsByTag)) + .Where(parent => parent is not null) + .Cast() + .Distinct() + .ToArray()); + Dictionary> mutableChildren = targets.ToDictionary( + target => target, + _ => new List()); + + foreach ((BuildTarget child, IReadOnlyList targetParents) in parents) + { + foreach (BuildTarget parent in targetParents) + { + mutableChildren[parent].Add(child); + } + } + + Dictionary> children = + mutableChildren.ToDictionary( + pair => pair.Key, + pair => (IReadOnlyList)pair.Value); + + // Group targets that produce equivalent image content and can share published images. + Dictionary> sharedBuildTargets = []; + foreach (IGrouping group in targets.GroupBy(GetSharedBuildKey)) + { + BuildTarget[] groupTargets = group.ToArray(); + foreach (BuildTarget target in groupTargets) + { + sharedBuildTargets[target] = groupTargets; + } + } + + return new BuildGraph( + targets, + parents, + children, + sharedBuildTargets); + } + + private static IReadOnlyDictionary GetFromImageOverrides( + ManifestInfo manifest, + PlatformInfo platform) => + platform.OverriddenFromImages + .Distinct(StringComparer.Ordinal) + .ToDictionary( + fromImage => fromImage, + fromImage => + { + string fromRepo = DockerHelper.GetRepo(fromImage); + RepoInfo repo = manifest.AllRepos.First(repo => + repo.FullModelName == fromRepo); + return DockerHelper.ReplaceRepo(fromImage, repo.QualifiedName); + }); + + private static IEnumerable GetTags(BuildTarget target) => + target.Platform.Tags + .Concat(target.Image.SharedTags) + .Select(tag => tag.FullyQualifiedName) + .Distinct(StringComparer.Ordinal); + + private static BuildTarget? ResolveParent( + BuildTarget child, + string fromImage, + IReadOnlyDictionary> targetsByTag) + { + if (!targetsByTag.TryGetValue(fromImage, out List? candidates)) + { + return null; + } + + BuildTarget[] distinctCandidates = candidates.Distinct().ToArray(); + BuildTarget[] platformMatches = distinctCandidates + .Where(candidate => HasSameTargetPlatform(candidate, child)) + .ToArray(); + BuildTarget[] matches = platformMatches.Length > 0 + ? platformMatches + : distinctCandidates; + + return matches.Length switch + { + 1 => matches[0], + _ => throw new InvalidOperationException( + $"Internal image '{fromImage}' has {matches.Length} candidate platforms for " + + $"'{child.Platform.DockerfilePathRelativeToManifest}'.") + }; + } + + private static bool HasSameTargetPlatform( + BuildTarget first, + BuildTarget second) => + first.Platform.Model.OS == second.Platform.Model.OS && + first.Platform.Model.OsVersion == second.Platform.Model.OsVersion && + first.Platform.Model.Architecture == second.Platform.Model.Architecture && + first.Platform.Model.Variant == second.Platform.Model.Variant; + + private static string GetSharedBuildKey(BuildTarget target) => + JsonSerializer.Serialize(new + { + target.Platform.DockerfilePathRelativeToManifest, + target.Platform.PlatformLabel, + BuildArgs = target.Platform.BuildArgs + .OrderBy(argument => argument.Key, StringComparer.Ordinal), + FromImageOverrides = target.FromImageOverrides + .OrderBy(image => image.Key, StringComparer.Ordinal) + }); +} diff --git a/src/ImageBuilder/Build/BuildPlan.cs b/src/ImageBuilder/Build/BuildPlan.cs new file mode 100644 index 000000000..e5ea9ef9f --- /dev/null +++ b/src/ImageBuilder/Build/BuildPlan.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using Microsoft.DotNet.ImageBuilder.Models.Image; + +namespace Microsoft.DotNet.ImageBuilder.Build; + +/// +/// Work that ImageBuilder must perform for a target. +/// +public enum BuildAction +{ + /// + /// The published image is valid and this invocation does not need it locally. + /// + NoAction, + + /// + /// Use the valid published image without running a Docker build. The image may need to be + /// pulled, imported, or retagged for this invocation. + /// + UsePublishedImage, + + /// + /// Use the valid published image and continue it through downstream processing because its + /// published metadata, such as tags, must be updated. + /// + PublishExistingImage, + + /// + /// Run a Docker build for the target. + /// + BuildImage +} + +public static class BuildActionExtensions +{ + public static int GetPriority(this BuildAction action) => + action switch + { + BuildAction.NoAction => 0, + BuildAction.UsePublishedImage => 1, + BuildAction.PublishExistingImage => 2, + BuildAction.BuildImage => 3, + _ => throw new ArgumentOutOfRangeException(nameof(action)) + }; +} + +/// +/// An explanation for a planned action, optionally linked to the reason that caused it. +/// +public sealed record BuildReason( + string Message, + BuildReason? CausedBy = null); + +/// +/// Published image-info associated with a build target. +/// +/// Target whose image-info supplied the data. +/// Existing published platform data. +/// Published image-level tags stored outside . +public sealed record PublishedImage( + BuildTarget Source, + PlatformData Image, + IReadOnlyList SharedTags); + +public sealed record BuildPlanItem( + BuildTarget Target, + BuildPolicyResult Decision, + PublishedImage? PublishedImage); diff --git a/src/ImageBuilder/Build/BuildPlanner.cs b/src/ImageBuilder/Build/BuildPlanner.cs new file mode 100644 index 000000000..96249dad9 --- /dev/null +++ b/src/ImageBuilder/Build/BuildPlanner.cs @@ -0,0 +1,279 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.ImageBuilder.Models.Image; +using Microsoft.DotNet.ImageBuilder.ViewModel; + +namespace Microsoft.DotNet.ImageBuilder.Build; + +/// +/// Calculates the image work required for a build without executing that work. +/// +public class BuildPlanner(ILogger logger) +{ + public virtual async Task CreatePlanAsync( + BuildGraph graph, + ImageArtifactDetails? imageInfo, + IBuildPolicy policy, + CancellationToken cancellationToken = default) + { + Dictionary publishedImages = CreatePublishedImageIndex(graph, imageInfo); + Dictionary planItems = []; + + // Shared builds act as one node. Evaluate and apply each node from roots to leaves so + // every parent decision is final before its children are considered. + foreach (var sharedBuildTargets in GetSharedBuildsInDependencyOrder(graph)) + { + foreach (BuildTarget target in sharedBuildTargets) + { + var context = new BuildPolicyContext(graph, target, publishedImages); + BuildPolicyResult decision = await policy.EvaluateAsync(context, cancellationToken); + + publishedImages.TryGetValue(target, out PublishedImage? publishedImage); + planItems.Add(target, CreateItem(target, decision, publishedImage)); + } + + PropagateBuildsFromParents(graph, sharedBuildTargets, planItems); + UnifySharedBuildActions(sharedBuildTargets, planItems); + } + + // Built images need their direct internal parents available locally. An unchanged parent + // can use its published image; it does not need its own parents because it is not rebuilt. + UsePublishedParentsForBuilds(graph, planItems, publishedImages); + + BuildPlanItem[] plan = graph.Targets.Select(target => planItems[target]).ToArray(); + LogPlan(plan); + return plan; + } + + /// + /// Finds previously published image data for each build target. + /// + private static Dictionary CreatePublishedImageIndex( + BuildGraph graph, + ImageArtifactDetails? imageInfo) + { + Dictionary targetsByPlatform = + graph.Targets.ToDictionary(target => target.Platform); + + Dictionary publishedImages = + imageInfo?.Repos + // Collect (image, platform) pairs for each platform + .SelectMany(repo => repo.Images) + .SelectMany( + image => image.Platforms.Select( + platform => (Platform: platform, SharedTags: image.Manifest?.SharedTags?.ToArray() ?? []) + ) + ) + .Where(item => + // PlatformInfo is only set when a published platform matches one in the current manifest. + item.Platform.PlatformInfo is not null + // Only select platforms that are part of the current build graph. + && targetsByPlatform.ContainsKey(item.Platform.PlatformInfo) + ) + .ToDictionary( + // SAFETY: PlatformInfo is checked in the Where clause above. + item => targetsByPlatform[item.Platform.PlatformInfo!], + item => new PublishedImage( + targetsByPlatform[item.Platform.PlatformInfo!], + item.Platform, + item.SharedTags) + ) + ?? []; + + // Equivalent targets can reuse one another's published image when only one target has + // a direct image-info entry. + foreach (IReadOnlyList sharedBuild in graph.SharedBuildTargets.Values.Distinct()) + { + BuildTarget? source = sharedBuild.FirstOrDefault(publishedImages.ContainsKey); + + if (source is null) + continue; + + foreach (BuildTarget target in sharedBuild) + { + publishedImages.TryAdd( + key: target, + value: new PublishedImage( + source, + publishedImages[source].Image, + publishedImages[source].SharedTags)); + } + } + + return publishedImages; + } + + private static List> GetSharedBuildsInDependencyOrder(BuildGraph graph) + { + IReadOnlyList[] sharedBuilds = graph.SharedBuildTargets.Values + .DistinctBy(targets => targets[0]) + .ToArray(); + + Dictionary> sharedBuildsByTarget = + sharedBuilds + .SelectMany(sharedBuild => sharedBuild.Select(target => (Target: target, SharedBuild: sharedBuild))) + .ToDictionary(item => item.Target, item => item.SharedBuild); + + List> ordered = []; + HashSet visiting = []; + HashSet visited = []; + + void Visit(IReadOnlyList sharedBuild) + { + BuildTarget key = sharedBuild[0]; + + if (visited.Contains(key)) + return; + + if (!visiting.Add(key)) + throw new InvalidOperationException($"Build dependency cycle detected at '{key.DisplayName}'."); + + var parents = sharedBuild.SelectMany(target => graph.Parents[target]).Distinct(); + + foreach (BuildTarget parent in parents) + { + IReadOnlyList parentBuild = sharedBuildsByTarget[parent]; + if (parentBuild[0] != key) + Visit(parentBuild); + } + + visiting.Remove(key); + visited.Add(key); + ordered.Add(sharedBuild); + } + + foreach (IReadOnlyList sharedBuild in sharedBuilds) + { + Visit(sharedBuild); + } + + return ordered; + } + + private static void PropagateBuildsFromParents( + BuildGraph graph, + IEnumerable sharedBuild, + Dictionary items) + { + foreach (BuildTarget target in sharedBuild) + { + BuildPlanItem item = items[target]; + + if (item.Decision.Action == BuildAction.BuildImage) + continue; + + BuildTarget? parent = graph.Parents[target] + .FirstOrDefault(parent => items[parent].Decision.Action == BuildAction.BuildImage); + + if (parent is null) + continue; + + items[target] = item with + { + Decision = new BuildPolicyResult( + BuildAction.BuildImage, + new BuildReason( + $"Dependency '{parent.DisplayName}' must build.", + items[parent].Decision.Reason)) + }; + } + } + + private static void UnifySharedBuildActions( + IEnumerable sharedBuild, + Dictionary items) + { + BuildPlanItem? invalidatedItem = sharedBuild + .Select(target => items[target]) + .FirstOrDefault(item => item.Decision.Action == BuildAction.BuildImage); + + if (invalidatedItem is null) + return; + + foreach (BuildTarget target in sharedBuild) + { + BuildPlanItem item = items[target]; + + if (item.Decision.Action == BuildAction.BuildImage) + continue; + + items[target] = item with + { + Decision = new BuildPolicyResult( + BuildAction.BuildImage, + new BuildReason( + $"Equivalent target '{invalidatedItem.Target.DisplayName}' must build.", + invalidatedItem.Decision.Reason)) + }; + } + } + + private static void UsePublishedParentsForBuilds( + BuildGraph graph, + Dictionary items, + Dictionary publishedImages) + { + var childrenToBuild = items.Values + .Where(item => item.Decision.Action == BuildAction.BuildImage) + .ToArray(); + + foreach (BuildPlanItem childItem in childrenToBuild) + { + foreach (BuildTarget parent in graph.Parents[childItem.Target]) + { + BuildPlanItem parentItem = items[parent]; + + if (parentItem.Decision.Action == BuildAction.NoAction) + { + if (!publishedImages.ContainsKey(parent)) + throw new InvalidOperationException( + $"Required dependency '{parent.DisplayName}' has no published image."); + + items[parent] = parentItem with + { + Decision = new BuildPolicyResult( + Action: BuildAction.UsePublishedImage, + Reason: new BuildReason( + $"The image is required by '{childItem.Target.DisplayName}'.", + childItem.Decision.Reason)) + }; + } + } + } + } + + private static BuildPlanItem CreateItem( + BuildTarget target, + BuildPolicyResult decision, + PublishedImage? publishedImage) + { + if ((decision.Action is BuildAction.UsePublishedImage or BuildAction.PublishExistingImage) + && publishedImage is null) + { + throw new InvalidOperationException( + $"Planning selected '{decision.Action}' for '{target.DisplayName}' " + + "without a published image."); + } + + return new BuildPlanItem(target, decision, publishedImage); + } + + private void LogPlan(IEnumerable plan) + { + foreach (BuildPlanItem item in plan) + { + logger.LogInformation( + "Build plan for {BuildTarget}: {Action}. {Reason}", + item.Target.DisplayName, + item.Decision.Action, + item.Decision.Reason); + } + } +} diff --git a/src/ImageBuilder/Build/BuildPolicies.cs b/src/ImageBuilder/Build/BuildPolicies.cs new file mode 100644 index 000000000..7e11fab4c --- /dev/null +++ b/src/ImageBuilder/Build/BuildPolicies.cs @@ -0,0 +1,342 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Microsoft.DotNet.ImageBuilder.Models.Image; + +namespace Microsoft.DotNet.ImageBuilder.Build; + +public sealed record BuildPolicyContext( + BuildGraph Graph, + BuildTarget Target, + IReadOnlyDictionary PublishedImages); + +/// +/// Work selected by a build policy and the reason it was selected. +/// +public sealed record BuildPolicyResult( + BuildAction Action, + BuildReason Reason); + +/// +/// Evaluates one aspect of the work required for a build target. +/// +public interface IBuildPolicy +{ + Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default); +} + +public static class CommonBuildPolicies +{ + public static CompositeBuildPolicy CreateForCachedImages( + BuildPolicyResult defaultResult, + ILogger logger, + BaseImageChangedPolicy baseImagePolicy, + IGitService gitService, + string sourceRepoUrl) + { + return new CompositeBuildPolicy( + defaultResult, + logger, + // Rebuild when no published image metadata exists. + new MissingPublishedImagePolicy(), + + // Rebuild when the base image digest has changed. + baseImagePolicy, + + // Rebuild when the Dockerfile has changed. + new DockerfileChangedPolicy(gitService, sourceRepoUrl), + + // Republish the existing image when its configured tags have changed. + new TagSetChangedPolicy()); + } +} + +/// +/// Applies every child policy and combines their results into one decision. The action with the +/// highest priority wins. +/// +public sealed class CompositeBuildPolicy( + BuildPolicyResult defaultResult, + ILogger logger, + params IBuildPolicy[] policies) : IBuildPolicy +{ + public async Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default) + { + BuildPolicyResult result = defaultResult; + foreach (IBuildPolicy policy in policies) + { + BuildPolicyResult policyResult = + await policy.EvaluateAsync(context, cancellationToken); + + logger.LogDebug( + "Build policy {BuildPolicy} for {BuildTarget} returned {Action}. {Reason}", + policy.GetType().Name, + context.Target.DisplayName, + policyResult.Action, + policyResult.Reason); + + if (policyResult.Action.GetPriority() > result.Action.GetPriority()) + { + result = policyResult; + } + } + + return result; + } +} + +public sealed class AlwaysBuildPolicy(string reason = "Caching is disabled.") : IBuildPolicy +{ + public Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult( + new BuildPolicyResult( + BuildAction.BuildImage, + new BuildReason(reason))); + } +} + +public sealed class MissingPublishedImagePolicy : IBuildPolicy +{ + public Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + BuildPolicyResult result = !context.PublishedImages.ContainsKey(context.Target) + ? new BuildPolicyResult( + BuildAction.BuildImage, + new BuildReason("No published image metadata exists.")) + : new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("Published image metadata exists.")); + return Task.FromResult(result); + } +} + +public sealed class TagSetChangedPolicy : IBuildPolicy +{ + public Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!context.PublishedImages.TryGetValue(context.Target, out var publishedImage)) + { + return Task.FromResult( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason( + "Published image metadata is unavailable, so tags cannot be compared."))); + } + + string[] expectedPlatformTags = context.Target.Platform.Tags + .Select(tag => tag.Name) + .ToArray(); + string[] expectedSharedTags = context.Target.Image.SharedTags + .Select(tag => tag.Name) + .ToArray(); + IEnumerable publishedPlatformTags = + publishedImage.Source == context.Target + ? publishedImage.Image.SimpleTags + : []; + IEnumerable publishedSharedTags = + publishedImage.Source == context.Target + ? publishedImage.SharedTags + : []; + bool tagsChanged = + !expectedPlatformTags.AreEquivalent(publishedPlatformTags) || + !expectedSharedTags.AreEquivalent(publishedSharedTags); + + BuildPolicyResult result = tagsChanged + ? new BuildPolicyResult( + BuildAction.PublishExistingImage, + new BuildReason( + $"Platform tags changed from [{string.Join(", ", publishedPlatformTags)}] " + + $"to [{string.Join(", ", expectedPlatformTags)}]; shared tags changed from " + + $"[{string.Join(", ", publishedSharedTags)}] to " + + $"[{string.Join(", ", expectedSharedTags)}].")) + : new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("Configured tags are unchanged.")); + return Task.FromResult(result); + } +} + +public sealed class DockerfileChangedPolicy( + IGitService gitService, + string sourceRepoUrl) : IBuildPolicy +{ + private readonly IGitService _gitService = + gitService ?? throw new ArgumentNullException(nameof(gitService)); + + public Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!context.PublishedImages.TryGetValue(context.Target, out var publishedImage)) + { + return Task.FromResult( + new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason( + "Published image metadata is unavailable, so the Dockerfile cannot be compared."))); + } + + string currentCommitUrl = _gitService.GetDockerfileCommitUrl( + context.Target.Platform, + sourceRepoUrl); + bool matches = publishedImage.Image.CommitUrl.Equals( + currentCommitUrl, + StringComparison.OrdinalIgnoreCase); + BuildPolicyResult result = matches + ? new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason($"Dockerfile is unchanged at '{currentCommitUrl}'.")) + : new BuildPolicyResult( + BuildAction.BuildImage, + new BuildReason( + $"Dockerfile changed from '{publishedImage.Image.CommitUrl}' " + + $"to '{currentCommitUrl}'.")); + return Task.FromResult(result); + } +} + +public sealed class BaseImageChangedPolicy : IBuildPolicy +{ + private readonly ImageDigestCache _imageDigests; + private readonly ImageNameResolver _imageNames; + private readonly bool _isDryRun; + private readonly bool _useLocalExternalImage; + + private BaseImageChangedPolicy( + ImageDigestCache imageDigests, + ImageNameResolver imageNames, + bool useLocalExternalImage, + bool isDryRun) + { + _imageDigests = imageDigests ?? throw new ArgumentNullException(nameof(imageDigests)); + _imageNames = imageNames ?? throw new ArgumentNullException(nameof(imageNames)); + _useLocalExternalImage = useLocalExternalImage; + _isDryRun = isDryRun; + } + + public static BaseImageChangedPolicy FromLocalImages( + ImageDigestCache imageDigests, + ImageNameResolver imageNames, + bool isDryRun) => + new(imageDigests, imageNames, useLocalExternalImage: true, isDryRun); + + public static BaseImageChangedPolicy FromRegistry( + ImageDigestCache imageDigests, + ImageNameResolver imageNames, + bool isDryRun) => + new(imageDigests, imageNames, useLocalExternalImage: false, isDryRun); + + public async Task EvaluateAsync( + BuildPolicyContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!context.PublishedImages.TryGetValue(context.Target, out var publishedImage)) + { + return new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason( + "Published image metadata is unavailable, so the base image cannot be compared.")); + } + + string? fromImage = context.Target.Platform.FinalStageFromImage; + if (fromImage is null) + { + return new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("The final stage has no base image.")); + } + + string? currentDigest = context.Target.Platform.IsInternalFromImage(fromImage) + ? GetInternalBaseImageDigest(context, fromImage) + : await GetExternalBaseImageDigestAsync(context.Target, fromImage); + string publicImage = _imageNames.GetFromImagePublicTag(fromImage); + string? previousValue = GetDigestSha(publishedImage.Image.BaseImageDigest); + string? currentValue = GetDigestSha(currentDigest); + bool matches = previousValue?.Equals( + currentValue, + StringComparison.OrdinalIgnoreCase) == true; + + BuildPolicyResult result = matches + ? new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason( + $"Base image '{publicImage}' is unchanged at '{currentValue}'.")) + : new BuildPolicyResult( + BuildAction.BuildImage, + new BuildReason( + $"Base image '{publicImage}' changed from " + + $"'{Display(previousValue)}' to '{Display(currentValue)}'.")); + return result; + } + + private static string? GetInternalBaseImageDigest( + BuildPolicyContext context, + string fromImage) + { + BuildTarget? parent = context.Graph.Parents[context.Target].FirstOrDefault(candidate => + candidate.Platform.Tags + .Concat(candidate.Image.SharedTags) + .Any(tag => tag.FullyQualifiedName == fromImage)); + return parent is not null && + context.PublishedImages.TryGetValue(parent, out var publishedImage) + ? publishedImage.Image.Digest + : null; + } + + private async Task GetExternalBaseImageDigestAsync( + BuildTarget target, + string fromImage) + { + if (_useLocalExternalImage) + { + string localImage = _imageNames.GetFromImageLocalTag(fromImage); + return await _imageDigests.GetLocalImageDigestAsync(localImage, _isDryRun); + } + + string registryImage = _imageNames.GetFinalStageImageNameForDigestQuery(target.Platform); + try + { + return await _imageDigests.GetManifestDigestShaAsync(registryImage, _isDryRun); + } + catch (Exception ex) when (IsImageNotFoundException(ex)) + { + return null; + } + } + + private static string? GetDigestSha(string? digest) => + string.IsNullOrWhiteSpace(digest) + ? null + : DockerHelper.GetDigestSha(digest); + + private static string Display(string? value) => value ?? ""; + + private static bool IsImageNotFoundException(Exception ex) => + ex is HttpRequestException { StatusCode: HttpStatusCode.NotFound } || + ex is RequestFailedException { Status: 404 }; +} diff --git a/src/ImageBuilder/Build/ImageDigestCache.cs b/src/ImageBuilder/Build/ImageDigestCache.cs new file mode 100644 index 000000000..858ecc504 --- /dev/null +++ b/src/ImageBuilder/Build/ImageDigestCache.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.ImageBuilder.Build; + +/// +/// Memoizes image digest lookups made while calculating and executing a build plan. +/// +public sealed class ImageDigestCache(Lazy manifestService) +{ + private readonly Lazy _inner = manifestService; + private readonly Dictionary _localDigestCache = []; + private readonly Dictionary _manifestDigestCache = []; + private readonly SemaphoreSlim _localDigestCacheLock = new(1); + private readonly SemaphoreSlim _manifestDigestCacheLock = new(1); + + public void AddDigest(string tag, string digest) + { + _localDigestCacheLock.Wait(); + try + { + _localDigestCache[tag] = digest; + } + finally + { + _localDigestCacheLock.Release(); + } + } + + public Task GetLocalImageDigestAsync(string tag, bool isDryRun) => + LockHelper.DoubleCheckedLockLookupAsync(_localDigestCacheLock, _localDigestCache, tag, + () => _inner.Value.GetLocalImageDigestAsync(tag, isDryRun), + // Don't allow null digests to be cached. A locally built image won't have a digest until + // it is pushed so if its digest is retrieved before pushing, we don't want that + // null to be cached. + val => !string.IsNullOrEmpty(val)); + + public Task GetManifestDigestShaAsync(string tag, bool isDryRun) => + LockHelper.DoubleCheckedLockLookupAsync(_manifestDigestCacheLock, _manifestDigestCache, tag, + () => _inner.Value.GetManifestDigestShaAsync(tag, isDryRun)); +} diff --git a/src/ImageBuilder/Commands/BuildCommand.cs b/src/ImageBuilder/Commands/BuildCommand.cs index 5ae85c30c..c418709cc 100644 --- a/src/ImageBuilder/Commands/BuildCommand.cs +++ b/src/ImageBuilder/Commands/BuildCommand.cs @@ -10,6 +10,7 @@ using System.Threading; using System.Threading.Tasks; using Azure.Core; +using Microsoft.DotNet.ImageBuilder.Build; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.ViewModel; @@ -25,7 +26,7 @@ public class BuildCommand : ManifestCommand private readonly Lazy _manifestService; private readonly IRegistryCredentialsProvider _registryCredentialsProvider; private readonly IAzureTokenCredentialProvider _tokenCredentialProvider; - private readonly IImageCacheService _imageCacheService; + private readonly BuildPlanner _buildPlanner; private readonly ImageDigestCache _imageDigestCache; private readonly List _processedTags = new List(); private readonly HashSet _builtPlatforms = new(); @@ -38,7 +39,9 @@ public class BuildCommand : ManifestCommand /// private readonly Dictionary _sourceDigestCopyLocationMapping = new(); + private BuildGraph? _buildGraph; private ImageArtifactDetails? _imageArtifactDetails; + private bool _hasPublishedImagesToUse; public BuildCommand( IManifestJsonService manifestJsonService, @@ -50,7 +53,7 @@ public BuildCommand( IManifestServiceFactory manifestServiceFactory, IRegistryCredentialsProvider registryCredentialsProvider, IAzureTokenCredentialProvider tokenCredentialProvider, - IImageCacheService imageCacheService) : base(manifestJsonService) + BuildPlanner buildPlanner) : base(manifestJsonService) { _dockerService = new DockerServiceCache(dockerService ?? throw new ArgumentNullException(nameof(dockerService))); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -59,7 +62,7 @@ public BuildCommand( _copyImageService = copyImageService ?? throw new ArgumentNullException(nameof(copyImageService)); _registryCredentialsProvider = registryCredentialsProvider ?? throw new ArgumentNullException(nameof(registryCredentialsProvider)); _tokenCredentialProvider = tokenCredentialProvider ?? throw new ArgumentNullException(nameof(tokenCredentialProvider)); - _imageCacheService = imageCacheService ?? throw new ArgumentNullException(nameof(imageCacheService)); + _buildPlanner = buildPlanner ?? throw new ArgumentNullException(nameof(buildPlanner)); // Lazily create services which need access to options ArgumentNullException.ThrowIfNull(manifestServiceFactory); @@ -100,10 +103,22 @@ public override async Task ExecuteAsync() _imageArtifactDetails = new ImageArtifactDetails(); } - await ExecuteWithDockerCredentialsAsync(PullBaseImagesAsync); - await BuildImagesAsync(); + ImageArtifactDetails? publishedImages = Options.ImageInfoSourcePath is null + ? null + : ImageInfoHelper.LoadFromFile( + Options.ImageInfoSourcePath, + Manifest, + skipManifestValidation: true); + + _buildGraph = BuildGraph.CreateFiltered(Manifest); + + await ExecuteWithDockerCredentialsAsync(() => PullBaseImagesAsync(_buildGraph)); + + BuildPlanItem[] plan = await CreateBuildPlanAsync(_buildGraph, publishedImages); + + await BuildImagesAsync(plan); - if (_processedTags.Count > 0 || _imageCacheService.HasAnyCachedPlatforms) + if (_processedTags.Count > 0 || _hasPublishedImagesToUse) { // Log in again to refresh token as it may have expired from a long build await ExecuteWithDockerCredentialsAsync(async () => @@ -161,19 +176,11 @@ private async Task PublishImageInfoAsync() throw new InvalidOperationException("Source repo URL must be provided when outputting to an image info file."); } - Dictionary platformDataByTag = new Dictionary(); - foreach (PlatformData platformData in GetProcessedPlatforms()) - { - if (platformData.PlatformInfo is not null) - { - foreach (TagInfo tag in platformData.PlatformInfo.Tags) - { - platformDataByTag.Add(tag.FullyQualifiedName, platformData); - } - } - } - IEnumerable processedPlatforms = GetProcessedPlatforms(); + Dictionary platformDataByPlatform = processedPlatforms + .Where(platform => platform.PlatformInfo is not null) + .ToDictionary(platform => platform.PlatformInfo!); + List platformsWithNoPushTags = new List(); foreach (PlatformData platform in processedPlatforms) @@ -185,7 +192,7 @@ private async Task PublishImageInfoAsync() if (Options.IsPushEnabled) { await SetPlatformDataDigestAsync(platform, tag.FullyQualifiedName); - SetPlatformDataBaseDigest(platform, platformDataByTag); + SetPlatformDataBaseDigest(platform, platformDataByPlatform); await SetPlatformDataLayersAsync(platform, tag.FullyQualifiedName); } @@ -235,12 +242,22 @@ private void SetPlatformDataCreatedDate(PlatformData platform, string tag) platform.Created = createdDate; } - private void SetPlatformDataBaseDigest(PlatformData platform, Dictionary platformDataByTag) + private void SetPlatformDataBaseDigest( + PlatformData platform, + IReadOnlyDictionary platformDataByPlatform) { string? baseImageDigest = platform.BaseImageDigest; if (platform.BaseImageDigest is null && platform.PlatformInfo?.FinalStageFromImage is not null) { - if (!platformDataByTag.TryGetValue(platform.PlatformInfo.FinalStageFromImage, out PlatformData? basePlatformData)) + BuildGraph graph = _buildGraph ?? throw new InvalidOperationException("Build graph has not been created."); + BuildTarget target = graph.Targets.First(target => target.Platform == platform.PlatformInfo); + + BuildTarget? parent = graph.Parents[target].SingleOrDefault( + parent => parent.Platform.Tags + .Concat(parent.Image.SharedTags) + .Any(tag => tag.FullyQualifiedName == platform.PlatformInfo.FinalStageFromImage)); + + if (parent is null || !platformDataByPlatform.TryGetValue(parent.Platform, out PlatformData? basePlatformData)) { throw new InvalidOperationException( $"Unable to find platform data for tag '{platform.PlatformInfo.FinalStageFromImage}'. " + @@ -300,30 +317,56 @@ private async Task SetPlatformDataDigestAsync(PlatformData platform, string tag) platform.Digest = digest; } - private async Task BuildImagesAsync() + private Task CreateBuildPlanAsync(BuildGraph graph, ImageArtifactDetails? publishedImages) + { + IBuildPolicy policy = Options.NoCache + ? new AlwaysBuildPolicy() + : CommonBuildPolicies.CreateForCachedImages( + defaultResult: new BuildPolicyResult( + BuildAction.UsePublishedImage, + new BuildReason( + "All checks passed, so this invocation will use the published image.")), + logger: _logger, + baseImagePolicy: BaseImageChangedPolicy.FromLocalImages( + _imageDigestCache, + _imageNameResolver.Value, + Options.IsDryRun), + gitService: _gitService, + sourceRepoUrl: Options.SourceRepoUrl ?? string.Empty); + + return _buildPlanner.CreatePlanAsync(graph, publishedImages, policy); + } + + private async Task BuildImagesAsync(IEnumerable plan) { _logger.LogInformation("BUILDING IMAGES"); - ImageArtifactDetails? srcImageArtifactDetails = null; - if (Options.ImageInfoSourcePath != null) - { - srcImageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoSourcePath, Manifest, skipManifestValidation: true); - } + BuildPlanItem[] executableItems = plan + .Where(item => item.Decision.Action != BuildAction.NoAction) + .ToArray(); + + _hasPublishedImagesToUse = executableItems.Any( + item => item.Decision.Action + is BuildAction.UsePublishedImage + or BuildAction.PublishExistingImage); - foreach (RepoInfo repoInfo in Manifest.FilteredRepos) + var repoPlans = executableItems.GroupBy(item => item.Target.Repo); + foreach (IGrouping repoPlan in repoPlans) { + RepoInfo repoInfo = repoPlan.Key; RepoData repoData = CreateRepoData(repoInfo); - RepoData? srcRepoData = srcImageArtifactDetails?.Repos.FirstOrDefault(srcRepo => srcRepo.Repo == repoInfo.Name); - foreach (ImageInfo image in repoInfo.FilteredImages) + var imagePlans = repoPlan.GroupBy(item => item.Target.Image); + foreach (IGrouping imagePlan in imagePlans) { + ImageInfo image = imagePlan.Key; ImageData imageData = CreateImageData(image); repoData.Images.Add(imageData); - ImageData? srcImageData = srcRepoData?.Images.FirstOrDefault(srcImage => srcImage.ManifestImage == image); - - foreach (PlatformInfo platform in image.FilteredPlatforms) + foreach (BuildPlanItem plannedImage in imagePlan) { + PlatformInfo platform = plannedImage.Target.Platform; + // Tag the built images with the shared tags as well as the platform tags. // Some tests and image FROM instructions depend on these tags. @@ -338,41 +381,30 @@ private async Task BuildImagesAsync() PlatformData platformData = CreatePlatformData(image, platform); imageData.Platforms.Add(platformData); - bool isCachedImage = false; - if (!Options.NoCache) + if (plannedImage.Decision.Action is BuildAction.UsePublishedImage or BuildAction.PublishExistingImage) { - ImageCacheResult cacheResult = await _imageCacheService.CheckForCachedImageAsync( - srcImageData, - platformData, - _imageDigestCache, - _imageNameResolver.Value, - sourceRepoUrl: Options.SourceRepoUrl, - isLocalBaseImageExpected: true, - isDryRun: Options.IsDryRun); - - if (cacheResult.State.HasFlag(ImageCacheState.Cached)) - { - isCachedImage = true; + PublishedImage publishedImage = plannedImage.PublishedImage ?? + throw new InvalidOperationException( + $"Build plan did not provide reusable metadata for '{platform.DockerfilePath}'."); - CopyPlatformDataFromCachedPlatform(platformData, cacheResult.Platform!); - platformData.IsUnchanged = cacheResult.State != ImageCacheState.CachedWithMissingTags; + CopyPlatformDataFromCachedPlatform(platformData, publishedImage.Image); + platformData.IsUnchanged = plannedImage.Decision.Action == BuildAction.UsePublishedImage; - await OnCacheHitAsync(repoInfo, allTagInfos, pullImage: cacheResult.IsNewCacheHit, cacheResult.Platform!.Digest); - } + await UsePublishedImageAsync(repoInfo, allTagInfos, publishedImage.Image.Digest); } - - if (!isCachedImage) + else if (plannedImage.Decision.Action == BuildAction.BuildImage) { _processedTags.AddRange(allTagInfos); - BuildImage(platform, allTags); + BuildImage(plannedImage.Target, allTags); _builtPlatforms.Add(platformData); if (Options.IsPushEnabled && platform.FinalStageFromImage is not null) { platformData.BaseImageDigest = - await _imageDigestCache.GetLocalImageDigestAsync( - _imageNameResolver.Value.GetFromImageLocalTag(platform.FinalStageFromImage), Options.IsDryRun); + await _imageDigestCache.GetLocalImageDigestAsync( + _imageNameResolver.Value.GetFromImageLocalTag(platform.FinalStageFromImage), + Options.IsDryRun); } } } @@ -385,23 +417,23 @@ await _imageDigestCache.GetLocalImageDigestAsync( } } - private void CopyPlatformDataFromCachedPlatform(PlatformData dstPlatform, PlatformData srcPlatform) + private void CopyPlatformDataFromCachedPlatform(PlatformData destination, PlatformData source) { // When a cache hit occurs for a Dockerfile, we want to transfer some of the metadata about the previously // published image so we don't need to recalculate it again. - dstPlatform.BaseImageDigest = srcPlatform.BaseImageDigest; - dstPlatform.Layers = new List(srcPlatform.Layers); + destination.BaseImageDigest = source.BaseImageDigest; + destination.Layers = [.. source.Layers]; } - private RepoData CreateRepoData(RepoInfo repoInfo) => - new RepoData - { - Repo = repoInfo.Name - }; + private RepoData CreateRepoData(RepoInfo repoInfo) => new RepoData + { + Repo = repoInfo.Name + }; private PlatformData CreatePlatformData(ImageInfo image, PlatformInfo platform) { PlatformData platformData = PlatformData.FromPlatformInfo(platform, image); + platformData.SimpleTags = platform.Tags .Select(tag => tag.Name) .OrderBy(name => name) @@ -412,11 +444,10 @@ private PlatformData CreatePlatformData(ImageInfo image, PlatformInfo platform) private ImageData CreateImageData(ImageInfo image) { - ImageData imageData = - new ImageData - { - ProductVersion = image.ProductVersion - }; + var imageData = new ImageData + { + ProductVersion = image.ProductVersion + }; if (image.SharedTags.Any()) { @@ -479,11 +510,16 @@ private void ValidatePlatformIsCompatibleWithBaseImage(PlatformInfo platform) } } - private void BuildImage(PlatformInfo platform, IEnumerable allTags) + private void BuildImage( + BuildTarget target, + IEnumerable allTags) { + PlatformInfo platform = target.Platform; ValidatePlatformIsCompatibleWithBaseImage(platform); - bool createdPrivateDockerfile = UpdateDockerfileFromCommands(platform, out string dockerfilePath); + bool createdPrivateDockerfile = UpdateDockerfileFromCommands( + target, + out string dockerfilePath); try { @@ -558,10 +594,13 @@ private void BuildImage(PlatformInfo platform, IEnumerable allTags) private IEnumerable GetDockerBuildOptions() => Options.DockerBuildOptions.Where(option => !string.IsNullOrWhiteSpace(option)); - private async Task OnCacheHitAsync(RepoInfo repo, IEnumerable allTags, bool pullImage, string sourceDigest) + private async Task UsePublishedImageAsync( + RepoInfo repo, + IEnumerable allTags, + string sourceDigest) { _logger.LogInformation(string.Empty); - _logger.LogInformation("CACHE HIT"); + _logger.LogInformation("USING PUBLISHED IMAGE"); _logger.LogInformation(string.Empty); // When a cache hit occurs on an image, we copy the image from its source location (e.g. mcr.microsoft.com) to its @@ -571,13 +610,13 @@ private async Task OnCacheHitAsync(RepoInfo repo, IEnumerable allTags, // The pulled image is then tagged with the same tags it would be tagged with had it been built locally. This allows // dependent Dockerfiles that reference those tags to seamlessly consume the pulled image. + bool pullImage = !_sourceDigestCopyLocationMapping.ContainsKey(sourceDigest); string copiedSourceDigest = sourceDigest; if (Options.IsPushEnabled) { copiedSourceDigest = await CopyCachedImage(allTags, sourceDigest); } - // Pull the image instead of building it if (pullImage) { await ExecuteWithDockerCredentialsAsync(() => @@ -635,7 +674,7 @@ await _copyImageService.ImportImageAsync( return sourceDigest; } - private async Task PullBaseImagesAsync() + private async Task PullBaseImagesAsync(BuildGraph graph) { _logger.LogInformation("PULLING LATEST BASE IMAGES"); @@ -647,7 +686,9 @@ private async Task PullBaseImagesAsync() HashSet pulledTags = []; HashSet externalFromImages = []; - foreach (PlatformInfo platform in Manifest.GetFilteredPlatforms()) + PlatformInfo[] platforms = graph.Targets.Select(target => target.Platform) .ToArray(); + + foreach (PlatformInfo platform in platforms) { IEnumerable platformExternalFromImages = platform.ExternalFromImages.Distinct(); externalFromImages.UnionWith(platformExternalFromImages); @@ -672,7 +713,7 @@ private async Task PullBaseImagesAsync() } IEnumerable finalStageExternalFromImages = - Manifest.GetFilteredPlatforms() + platforms .Where(platform => platform.FinalStageFromImage is not null && !platform.IsInternalFromImage(platform.FinalStageFromImage)) @@ -727,8 +768,11 @@ private void PushImages() } } - private bool UpdateDockerfileFromCommands(PlatformInfo platform, out string dockerfilePath) + private bool UpdateDockerfileFromCommands( + BuildTarget target, + out string dockerfilePath) { + PlatformInfo platform = target.Platform; bool updateDockerfile = false; dockerfilePath = platform.DockerfilePath; @@ -739,9 +783,7 @@ private bool UpdateDockerfileFromCommands(PlatformInfo platform, out string dock foreach (string fromImage in platform.OverriddenFromImages) { - string fromRepo = DockerHelper.GetRepo(fromImage); - RepoInfo repo = Manifest.FilteredRepos.First(r => r.FullModelName == fromRepo); - string newFromImage = DockerHelper.ReplaceRepo(fromImage, repo.QualifiedName); + string newFromImage = target.FromImageOverrides[fromImage]; _logger.LogInformation($"Replacing FROM `{fromImage}` with `{newFromImage}`"); Regex fromRegex = new Regex($@"FROM\s+{Regex.Escape(fromImage)}[^\s\r\n]*"); dockerfileContents = fromRegex.Replace(dockerfileContents, $"FROM {newFromImage}"); diff --git a/src/ImageBuilder/Commands/GenerateBuildMatrixCommand.cs b/src/ImageBuilder/Commands/GenerateBuildMatrixCommand.cs index 53918306d..de5c72f30 100644 --- a/src/ImageBuilder/Commands/GenerateBuildMatrixCommand.cs +++ b/src/ImageBuilder/Commands/GenerateBuildMatrixCommand.cs @@ -3,11 +3,11 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Microsoft.DotNet.ImageBuilder.Build; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.Models.Manifest; using Microsoft.DotNet.ImageBuilder.ViewModel; @@ -21,14 +21,22 @@ public class GenerateBuildMatrixCommand : ManifestCommand _imageArtifactDetails; private static readonly char[] s_pathSeparators = { '/', '\\' }; private static readonly Regex s_versionRegex = new(@$"^(?<{VersionRegGroupName}>(\d|\.)+).*$"); - private readonly IImageCacheService _imageCacheService; + private readonly BuildPlanner _buildPlanner; + private readonly IGitService _gitService; private readonly ILogger _logger; private readonly ImageDigestCache _imageDigestCache; private readonly Lazy _imageNameResolver; - - public GenerateBuildMatrixCommand(IManifestJsonService manifestJsonService, IImageCacheService imageCacheService, IManifestServiceFactory manifestServiceFactory, ILogger logger) : base(manifestJsonService) + private BuildGraph? _dependencyGraph; + + public GenerateBuildMatrixCommand( + IManifestJsonService manifestJsonService, + BuildPlanner buildPlanner, + IGitService gitService, + IManifestServiceFactory manifestServiceFactory, + ILogger logger) : base(manifestJsonService) { - _imageCacheService = imageCacheService ?? throw new ArgumentNullException(nameof(imageCacheService)); + _buildPlanner = buildPlanner ?? throw new ArgumentNullException(nameof(buildPlanner)); + _gitService = gitService ?? throw new ArgumentNullException(nameof(gitService)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _imageArtifactDetails = new Lazy(() => { @@ -104,7 +112,7 @@ private void AddDockerfilePathLegs( { // Pass 1: Find direct dependencies from the Dockerfile's FROM statement IEnumerable> subgraphs = platformGrouping.GetCompleteSubgraphs( - platform => Manifest.GetParents(platform, platformGrouping)); + platform => GetParents(platform, platformGrouping)); // Pass 2: Combine subgraphs that have a common Dockerfile path for the root image subgraphs = ConsolidateSubgraphs(subgraphs, platform => platform.DockerfilePath); @@ -198,9 +206,12 @@ private IEnumerable GetCustomLegGroupPlatforms(PlatformInfo platfo { IEnumerable dependencyPlatforms = group.Dependencies .Select(dependency => Manifest.GetPlatformByTag(dependency)); - return dependencyPlatforms - .Concat(dependencyPlatforms - .SelectMany(dependencyPlatform => Manifest.GetAncestors(dependencyPlatform, Manifest.GetFilteredPlatforms()))); + + return dependencyPlatforms.Concat( + dependencyPlatforms.SelectMany( + dependencyPlatform => GetAncestors(dependencyPlatform, Manifest.GetFilteredPlatforms()) + ) + ); }) .Distinct(); } @@ -279,7 +290,7 @@ private void AddVersionedOsLegs(BuildMatrixInfo matrix, // of platforms in Pass 2 to only test what was already built. IEnumerable> subgraphs = platformGrouping .GetCompleteSubgraphs(platform => - Manifest.GetParents(platform, allPlatforms) + GetParents(platform, allPlatforms) .Union(GetCustomLegGroupPlatforms(platform, CustomBuildLegDependencyType.Integral))); // Pass 2: Filter subgraphs to only images that are in the current platform group @@ -297,7 +308,7 @@ private void AddVersionedOsLegs(BuildMatrixInfo matrix, // Pass 5: Append the parent graph of each platform to each respective subgraph subgraphs = subgraphs.GetCompleteSubgraphs( - subgraph => subgraph.Select(platform => Manifest.GetAncestors(platform, platformGrouping))) + subgraph => subgraph.Select(platform => GetAncestors(platform, platformGrouping))) .Select(set => set .SelectMany(subgraph => subgraph) .Distinct()) @@ -407,97 +418,93 @@ private static string FormatMatrixName(IEnumerable parts) return allParts.First() + string.Join(string.Empty, allParts.Skip(1).Select(part => part.FirstCharToUpper())); } - private async Task> GetPlatformsAsync() + /// + /// Selects the manifest platforms that require build matrix legs. + /// + private async Task> GetPlatformsToBuildAsync() { - IEnumerable filteredRepos = Manifest.FilteredRepos.ToList(); + // Matrix grouping still needs dependency relationships from the complete manifest. + _dependencyGraph = BuildGraph.Create(Manifest); + ImageArtifactDetails? imageInfo = _imageArtifactDetails.Value; + + // Do not schedule platforms that an earlier build already completed unchanged. + HashSet completedPlatforms = imageInfo?.Repos + .SelectMany(repo => repo.Images) + .SelectMany(image => image.Platforms) + .Where(platform => platform.IsUnchanged && platform.PlatformInfo is not null) + .Select(platform => platform.PlatformInfo!) + .ToHashSet() + ?? []; + + // Build a second, filtered dependency graph... + BuildGraph graph = BuildGraph.CreateFiltered(Manifest, platform => !completedPlatforms.Contains(platform)); + + // ...and then check which images actually need to be built. + IBuildPolicy policy = Options.TrimCachedImages + ? CommonBuildPolicies.CreateForCachedImages( + defaultResult: new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("All checks passed, so no work is required.")), + logger: _logger, + baseImagePolicy: BaseImageChangedPolicy.FromRegistry( + _imageDigestCache, + _imageNameResolver.Value, + Options.IsDryRun), + gitService: _gitService, + sourceRepoUrl: Options.SourceRepoUrl ?? string.Empty) + : new AlwaysBuildPolicy(); - if (_imageArtifactDetails.Value is null) - { - return filteredRepos.SelectMany(repo => repo.FilteredImages).SelectMany(image => image.FilteredPlatforms); - } + BuildPlanItem[] plan = await _buildPlanner.CreatePlanAsync(graph, imageInfo, policy); - IEnumerable<(PlatformInfo PlatformInfo, ImageData? ImageData, PlatformData? PlatformData)> platformMappings = - filteredRepos.SelectMany(repo => - repo.FilteredImages - .SelectMany(image => image.FilteredPlatforms) - .Select(platform => - { - (PlatformData Platform, ImageData Image)? matchingPlatform = ImageInfoHelper.GetMatchingPlatformData(platform, repo, _imageArtifactDetails.Value); - return (platform, matchingPlatform?.Image, matchingPlatform?.Platform); - }) - .Where(platformMapping => platformMapping.Platform is null || !platformMapping.Platform.IsUnchanged)); + IEnumerable plannedPlatforms = plan + .Where(item => item.Decision.Action != BuildAction.NoAction) + .Select(item => item.Target.Platform); - if (!Options.TrimCachedImages) - { - return platformMappings.Select(platformMapping => platformMapping.PlatformInfo); - } + return Options.TrimCachedImages + ? plannedPlatforms.OrderBy(platform => platform.DockerfilePath) + : plannedPlatforms; + } - _logger.LogInformation("Trimming platforms based on image cache state..."); - - // Here we will trim the platforms based on their image cache state. This reduces the amount of jobs that need to - // be run. Otherwise, you may spin up a bunch of jobs that end up processing a bunch of cached images and - // essentially becomes a no-op. - - // We need to group the platforms according to their parent dependency hierarchy. This is important because we must - // treat the hierarchy as a unit. For example, if runtime-deps is cached but runtime (which is a descendant of - // runtime-deps) is not, then we need to ensure that both runtime-deps and runtime is included. We do not want to - // trim just the runtime-deps platform in that case. - IEnumerable> subgraphs = - platformMappings.GetCompleteSubgraphs( - platformGrouping => - Manifest.GetParents( - platformGrouping.PlatformInfo, - platformMappings.Select(m => m.PlatformInfo) - ).Select(platformInfo => platformMappings.First(mapping => mapping.PlatformInfo == platformInfo))); - - ConcurrentBag nonCachedPlatforms = []; - await Parallel.ForEachAsync(subgraphs, async (subgraph, _) => + private IEnumerable GetParents( + PlatformInfo platform, + IEnumerable availablePlatforms) + { + if (_dependencyGraph is null) { - ConcurrentBag subgraphNonCachedPlatforms = []; - await Parallel.ForEachAsync(subgraph, async (platformMapping, _) => - { - if (platformMapping.PlatformData is null) - { - _logger.LogInformation($"Image info not found for '{platformMapping.PlatformInfo.DockerfilePath}'. Including path in matrix."); - subgraphNonCachedPlatforms.Add(platformMapping.PlatformInfo); - return; - } + throw new InvalidOperationException( + "The dependency graph must be created before generating matrix legs."); + } - ImageCacheResult cacheResult = await _imageCacheService.CheckForCachedImageAsync( - platformMapping.ImageData, - platformMapping.PlatformData, - _imageDigestCache, - _imageNameResolver.Value, - Options.SourceRepoUrl, - isLocalBaseImageExpected: false, - Options.IsDryRun); + HashSet available = availablePlatforms.ToHashSet(); + BuildTarget target = _dependencyGraph.Targets.First( + target => target.Platform == platform); + return _dependencyGraph.Parents[target] + .Select(parent => parent.Platform) + .Where(available.Contains); + } - bool includePlatformInMatrix = !cacheResult.State.HasFlag(ImageCacheState.Cached); + private IEnumerable GetAncestors( + PlatformInfo platform, + IEnumerable availablePlatforms) + { + HashSet available = availablePlatforms.ToHashSet(); + HashSet ancestors = []; + Queue remaining = new(GetParents(platform, available)); - _logger.LogInformation( - $"Image '{platformMapping.PlatformInfo.DockerfilePath}' cache state is {cacheResult.State}. Included in matrix: {includePlatformInMatrix}"); + while (remaining.TryDequeue(out PlatformInfo? ancestor)) + { + if (!ancestors.Add(ancestor)) + { + continue; + } - if (includePlatformInMatrix) - { - subgraphNonCachedPlatforms.Add(platformMapping.PlatformInfo); - } - }); - - // As mentioned above, we need to treat the hierarchy as a unit so even though a subset of the platforms - // in the hierarchy may be cached, they all need to be included. Only in the case where they're all - // cached, should they be excluded. To determine what needs to be included, it can be simplified to just - // check whether there are any platforms identified within the hierarchy as not being cached. If so, then - // include the whole hierarchy as non-cached platforms. - if (!subgraphNonCachedPlatforms.IsEmpty) + foreach (PlatformInfo parent in GetParents(ancestor, available)) { - foreach ((PlatformInfo PlatformInfo, ImageData? ImageData, PlatformData? PlatformData) platformMapping in subgraph) - { - nonCachedPlatforms.Add(platformMapping.PlatformInfo); - } + remaining.Enqueue(parent); } - }); + } - return nonCachedPlatforms.OrderBy(platform => platform.DockerfilePath); + return ancestors; } public async Task> GenerateMatrixInfoAsync() @@ -505,7 +512,8 @@ public async Task> GenerateMatrixInfoAsync() List matrices = []; // The sort order used here is arbitrary and simply helps the readability of the output. - IOrderedEnumerable> platformGroups = (await GetPlatformsAsync()) + IOrderedEnumerable> platformGroups = + (await GetPlatformsToBuildAsync()) .GroupBy(platform => CreatePlatformId(platform)) .OrderBy(platformGroup => platformGroup.Key.OS) .ThenByDescending(platformGroup => platformGroup.Key.OsVersion) diff --git a/src/ImageBuilder/Commands/GetStaleImagesCommand.cs b/src/ImageBuilder/Commands/GetStaleImagesCommand.cs index a21866026..fded3b761 100644 --- a/src/ImageBuilder/Commands/GetStaleImagesCommand.cs +++ b/src/ImageBuilder/Commands/GetStaleImagesCommand.cs @@ -5,8 +5,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.ImageBuilder.Build; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.ViewModel; using Newtonsoft.Json; @@ -16,8 +16,8 @@ namespace Microsoft.DotNet.ImageBuilder.Commands { public class GetStaleImagesCommand : Command { - private readonly Dictionary _imageDigests = new(); - private readonly SemaphoreSlim _imageDigestsLock = new(1); + private readonly BuildPlanner _buildPlanner; + private readonly ImageDigestCache _imageDigestCache; private readonly Lazy _manifestService; private readonly IManifestJsonService _manifestJsonService; private readonly ILogger _logger; @@ -29,12 +29,14 @@ public GetStaleImagesCommand( IManifestJsonService manifestJsonService, ILogger logger, IOctokitClientFactory octokitClientFactory, - IGitService gitService) + IGitService gitService, + BuildPlanner buildPlanner) { _manifestJsonService = manifestJsonService ?? throw new ArgumentNullException(nameof(manifestJsonService)); _logger = logger; _octokitClientFactory = octokitClientFactory; _gitService = gitService; + _buildPlanner = buildPlanner ?? throw new ArgumentNullException(nameof(buildPlanner)); // Don't worry about authenticating to our own ACR, since we are checking base image digests from public // registries instead of our staging location. Registry credentials are needed however to prevent rate @@ -42,6 +44,7 @@ public GetStaleImagesCommand( ArgumentNullException.ThrowIfNull(manifestServiceFactory); _manifestService = new Lazy(() => manifestServiceFactory.Create(Options.CredentialsOptions)); + _imageDigestCache = new ImageDigestCache(_manifestService); } protected override string Description => "Gets paths to images whose base images are out-of-date"; @@ -97,110 +100,33 @@ private async Task> GetPathsToRebuildAsync(Models.Subscripti repoPrefix: null, sourceRepoPrefix: Options.SourceRepoPrefix); - List pathsToRebuild = new(); - - foreach (RepoInfo repo in manifest.FilteredRepos) - { - IEnumerable platforms = repo.FilteredImages - .SelectMany(image => image.FilteredPlatforms) - .Where(platform => platform.FinalStageFromImage is not null && !platform.IsInternalFromImage(platform.FinalStageFromImage)); - - foreach (PlatformInfo platform in platforms) - { - pathsToRebuild.AddRange( - await GetPathsToRebuildAsync(manifest, platform, repo, imageArtifactDetails, imageNameResolver)); - } - } - - return pathsToRebuild.Distinct().ToList(); - } - - private static IEnumerable GetDescendants(PlatformInfo platform, ManifestInfo manifest) => - manifest.GetDescendants(platform, manifest.GetAllPlatforms().ToList(), includeAncestorsOfDescendants: true) - .Prepend(platform); - - private async Task> GetPathsToRebuildAsync( - ManifestInfo manifest, - PlatformInfo platform, - RepoInfo repo, - ImageArtifactDetails imageArtifactDetails, - ImageNameResolverForMatrix imageNameResolver) - { - string? fromImage = platform.FinalStageFromImage; - if (fromImage is null) - { - _logger.LogInformation( - "Dockerfile {DockerfilePath} has no base image. It is automatically considered up-to-date.", - platform.DockerfilePath); - - return []; - } - - (PlatformData Platform, ImageData Image)? matchingPlatform = - ImageInfoHelper.GetMatchingPlatformData(platform, repo, imageArtifactDetails); - - if (matchingPlatform is null) - { - _logger.LogWarning( - "Image info not found for '{DockerfilePath}'. It will be queued for rebuild.", - platform.DockerfilePath); - - IEnumerable dependentPlatforms = GetDescendants(platform, manifest); - return dependentPlatforms.Select(p => p.Model.Dockerfile).ToList(); - } - - // Resolve where to actually fetch the digest from. For external base images this - // points to the mirror location in the staging registry; for internal images it is the - // original FROM tag. The "public" form is the canonical reference matching what gets - // recorded in image-info.json and so is the right repo to use in the digest comparison - // string below. - string baseImagePullReference = imageNameResolver.GetFromImagePullTag(fromImage); - string baseImagePublicReference = imageNameResolver.GetFromImagePublicTag(fromImage); - - // Cache the manifest digest by pull reference. The digest is a function of where we - // actually pull bytes from, so the pull reference is the correct cache key. - string baseImageManifestDigest = - await LockHelper.DoubleCheckedLockLookupAsync( - semaphore: _imageDigestsLock, - dictionary: _imageDigests, - key: baseImagePullReference, - getValue: () => - // This reaches out to the registry to fetch the digest from the pull - // reference. For external images, this fetches from the mirror. - _manifestService.Value.GetManifestDigestShaAsync(baseImagePullReference, Options.IsDryRun)); - - // Build a digest-pinned reference of the form '@sha256:' (e.g. - // 'mcr.microsoft.com/dotnet/runtime@sha256:abc123...'). This must be built per-call - // from this platform's own public reference — two FROM spellings (e.g. 'almalinux:8' - // vs 'library/almalinux:8') can share a pull reference but resolve to different - // public references, so the formed string cannot be cached or shared across platforms. - // The shape matches what's stored in Platform.BaseImageDigest so the equality check - // below is meaningful. - string currentBaseImageDigestReference = - DockerHelper.GetDigestString( - repo: DockerHelper.GetRepo(baseImagePublicReference), - sha: baseImageManifestDigest); - - bool shouldRebuildImage = matchingPlatform.Value.Platform.BaseImageDigest != currentBaseImageDigestReference; - - _logger.LogInformation( - "Dockerfile {DockerfilePath} was last built with base image {BaseImagePublicReference} at digest" - + " {LastBuildBaseImageDigestReference}. Image {BaseImagePullReference} has current digest" - + " {CurrentBaseImageDigestReference}. Up to date: {IsUpToDate}.", - platform.DockerfilePath, - baseImagePublicReference, - matchingPlatform.Value.Platform.BaseImageDigest, - baseImagePullReference, - currentBaseImageDigestReference, - !shouldRebuildImage); - - if (shouldRebuildImage) - { - IEnumerable dependentPlatforms = GetDescendants(platform, manifest); - return dependentPlatforms.Select(p => p.Model.Dockerfile).ToList(); - } - - return []; + BuildGraph graph = BuildGraph.CreateFiltered(manifest); + + // This command only reports images made stale by missing metadata or base image updates. + // Dockerfile and tag changes are handled by normal build planning. + IBuildPolicy policy = new CompositeBuildPolicy( + defaultResult: new BuildPolicyResult( + BuildAction.NoAction, + new BuildReason("All checks passed, so no work is required.")), + logger: _logger, + policies: + [ + // Rebuild when no published image metadata exists. + new MissingPublishedImagePolicy(), + + // Rebuild when the registry digest for the base image has changed. + BaseImageChangedPolicy.FromRegistry( + _imageDigestCache, + imageNameResolver, + Options.IsDryRun) + ]); + + BuildPlanItem[] plan = await _buildPlanner.CreatePlanAsync(graph, imageArtifactDetails, policy); + + return plan + .Where(item => item.Decision.Action != BuildAction.NoAction) + .Select(item => item.Target.Platform.Model.Dockerfile) + .Distinct(); } private async Task GetImageInfoForSubscriptionAsync(Models.Subscription.Subscription subscription, ManifestInfo manifest) diff --git a/src/ImageBuilder/ImageBuilder.cs b/src/ImageBuilder/ImageBuilder.cs index fe496a706..d4e749470 100644 --- a/src/ImageBuilder/ImageBuilder.cs +++ b/src/ImageBuilder/ImageBuilder.cs @@ -4,6 +4,7 @@ using System; using System.Threading; +using Microsoft.DotNet.ImageBuilder.Build; using Microsoft.DotNet.ImageBuilder.Commands; using Microsoft.DotNet.ImageBuilder.Commands.Signing; using Microsoft.DotNet.ImageBuilder.Configuration; @@ -96,7 +97,7 @@ public static IHost CreateAppHost() "image-builder-oras-timeout", pipeline => pipeline.AddTimeout(TimeSpan.FromSeconds(10))); - builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddMemoryCache(); diff --git a/src/ImageBuilder/ImageCacheService.cs b/src/ImageBuilder/ImageCacheService.cs deleted file mode 100644 index 7641d99a8..000000000 --- a/src/ImageBuilder/ImageCacheService.cs +++ /dev/null @@ -1,304 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using Azure; -using Microsoft.DotNet.ImageBuilder.Models.Image; -using Microsoft.DotNet.ImageBuilder.ViewModel; - -namespace Microsoft.DotNet.ImageBuilder; - -/// -/// Service for determining whether a Docker image can be retrieved from a cache -/// rather than being rebuilt. -/// -public interface IImageCacheService -{ - /// - /// Gets whether any platforms have been identified as cached during this session. - /// - bool HasAnyCachedPlatforms { get; } - - /// - /// Checks whether a previously built image can be reused from cache. - /// - /// Image data from the source image-info file, if available. - /// Platform data for the image being checked. - /// Cache for looking up image digests from registries. - /// Resolver for constructing image names for digest queries. - /// URL of the source repository containing the Dockerfiles. - /// Whether the base image is expected to exist locally rather than in a remote registry. - /// Whether this is a dry run that should skip actual registry calls. - Task CheckForCachedImageAsync( - ImageData? srcImageData, - PlatformData platformData, - ImageDigestCache imageDigestCache, - ImageNameResolver imageNameResolver, - string? sourceRepoUrl, - bool isLocalBaseImageExpected, - bool isDryRun); -} - -/// -public class ImageCacheService : IImageCacheService -{ - private readonly ILogger _logger; - private readonly IGitService _gitService; - - private readonly object _cachedPlatformsLock = new(); - - /// - /// Metadata about Dockerfiles whose images have been retrieved from the cache. - /// Keyed by the build cache key derived from the platform's Dockerfile path and build args. - /// - private readonly Dictionary _cachedPlatforms = []; - - public ImageCacheService(ILogger logger, IGitService gitService) - { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _gitService = gitService ?? throw new ArgumentNullException(nameof(gitService)); - } - - /// - public bool HasAnyCachedPlatforms - { - get - { - lock (_cachedPlatformsLock) - { - return _cachedPlatforms.Any(); - } - } - } - - /// - public async Task CheckForCachedImageAsync( - ImageData? srcImageData, - PlatformData platformData, - ImageDigestCache imageDigestCache, - ImageNameResolver imageNameResolver, - string? sourceRepoUrl, - bool isLocalBaseImageExpected, - bool isDryRun) - { - ImageCacheState cacheState = ImageCacheState.NotCached; - bool isNewCacheHit = false; - PlatformData? srcPlatformData = srcImageData?.Platforms - .FirstOrDefault(srcPlatform => srcPlatform.PlatformInfo == platformData.PlatformInfo); - - if (platformData.PlatformInfo is null) - { - throw new Exception("Expected platform info to be set"); - } - - string cacheKey = GetBuildCacheKey(platformData.PlatformInfo); - lock (_cachedPlatformsLock) - { - if (_cachedPlatforms.TryGetValue(cacheKey, out PlatformData? cachedPlatform)) - { - cacheState = ImageCacheState.Cached; - if (srcPlatformData is null || - !CachedPlatformHasAllTagsPublished(srcPlatformData)) - { - cacheState = ImageCacheState.CachedWithMissingTags; - } - return new ImageCacheResult(cacheState, isNewCacheHit, cachedPlatform); - } - } - - // If this Dockerfile has been built and published before - if (srcPlatformData != null) - { - bool isCachedImage = await CheckForCachedImageFromImageInfoAsync( - platformData.PlatformInfo, - srcPlatformData, - imageDigestCache, - imageNameResolver, - sourceRepoUrl, - isLocalBaseImageExpected, - isDryRun); - - if (isCachedImage) - { - isNewCacheHit = true; - cacheState = ImageCacheState.Cached; - if (!CachedPlatformHasAllTagsPublished(srcPlatformData)) - { - cacheState = ImageCacheState.CachedWithMissingTags; - } - lock (_cachedPlatformsLock) - { - _cachedPlatforms[cacheKey] = srcPlatformData; - } - } - } - - return new ImageCacheResult(cacheState, isNewCacheHit, srcPlatformData); - } - - /// - /// Checks whether the source platform data has all its expected tags published. - /// - private static bool CachedPlatformHasAllTagsPublished(PlatformData srcPlatformData) => - (srcPlatformData.PlatformInfo?.Tags ?? []) - .Select(tag => tag.Name) - .AreEquivalent(srcPlatformData.SimpleTags); - - /// - /// Determines whether a previously published image can be reused by comparing the base image - /// digest and Dockerfile commit against the current state. - /// - private async Task CheckForCachedImageFromImageInfoAsync( - PlatformInfo platform, - PlatformData srcPlatformData, - ImageDigestCache imageDigestCache, - ImageNameResolver imageNameResolver, - string? sourceRepoUrl, - bool isLocalBaseImageExpected, - bool isDryRun) - { - _logger.LogInformation("Checking for cached image for '{DockerfilePath}'", platform.DockerfilePathRelativeToManifest); - - // If the previously published image was based on an image that is still the latest version AND - // the Dockerfile hasn't changed since it was last published - if (await IsBaseImageDigestUpToDateAsync( - platform, srcPlatformData, imageDigestCache, imageNameResolver, isLocalBaseImageExpected, isDryRun) && - IsDockerfileUpToDate(platform, srcPlatformData, sourceRepoUrl)) - { - return true; - } - - _logger.LogInformation("CACHE MISS"); - _logger.LogInformation(string.Empty); - - return false; - } - - /// - /// Checks whether the base image digest recorded in image-info matches the current digest - /// available from the registry. - /// - private async Task IsBaseImageDigestUpToDateAsync( - PlatformInfo platform, - PlatformData srcPlatformData, - ImageDigestCache imageDigestCache, - ImageNameResolver imageNameResolver, - bool isLocalImageExpected, - bool isDryRun) - { - _logger.LogInformation(string.Empty); - - if (platform.FinalStageFromImage is null) - { - _logger.LogInformation("Image does not have a base image. By default, it is considered up-to-date."); - return true; - } - - string queryImage = imageNameResolver.GetFinalStageImageNameForDigestQuery(platform); - - string? currentSha; - if (isLocalImageExpected) - { - currentSha = await imageDigestCache.GetLocalImageDigestAsync( - imageNameResolver.GetFromImageLocalTag(platform.FinalStageFromImage), isDryRun); - if (currentSha is not null) - { - currentSha = DockerHelper.GetDigestSha(currentSha); - } - } - else - { - try - { - currentSha = await imageDigestCache.GetManifestDigestShaAsync(queryImage, isDryRun); - } - // Handle cases where the image is not found in the registry yet. - // Other errors (e.g., authentication failures) should propagate so - // they are not silently swallowed. See https://github.com/dotnet/docker-tools/issues/1964 - catch (Exception ex) when (IsImageNotFoundException(ex)) - { - currentSha = null; - } - } - - string? imageInfoSha = srcPlatformData.BaseImageDigest is not null ? - DockerHelper.GetDigestSha(srcPlatformData.BaseImageDigest) : - null; - - bool baseImageDigestMatches = imageInfoSha?.Equals(currentSha, StringComparison.OrdinalIgnoreCase) == true; - - _logger.LogInformation("Image info's base image digest SHA: {ImageInfoSha}", imageInfoSha); - _logger.LogInformation("Latest base image digest SHA: {CurrentSha}", currentSha); - _logger.LogInformation("Base image digests match: {BaseImageDigestMatches}", baseImageDigestMatches); - return baseImageDigestMatches; - } - - /// - /// Checks whether the Dockerfile has changed since the last published build by comparing - /// the current git commit URL against the one recorded in image-info. - /// - private bool IsDockerfileUpToDate(PlatformInfo platform, PlatformData srcPlatformData, string? sourceRepoUrl) - { - string currentCommitUrl = _gitService.GetDockerfileCommitUrl(platform, sourceRepoUrl); - bool commitShaMatches = false; - if (srcPlatformData.CommitUrl is not null) - { - commitShaMatches = srcPlatformData.CommitUrl.Equals(currentCommitUrl, StringComparison.OrdinalIgnoreCase); - } - - _logger.LogInformation(string.Empty); - _logger.LogInformation("Image info's Dockerfile commit: {CommitUrl}", srcPlatformData.CommitUrl); - _logger.LogInformation("Latest Dockerfile commit: {CurrentCommitUrl}", currentCommitUrl); - _logger.LogInformation("Dockerfile commits match: {CommitShaMatches}", commitShaMatches); - return commitShaMatches; - } - - /// - /// Builds a cache key that uniquely identifies a platform build based on its Dockerfile path - /// and build arguments. - /// - private static string GetBuildCacheKey(PlatformInfo platform) => - $"{platform.DockerfilePathRelativeToManifest}-" + - string.Join('-', platform.BuildArgs.Select(kvp => $"{kvp.Key}={kvp.Value}").ToArray()); - - /// - /// Returns true if the exception represents an HTTP 404 Not Found response, - /// indicating the image does not exist in the registry. - /// - private static bool IsImageNotFoundException(Exception ex) => - (ex is HttpRequestException { StatusCode: HttpStatusCode.NotFound }) || - (ex is RequestFailedException { Status: 404 }); -} - -[Flags] -public enum ImageCacheState -{ - /// - /// Indicates a previously built image was not found in the registry. - /// - NotCached = 0, - - /// - /// Indicates a previously built image was found in the registry. - /// - Cached = 1, - - /// - /// Indicates a previously built image was found in the registry but is missing new tags. - /// - CachedWithMissingTags = Cached | 2 -} - -/// -/// The result of checking whether an image is cached. -/// -/// The cache state of the image. -/// Whether this is a newly discovered cache hit (not previously known in this session). -/// The source platform data associated with the cached image, if available. -public record ImageCacheResult(ImageCacheState State, bool IsNewCacheHit, PlatformData? Platform); diff --git a/src/ImageBuilder/ImageDigestCache.cs b/src/ImageBuilder/ImageDigestCache.cs deleted file mode 100644 index 327675576..000000000 --- a/src/ImageBuilder/ImageDigestCache.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.DotNet.ImageBuilder -{ - public class ImageDigestCache(Lazy manifestService) - { - private readonly Lazy _inner = manifestService; - private readonly Dictionary _localDigestCache = []; - private readonly Dictionary _manifestDigestCache = []; - private readonly SemaphoreSlim _localDigestCacheLock = new(1); - private readonly SemaphoreSlim _manifestDigestCacheLock = new(1); - - public void AddDigest(string tag, string digest) - { - _localDigestCacheLock.Wait(); - try - { - _localDigestCache[tag] = digest; - } - finally - { - _localDigestCacheLock.Release(); - } - } - - public Task GetLocalImageDigestAsync(string tag, bool isDryRun) => - LockHelper.DoubleCheckedLockLookupAsync(_localDigestCacheLock, _localDigestCache, tag, - () => _inner.Value.GetLocalImageDigestAsync(tag, isDryRun), - // Don't allow null digests to be cached. A locally built image won't have a digest until - // it is pushed so if its digest is retrieved before pushing, we don't want that - // null to be cached. - val => !string.IsNullOrEmpty(val)); - - public Task GetManifestDigestShaAsync(string tag, bool isDryRun) => - LockHelper.DoubleCheckedLockLookupAsync(_manifestDigestCacheLock, _manifestDigestCache, tag, - () => _inner.Value.GetManifestDigestShaAsync(tag, isDryRun)); - } -}