From c0e0a68fc07f99c33eba2ae731e28da5a6aeffed Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 27 Oct 2026 10:06:46 -0700 Subject: [PATCH 01/14] Initial release of AWS library for programmatically building OCI images without a container runtime --- .autover/autover.json | 4 + .../1df94c4d-8841-4630-89ca-9324e3bcaf79.json | 11 + Libraries/AWS.OCI.Builder.slnf | 10 + Libraries/Libraries.sln | 45 ++ .../AWS.OCI.Builder/AWS.OCI.Builder.csproj | 36 + Libraries/src/AWS.OCI.Builder/Descriptor.cs | 31 + .../src/AWS.OCI.Builder/EcrPushOptions.cs | 68 ++ Libraries/src/AWS.OCI.Builder/FileManifest.cs | 85 ++ .../src/AWS.OCI.Builder/ImageReference.cs | 151 ++++ .../AWS.OCI.Builder/Images/ImageBuilder.cs | 173 +++++ .../src/AWS.OCI.Builder/Images/ImageConfig.cs | 232 ++++++ Libraries/src/AWS.OCI.Builder/Images/Layer.cs | 402 ++++++++++ .../src/AWS.OCI.Builder/Images/Manifest.cs | 171 ++++ .../src/AWS.OCI.Builder/LayerPushProgress.cs | 103 +++ Libraries/src/AWS.OCI.Builder/MediaTypes.cs | 37 + .../src/AWS.OCI.Builder/OciImageBuilder.cs | 607 +++++++++++++++ Libraries/src/AWS.OCI.Builder/README.md | 135 ++++ .../Registry/AuthHandshakeHandler.cs | 296 +++++++ .../Registry/IRegistryCredentialSource.cs | 60 ++ .../Registry/RealmValidator.cs | 108 +++ .../Registry/RegistryClient.cs | 682 ++++++++++++++++ .../Registry/RegistryExceptions.cs | 72 ++ .../src/AWS.OCI.Builder/UnixPermissions.cs | 79 ++ .../AWS.OCI.Builder.IntegrationTests.csproj | 27 + .../EcrPushIntegrationTests.cs | 365 +++++++++ .../AWS.OCI.Builder.Tests.csproj | 27 + .../AuthHandshakeHandlerTests.cs | 108 +++ .../DigestVerificationTests.cs | 133 ++++ .../FileManifestTests.cs | 21 + .../ImageAssemblyTests.cs | 263 +++++++ .../ImageReferenceTests.cs | 83 ++ .../test/AWS.OCI.Builder.Tests/LayerTests.cs | 340 ++++++++ .../OciImageBuilderTests.cs | 729 ++++++++++++++++++ .../RealmValidatorTests.cs | 46 ++ .../UnixPermissionsTests.cs | 79 ++ buildtools/build.proj | 1 + 36 files changed, 5820 insertions(+) create mode 100644 .autover/changes/1df94c4d-8841-4630-89ca-9324e3bcaf79.json create mode 100644 Libraries/AWS.OCI.Builder.slnf create mode 100644 Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj create mode 100644 Libraries/src/AWS.OCI.Builder/Descriptor.cs create mode 100644 Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs create mode 100644 Libraries/src/AWS.OCI.Builder/FileManifest.cs create mode 100644 Libraries/src/AWS.OCI.Builder/ImageReference.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Images/Layer.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Images/Manifest.cs create mode 100644 Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs create mode 100644 Libraries/src/AWS.OCI.Builder/MediaTypes.cs create mode 100644 Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs create mode 100644 Libraries/src/AWS.OCI.Builder/README.md create mode 100644 Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs create mode 100644 Libraries/src/AWS.OCI.Builder/UnixPermissions.cs create mode 100644 Libraries/test/AWS.OCI.Builder.IntegrationTests/AWS.OCI.Builder.IntegrationTests.csproj create mode 100644 Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs diff --git a/.autover/autover.json b/.autover/autover.json index 4620f2f6e..670e0ba97 100644 --- a/.autover/autover.json +++ b/.autover/autover.json @@ -158,6 +158,10 @@ { "Name": "Amazon.Lambda.AppSyncEvents", "Path": "Libraries/src/Amazon.Lambda.AppSyncEvents/Amazon.Lambda.AppSyncEvents.csproj" + }, + { + "Name": "AWS.OCI.Builder", + "Path": "Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj" } ], "UseCommitsForChangelog": false, diff --git a/.autover/changes/1df94c4d-8841-4630-89ca-9324e3bcaf79.json b/.autover/changes/1df94c4d-8841-4630-89ca-9324e3bcaf79.json new file mode 100644 index 000000000..0cdcce71f --- /dev/null +++ b/.autover/changes/1df94c4d-8841-4630-89ca-9324e3bcaf79.json @@ -0,0 +1,11 @@ +{ + "Projects": [ + { + "Name": "AWS.OCI.Builder", + "Type": "Minor", + "ChangelogMessages": [ + "Initial release of AWS library for programmatically building OCI images without a container runtime" + ] + } + ] +} \ No newline at end of file diff --git a/Libraries/AWS.OCI.Builder.slnf b/Libraries/AWS.OCI.Builder.slnf new file mode 100644 index 000000000..12b00af44 --- /dev/null +++ b/Libraries/AWS.OCI.Builder.slnf @@ -0,0 +1,10 @@ +{ + "solution": { + "path": "Libraries.sln", + "projects": [ + "src\\AWS.OCI.Builder\\AWS.OCI.Builder.csproj", + "test\\AWS.OCI.Builder.IntegrationTests\\AWS.OCI.Builder.IntegrationTests.csproj", + "test\\AWS.OCI.Builder.Tests\\AWS.OCI.Builder.Tests.csproj" + ] + } +} \ No newline at end of file diff --git a/Libraries/Libraries.sln b/Libraries/Libraries.sln index 89a88dd60..0d4118486 100644 --- a/Libraries/Libraries.sln +++ b/Libraries/Libraries.sln @@ -175,6 +175,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestDurableServerlessApp", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestDurableServerlessApp.IntegrationTests", "test\TestDurableServerlessApp.IntegrationTests\TestDurableServerlessApp.IntegrationTests.csproj", "{A89D56F9-63CC-4B47-BFED-693E3410993D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.OCI.Builder", "src\AWS.OCI.Builder\AWS.OCI.Builder.csproj", "{BB37A746-2351-9685-2D1F-1571F8F23E06}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.OCI.Builder.Tests", "test\AWS.OCI.Builder.Tests\AWS.OCI.Builder.Tests.csproj", "{7636C459-D2A3-30DB-4964-CAF22E6BA256}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWS.OCI.Builder.IntegrationTests", "test\AWS.OCI.Builder.IntegrationTests\AWS.OCI.Builder.IntegrationTests.csproj", "{39FD406D-D77B-9593-CF95-B1BE32F4B7F9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1109,6 +1115,42 @@ Global {A89D56F9-63CC-4B47-BFED-693E3410993D}.Release|x64.Build.0 = Release|Any CPU {A89D56F9-63CC-4B47-BFED-693E3410993D}.Release|x86.ActiveCfg = Release|Any CPU {A89D56F9-63CC-4B47-BFED-693E3410993D}.Release|x86.Build.0 = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|x64.ActiveCfg = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|x64.Build.0 = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|x86.ActiveCfg = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Debug|x86.Build.0 = Debug|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|Any CPU.Build.0 = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|x64.ActiveCfg = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|x64.Build.0 = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|x86.ActiveCfg = Release|Any CPU + {BB37A746-2351-9685-2D1F-1571F8F23E06}.Release|x86.Build.0 = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|x64.ActiveCfg = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|x64.Build.0 = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|x86.ActiveCfg = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Debug|x86.Build.0 = Debug|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|Any CPU.Build.0 = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|x64.ActiveCfg = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|x64.Build.0 = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|x86.ActiveCfg = Release|Any CPU + {7636C459-D2A3-30DB-4964-CAF22E6BA256}.Release|x86.Build.0 = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|x64.ActiveCfg = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|x64.Build.0 = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|x86.ActiveCfg = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Debug|x86.Build.0 = Debug|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|Any CPU.Build.0 = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|x64.ActiveCfg = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|x64.Build.0 = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|x86.ActiveCfg = Release|Any CPU + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1195,6 +1237,9 @@ Global {0460CF9D-B5CC-47C5-9B30-CEA84695AB3B} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} {616E108A-9ED5-4282-B1D4-7FD49BC6DA2E} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} {A89D56F9-63CC-4B47-BFED-693E3410993D} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} + {BB37A746-2351-9685-2D1F-1571F8F23E06} = {AAB54E74-20B1-42ED-BC3D-CE9F7BC7FD12} + {7636C459-D2A3-30DB-4964-CAF22E6BA256} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} + {39FD406D-D77B-9593-CF95-B1BE32F4B7F9} = {1DE4EE60-45BA-4EF7-BE00-B9EB861E4C69} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {503678A4-B8D1-4486-8915-405A3E9CF0EB} diff --git a/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj b/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj new file mode 100644 index 000000000..0db477b77 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj @@ -0,0 +1,36 @@ + + + + + + $(DefaultPackageTargets) + AWS.OCI.Builder + AWS.OCI.Builder + Amazon.OCI.Builder + A .NET library for building OCI container images programmatically — overlay local files as a new layer on a base image and push the result to Amazon ECR, without a Docker daemon. + enable + enable + + + 0.5.0 + AWS.OCI.Builder + aws;ecr;oci;container;docker;image;registry + README.md + https://github.com/aws/aws-lambda-dotnet + git + https://github.com/aws/aws-lambda-dotnet + + + + + + + + + + + <_Parameter1>AWS.OCI.Builder.Tests, PublicKey="0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4" + + + + diff --git a/Libraries/src/AWS.OCI.Builder/Descriptor.cs b/Libraries/src/AWS.OCI.Builder/Descriptor.cs new file mode 100644 index 000000000..4086164d3 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Descriptor.cs @@ -0,0 +1,31 @@ +using System.Text.Json.Serialization; + +namespace Amazon.OCI.Builder; + +/// +/// An OCI content descriptor: a typed, content-addressable reference to a blob +/// (a layer, a config, or a manifest) by its media type, size, and digest. +/// See the OCI Image Specification "descriptor" definition. +/// +public sealed record Descriptor +{ + /// The media type of the referenced content (for example an image config or a gzipped layer). + [JsonPropertyName("mediaType")] + public required string MediaType { get; init; } + + /// The size, in bytes, of the referenced content. + [JsonPropertyName("size")] + public required long Size { get; init; } + + /// + /// The digest of the referenced content, in "sha256:<hex>" form. For a layer this is the + /// digest of the compressed (gzipped) tarball — the value used to address the blob in the registry. + /// + [JsonPropertyName("digest")] + public required string Digest { get; init; } + + /// Optional alternate download locations for the content. + [JsonPropertyName("urls")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string[]? Urls { get; init; } +} diff --git a/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs new file mode 100644 index 000000000..0db5d339a --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs @@ -0,0 +1,68 @@ +namespace Amazon.OCI.Builder; + +/// +/// Options controlling how an image is pushed to Amazon ECR. +/// +/// +/// All properties are optional. The ECR authorization token and target region are not configured here: the +/// token is passed directly to , and the region is taken from the +/// ECR host in the target reference. +/// +public sealed class EcrPushOptions +{ + /// + /// The default value of : 100 MiB (binary megabytes). Chosen so that + /// common base images — whose individual layers run to tens of MiB — are copied entirely in memory, while + /// genuinely large layers spill to disk. + /// + public const long DefaultMaxInMemoryLayerBytes = 100L * 1024 * 1024; + + /// + /// The size threshold, in bytes, that decides how a base-image layer is copied to ECR during a push. A base + /// layer whose size is at or below this value is buffered in memory; one that exceeds it is streamed through a + /// temporary file on disk instead, bounding peak memory for very large base images. Defaults to + /// (100 MiB, binary). + /// + /// + /// + /// This affects only the copy of base-image layers (which are downloaded from the source registry and + /// re-uploaded to ECR). Layers built by this library are produced in memory by construction and are uploaded + /// from memory regardless of this setting. + /// + /// + /// Data integrity is preserved on both paths: the in-memory path verifies the downloaded bytes against the + /// layer's digest before upload, and the spill-to-disk path verifies the digest while streaming to disk + /// and re-reads the file from disk to verify it again before upload — so a faulty disk write that + /// corrupts the spilled layer is caught and the push fails rather than committing corrupt bytes to ECR. The + /// temporary file is deleted after the upload completes (and on failure or cancellation). + /// + /// + /// The value is interpreted as binary bytes. Set it very high to always buffer in memory, or to 0 to + /// always spill to disk. A negative value is rejected at push time. + /// + /// + public long MaxInMemoryLayerBytes { get; init; } = DefaultMaxInMemoryLayerBytes; + + /// + /// An optional sink that receives a event as each layer is pushed, letting a + /// caller observe push progress (for example to render a per-layer progress bar). Defaults to + /// , meaning no progress is reported. + /// + /// + /// + /// Each layer reports a event, zero or more + /// events as its bytes are sent, and a terminal + /// event; a layer the registry already has reports a single + /// event with set. Each event + /// carries the layer's stable / , + /// so the contract holds even if layers are later uploaded concurrently — see . + /// + /// + /// Use for the common case: it captures the current + /// and raises callbacks on it, which keeps UI updates on + /// the UI thread. Any custom implementation must be thread-safe, since a future parallel push may report from + /// multiple threads at once. + /// + /// + public IProgress? Progress { get; init; } +} diff --git a/Libraries/src/AWS.OCI.Builder/FileManifest.cs b/Libraries/src/AWS.OCI.Builder/FileManifest.cs new file mode 100644 index 000000000..ef5742d03 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/FileManifest.cs @@ -0,0 +1,85 @@ +using System.Collections; + +namespace Amazon.OCI.Builder; + +/// +/// Describes a set of local files and/or directories to add to an image, each mapped to an absolute +/// destination path inside the container. A single becomes a single image layer. +/// +/// +/// Collection-initializer syntax is supported, for example: +/// +/// var files = new FileManifest +/// { +/// { "/local/app.dll", "/app/app.dll" }, +/// { "/local/config", "/app/config" }, +/// }; +/// +/// +/// File permissions in the resulting layer: when an entry's is not set, +/// the layer builder derives the Unix mode from the source file on Linux/macOS (preserving an executable bit), +/// and falls back to a standard 0644 for files / 0755 for directories on Windows, which has no Unix +/// mode bits to read. To guarantee a file is executable in the container regardless of the build host — the only +/// way to get the executable bit when building on Windows — use or pass an explicit +/// to . +/// +/// +public sealed class FileManifest : IEnumerable +{ + private readonly List _entries = new(); + + /// The entries that make up this manifest. + public IReadOnlyList Entries => _entries; + + /// + /// Adds a local file or directory to be placed at inside the container, with + /// its Unix mode derived from the source (on Linux/macOS) or defaulted (on Windows). + /// + /// An absolute or relative path to a file or directory on the local filesystem. + /// The absolute destination path inside the container (for example /app/app.dll). + public void Add(string localPath, string containerPath) + => _entries.Add(new FileManifestEntry(localPath, containerPath)); + + /// + /// Adds a local file or directory with an explicit Unix (for example + /// (UnixFileMode)0b111_101_101 for 0755). The explicit mode is applied to the mapped file, or to + /// every file produced by a mapped directory, overriding any source-derived or default mode. + /// + /// An absolute or relative path to a file or directory on the local filesystem. + /// The absolute destination path inside the container. + /// The Unix file mode to stamp on the entry. + public void Add(string localPath, string containerPath, UnixFileMode mode) + => _entries.Add(new FileManifestEntry(localPath, containerPath, mode)); + + /// + /// Adds a local file (or directory of files) and marks it executable (0755) in the container. Use this + /// for scripts and binaries so they are runnable regardless of the build host — on Windows the source file has + /// no executable bit to preserve, so this is the only way to produce an executable entry there. + /// + /// An absolute or relative path to a file or directory on the local filesystem. + /// The absolute destination path inside the container. + public void AddExecutable(string localPath, string containerPath) + => _entries.Add(new FileManifestEntry(localPath, containerPath, FileManifestEntry.ExecutableMode)); + + /// + public IEnumerator GetEnumerator() => _entries.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} + +/// A single local-to-container path mapping within a . +/// The source path on the local filesystem. +/// The absolute destination path inside the container. +/// +/// An explicit Unix file mode for the entry, or to derive it from the source (on +/// Linux/macOS) or use the default (on Windows). For a directory mapping, an explicit mode applies to the files +/// it contains. +/// +public readonly record struct FileManifestEntry(string LocalPath, string ContainerPath, UnixFileMode? Mode = null) +{ + /// A standard executable file mode: 0755 (rwxr-xr-x). + public const UnixFileMode ExecutableMode = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute; +} diff --git a/Libraries/src/AWS.OCI.Builder/ImageReference.cs b/Libraries/src/AWS.OCI.Builder/ImageReference.cs new file mode 100644 index 000000000..7ad7f0ecf --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/ImageReference.cs @@ -0,0 +1,151 @@ +using System.Text.RegularExpressions; + +namespace Amazon.OCI.Builder; + +/// +/// A parsed container image reference: the registry host, repository, and a tag or digest reference. +/// +/// +/// +/// Handles the common reference forms: +/// +/// +/// mcr.microsoft.com/dotnet/aspnet:8.0 — explicit registry host. +/// 123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest — an ECR reference. +/// library/ubuntu:22.04 / ubuntu — Docker Hub shorthand (host defaults to +/// registry-1.docker.io, the library/ prefix is added for single-segment repositories). +/// A @sha256:... digest reference in place of a tag. +/// +/// +/// The distinction between a registry host and the first path segment follows the Docker rule: the first segment +/// is a host only if it contains a . or :, or is exactly localhost. Otherwise the whole +/// string is a Docker Hub repository. +/// +/// +public sealed partial class ImageReference +{ + private const string DockerHubHost = "registry-1.docker.io"; + + private ImageReference(string registry, string repository, string reference, bool referenceIsDigest) + { + Registry = registry; + Repository = repository; + Reference = reference; + ReferenceIsDigest = referenceIsDigest; + } + + /// The registry host, for example mcr.microsoft.com or an ECR endpoint. + public string Registry { get; } + + /// The repository path, for example dotnet/aspnet or myapp. + public string Repository { get; } + + /// The tag (for example 8.0) or digest (for example sha256:...). + public string Reference { get; } + + /// Whether is a digest () rather than a tag. + public bool ReferenceIsDigest { get; } + + /// Whether this reference points at an Amazon ECR registry. + public bool IsEcr => EcrHostRegex().IsMatch(Registry); + + /// + /// The AWS region embedded in an ECR registry host (for example us-west-2), or + /// when the reference is not an ECR reference. + /// + public string? EcrRegion + { + get + { + Match match = EcrHostRegex().Match(Registry); + return match.Success ? match.Groups["region"].Value : null; + } + } + + /// + /// Parses an image reference string. A missing tag defaults to latest. + /// + /// The reference is null, empty, or malformed. + public static ImageReference Parse(string reference) + { + ArgumentException.ThrowIfNullOrWhiteSpace(reference); + + string remainder = reference; + string registry; + + int firstSlash = remainder.IndexOf('/'); + string firstSegment = firstSlash < 0 ? remainder : remainder[..firstSlash]; + + // The first path segment is a registry host only if it looks like one (contains '.' or ':', or is + // "localhost"); otherwise the whole reference is a Docker Hub repository. + bool firstSegmentIsHost = firstSlash > 0 && + (firstSegment.Contains('.', StringComparison.Ordinal) || + firstSegment.Contains(':', StringComparison.Ordinal) || + firstSegment.Equals("localhost", StringComparison.Ordinal)); + + if (firstSegmentIsHost) + { + registry = firstSegment; + remainder = remainder[(firstSlash + 1)..]; + } + else + { + registry = DockerHubHost; + } + + // Split off the tag/digest. A '@' marks a digest; otherwise the last ':' after the final '/' is the tag. + string repository; + string refValue; + bool isDigest; + + int atIndex = remainder.IndexOf('@'); + if (atIndex >= 0) + { + repository = remainder[..atIndex]; + refValue = remainder[(atIndex + 1)..]; + isDigest = true; + } + else + { + int lastSlash = remainder.LastIndexOf('/'); + int tagColon = remainder.IndexOf(':', lastSlash + 1); + if (tagColon >= 0) + { + repository = remainder[..tagColon]; + refValue = remainder[(tagColon + 1)..]; + } + else + { + repository = remainder; + refValue = "latest"; + } + isDigest = false; + } + + if (repository.Length == 0) + { + throw new ArgumentException($"The image reference '{reference}' has no repository.", nameof(reference)); + } + if (refValue.Length == 0) + { + throw new ArgumentException($"The image reference '{reference}' has an empty tag or digest.", nameof(reference)); + } + + // Docker Hub shorthand: a single-segment repository (e.g. "ubuntu") lives under "library/". + if (registry == DockerHubHost && !repository.Contains('/', StringComparison.Ordinal)) + { + repository = "library/" + repository; + } + + return new ImageReference(registry, repository, refValue, isDigest); + } + + /// Returns the canonical registry/repository:tag (or @digest) form. + public override string ToString() + => ReferenceIsDigest ? $"{Registry}/{Repository}@{Reference}" : $"{Registry}/{Repository}:{Reference}"; + + // Matches an ECR host and captures the region: .dkr.ecr..amazonaws.com + // (also supports the FIPS and China/GovCloud partitions' domains). + [GeneratedRegex(@"^\d{12}\.dkr\.ecr(-fips)?\.(?[a-z0-9-]+)\.amazonaws\.com(\.cn)?$", RegexOptions.IgnoreCase)] + private static partial Regex EcrHostRegex(); +} diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs new file mode 100644 index 000000000..c074be358 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs @@ -0,0 +1,173 @@ +using System.Security.Cryptography; + +namespace Amazon.OCI.Builder.Images; + +/// +/// Assembles a derived image from a base image's config and manifest plus one or more overlaid +/// s. It keeps the config's rootfs.diff_ids/history and the manifest's +/// layers in lockstep, applies config customizations (entrypoint, env, etc.), and produces the final +/// config and manifest bytes together with their digests. +/// +/// +/// +/// This type performs only the in-memory config/manifest manipulation — the "overlay a layer" step. It +/// does not fetch the base image or push the result; those are the registry layer's job. A caller fetches the +/// base manifest and config, constructs an from them, overlays layers, then hands the +/// built artifacts to the registry client for upload. +/// +/// +/// The output preserves the base image's schema family: if the base manifest is a Docker V2 manifest, the derived +/// manifest and the layer descriptors use the Docker media types; otherwise OCI media types are used. This avoids +/// producing a manifest whose media types are inconsistent with its config, which some registries reject. +/// +/// +public sealed class ImageBuilder +{ + private readonly ImageConfig _config; + private readonly Manifest _manifest; + private readonly bool _isDockerSchema; + + private ImageBuilder(ImageConfig config, Manifest manifest, bool isDockerSchema) + { + _config = config; + _manifest = manifest; + _isDockerSchema = isDockerSchema; + } + + /// The media type to use for layers overlaid onto this image (matching the base image's schema). + public string LayerMediaType => _isDockerSchema ? MediaTypes.DockerLayerGzip : MediaTypes.OciLayerGzipV1; + + /// The config blob media type for this image's schema. + public string ConfigMediaType => _isDockerSchema ? MediaTypes.DockerImageConfigV1 : MediaTypes.OciImageConfigV1; + + /// The manifest media type for this image's schema. + public string ManifestMediaType => _isDockerSchema ? MediaTypes.DockerManifestV2 : MediaTypes.OciManifestV1; + + /// The config being assembled, exposed for customization (entrypoint, env, labels, ...). + public ImageConfig Config => _config; + + /// + /// Creates a builder from an already-fetched base image's config and manifest. + /// + /// The base image's config blob JSON. + /// The base image's single-platform manifest JSON (not an index). + /// + /// The media type the registry reported for the base manifest. Used to decide whether the derived image uses + /// Docker or OCI media types. When null, OCI is assumed. + /// + /// The config or manifest JSON is missing or invalid. + public static ImageBuilder FromBaseImage(string baseConfigJson, string baseManifestJson, string? baseManifestMediaType) + { + ImageConfig config = ImageConfig.Parse(baseConfigJson); + Manifest manifest = Manifest.Parse(baseManifestJson); + + bool isDocker = IsDockerSchema(baseManifestMediaType, manifest.MediaType); + return new ImageBuilder(config, manifest, isDocker); + } + + /// + /// Overlays a built onto the image: appends its descriptor to the manifest's + /// layers and its diff_id (plus a history entry) to the config. The layer's media type is + /// coerced to the image's schema so the manifest stays internally consistent. + /// + /// A layer produced by . + /// Optional free-text description recorded in the config history for this layer. + public void AddLayer(Layer layer, string? createdBy = null) + { + ArgumentNullException.ThrowIfNull(layer); + + // Keep the layer descriptor's media type aligned with the image schema rather than trusting whatever the + // layer was built with, so a layer built as OCI can still be overlaid onto a Docker-schema base and vice + // versa without producing a mixed-schema manifest. + _manifest.Layers.Add(new Descriptor + { + MediaType = LayerMediaType, + Size = layer.Size, + Digest = layer.Digest, + }); + + _config.AddLayer(layer.DiffId, createdBy); + } + + /// + /// Finalizes the image: serializes the (mutated) config, points the manifest's config descriptor at the + /// fresh config digest, sets the manifest media type to the image schema, and serializes the manifest. The + /// returned carries everything the registry layer needs to upload the image. + /// + public BuiltImage Build() + { + // 1. Serialize the config and compute its digest; the manifest must reference these exact bytes. + byte[] configBytes = _config.ToJsonBytes(); + string configDigest = ComputeDigest(configBytes); + + _manifest.Config = new Descriptor + { + MediaType = ConfigMediaType, + Size = configBytes.LongLength, + Digest = configDigest, + }; + _manifest.MediaType = ManifestMediaType; + + // 2. Serialize the manifest and compute the image digest over those bytes. + byte[] manifestBytes = _manifest.ToJsonBytes(); + string manifestDigest = ComputeDigest(manifestBytes); + + return new BuiltImage( + configBytes, + configDigest, + ConfigMediaType, + manifestBytes, + manifestDigest, + ManifestMediaType); + } + + /// + /// Decides whether the image should be treated as Docker-schema (vs OCI). The reported manifest media type is + /// the strongest signal; the manifest's own mediaType is next; the config's media type field is a last + /// resort. Anything not clearly Docker defaults to OCI. + /// + private static bool IsDockerSchema(string? reportedManifestMediaType, string? manifestMediaType) + { + foreach (string? candidate in new[] { reportedManifestMediaType, manifestMediaType }) + { + if (string.IsNullOrEmpty(candidate)) + { + continue; + } + if (candidate.StartsWith("application/vnd.docker.", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + if (candidate.StartsWith("application/vnd.oci.", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + } + return false; + } + + private static string ComputeDigest(byte[] data) + { + byte[] hash = SHA256.HashData(data); + // Convert.ToHexStringLower exists only on net9.0+, so lower-case explicitly for net8.0 compatibility. + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } +} + +/// +/// The finished image artifacts produced by : the config and manifest bytes with +/// their digests and media types, ready for the registry layer to upload. +/// +/// The canonical config blob bytes to upload. +/// The sha256:... digest of . +/// The config blob media type. +/// The canonical manifest bytes to upload (and PUT under the target tag). +/// The sha256:... digest of (the image digest). +/// The manifest media type to send as the Content-Type on upload. +public sealed record BuiltImage( + byte[] ConfigBytes, + string ConfigDigest, + string ConfigMediaType, + byte[] ManifestBytes, + string ManifestDigest, + string ManifestMediaType); diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs new file mode 100644 index 000000000..4ba1e893a --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs @@ -0,0 +1,232 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Amazon.OCI.Builder.Images; + +/// +/// A mutable view over an OCI / Docker image configuration JSON document — the object referenced by an image +/// manifest's config descriptor. It carries the execution parameters (entrypoint, env, working directory, +/// user, exposed ports, labels) and the rootfs.diff_ids / history that describe the layer stack. +/// +/// +/// +/// The image config schema has many fields and registries reject an image whose config digest does not match its +/// bytes, so this type is deliberately a thin wrapper over a of the original +/// base-image config rather than a hand-modeled POCO. Preserving every field the base set (even ones this library +/// does not understand) keeps the derived image valid and avoids silently dropping data. Mutations touch only the +/// specific nodes they need. +/// +/// +/// Call to get the canonical bytes to upload; the config digest is the SHA-256 of those +/// bytes. All serialization is compact (no insignificant whitespace) and preserves insertion order so the output +/// is stable. +/// +/// +public sealed class ImageConfig +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + // Compact output: the config digest is computed over these exact bytes, so no extra whitespace. + WriteIndented = false, + // Do not re-escape non-ASCII; match what registries and other tools emit. + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + // Required on .NET 8: serializing a JsonNode through a custom options instance throws unless the options + // carry a TypeInfoResolver. .NET 9+ supplies a default implicitly, but we set it explicitly for net8.0. + TypeInfoResolver = new System.Text.Json.Serialization.Metadata.DefaultJsonTypeInfoResolver(), + }; + + private readonly JsonObject _root; + + private ImageConfig(JsonObject root) => _root = root; + + /// Parses an existing image config document (typically the base image's config blob). + /// The config JSON. + /// The JSON is null, empty, or not a JSON object. + public static ImageConfig Parse(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + throw new ArgumentException("The image config JSON must not be null or empty.", nameof(json)); + } + + JsonNode? node; + try + { + node = JsonNode.Parse(json); + } + catch (JsonException ex) + { + throw new ArgumentException("The image config is not valid JSON.", nameof(json), ex); + } + + if (node is not JsonObject obj) + { + throw new ArgumentException("The image config JSON must be a JSON object.", nameof(json)); + } + + return new ImageConfig(obj); + } + + /// The OS this image targets (the config's os field), for example linux. + public string? Os => _root["os"]?.GetValue(); + + /// The CPU architecture this image targets (the config's architecture field), for example amd64. + public string? Architecture => _root["architecture"]?.GetValue(); + + /// + /// Appends a new layer to the config: adds its to rootfs.diff_ids and records + /// a matching history entry. This must be called for each layer added to the manifest, in the same order. + /// + /// The layer's uncompressed-tar digest (). + /// An optional free-text description recorded in the history entry. + public void AddLayer(string diffId, string? createdBy = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(diffId); + + // rootfs.diff_ids — create the structure if the base config somehow lacked it. + if (_root["rootfs"] is not JsonObject rootfs) + { + rootfs = new JsonObject { ["type"] = "layers" }; + _root["rootfs"] = rootfs; + } + if (rootfs["diff_ids"] is not JsonArray diffIds) + { + diffIds = new JsonArray(); + rootfs["diff_ids"] = diffIds; + } + diffIds.Add(diffId); + + // history — one entry per layer keeps the array aligned with the layer stack. The timestamp is fixed so + // the config stays reproducible (see Layer's reproducibility rationale). + if (_root["history"] is not JsonArray history) + { + history = new JsonArray(); + _root["history"] = history; + } + var entry = new JsonObject + { + ["created"] = "1970-01-01T00:00:00Z", + }; + if (!string.IsNullOrEmpty(createdBy)) + { + entry["created_by"] = createdBy; + } + history.Add(entry); + } + + /// Sets the container entrypoint (the config's config.Entrypoint). + public void SetEntrypoint(params string[] entrypoint) + => SetConfigArray("Entrypoint", entrypoint); + + /// Sets the container default command/arguments (the config's config.Cmd). + public void SetCmd(params string[] cmd) + => SetConfigArray("Cmd", cmd); + + /// Sets the working directory (the config's config.WorkingDir). + public void SetWorkingDirectory(string workingDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(workingDirectory); + ConfigObject()["WorkingDir"] = workingDirectory; + } + + /// Sets the user the container runs as (the config's config.User). + public void SetUser(string user) + { + ArgumentException.ThrowIfNullOrWhiteSpace(user); + ConfigObject()["User"] = user; + } + + /// + /// Adds or overwrites an environment variable (merged into the config's config.Env, which is an array of + /// KEY=value strings). An existing entry for the same key is replaced in place. + /// + public void SetEnvironmentVariable(string name, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(value); + + JsonObject config = ConfigObject(); + if (config["Env"] is not JsonArray env) + { + env = new JsonArray(); + config["Env"] = env; + } + + string prefix = name + "="; + string entry = prefix + value; + for (int i = 0; i < env.Count; i++) + { + if (env[i]?.GetValue() is { } existing && existing.StartsWith(prefix, StringComparison.Ordinal)) + { + env[i] = entry; + return; + } + } + env.Add(entry); + } + + /// Adds or overwrites a label (the config's config.Labels map). + public void SetLabel(string key, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(value); + + JsonObject config = ConfigObject(); + if (config["Labels"] is not JsonObject labels) + { + labels = new JsonObject(); + config["Labels"] = labels; + } + labels[key] = value; + } + + /// Marks a port as exposed (added to the config's config.ExposedPorts set). + /// A port spec such as 8080/tcp; a bare number is treated as /tcp. + public void ExposePort(string port) + { + ArgumentException.ThrowIfNullOrWhiteSpace(port); + string key = port.Contains('/', StringComparison.Ordinal) ? port : port + "/tcp"; + + JsonObject config = ConfigObject(); + if (config["ExposedPorts"] is not JsonObject ports) + { + ports = new JsonObject(); + config["ExposedPorts"] = ports; + } + // The value is an empty object per the image-spec convention. + ports[key] = new JsonObject(); + } + + /// Serializes the config to its canonical, compact UTF-8 bytes (what the config digest is computed over). + public byte[] ToJsonBytes() => JsonSerializer.SerializeToUtf8Bytes(_root, SerializerOptions); + + /// Serializes the config to its canonical, compact JSON string. + public string ToJsonString() => _root.ToJsonString(SerializerOptions); + + /// Returns the config sub-object, creating it if absent. + private JsonObject ConfigObject() + { + if (_root["config"] is not JsonObject config) + { + config = new JsonObject(); + _root["config"] = config; + } + return config; + } + + private void SetConfigArray(string key, string[] values) + { + ArgumentNullException.ThrowIfNull(values); + if (values.Length == 0) + { + throw new ArgumentException($"'{key}' must contain at least one element.", nameof(values)); + } + + var array = new JsonArray(); + foreach (string value in values) + { + array.Add(value); + } + ConfigObject()[key] = array; + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs new file mode 100644 index 000000000..2e49902cf --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs @@ -0,0 +1,402 @@ +using System.Formats.Tar; +using System.IO.Compression; +using System.Security.Cryptography; + +namespace Amazon.OCI.Builder.Images; + +/// +/// A single image layer produced from a : a gzipped tar archive of the mapped files, +/// together with the two digests the OCI format requires. +/// +/// +/// +/// An image layer is identified by two different SHA-256 hashes, and the distinction matters: +/// +/// +/// +/// +/// — the hash of the compressed (gzipped) tar bytes. This is the blob's +/// content address: it is what the registry stores the layer under and what the image manifest references. +/// +/// +/// +/// +/// — the hash of the uncompressed tar bytes. This is what the image config's +/// rootfs.diff_ids array references, so the runtime can verify the extracted filesystem. +/// +/// +/// +/// +/// Layers are written deterministically so that the same inputs always yield the same digests: entries are +/// emitted in a stable sorted order, every entry (including synthesized parent directories) is stamped with a +/// fixed epoch timestamp, and paths are normalized to forward-slash, non-rooted form. This reproducibility is +/// what lets callers skip re-pushing an unchanged layer. +/// +/// +public sealed class Layer +{ + /// A fixed timestamp used for every tar entry, so identical inputs produce identical archives. + private static readonly DateTimeOffset ReproducibleTimestamp = DateTimeOffset.UnixEpoch; + + private Layer(byte[] gzippedTarBytes, string digest, string diffId, long size, string mediaType) + { + GzippedTarBytes = gzippedTarBytes; + Digest = digest; + DiffId = diffId; + Size = size; + MediaType = mediaType; + } + + /// The compressed (gzipped) tar archive bytes — the blob content to upload. + public byte[] GzippedTarBytes { get; } + + /// + /// The sha256:... digest of the compressed tar. Addresses the blob in the registry and is + /// referenced by the image manifest's layer descriptor. + /// + public string Digest { get; } + + /// + /// The sha256:... digest of the uncompressed tar. Referenced by the image config's + /// rootfs.diff_ids. + /// + public string DiffId { get; } + + /// The size, in bytes, of the compressed tar (). + public long Size { get; } + + /// The layer media type (OCI or Docker gzipped-layer type, matching the target image's schema). + public string MediaType { get; } + + /// + /// A content descriptor for this layer, suitable for inclusion in an image manifest's layers array. + /// + public Descriptor ToDescriptor() => new() + { + MediaType = MediaType, + Size = Size, + Digest = Digest, + }; + + /// + /// Builds a gzipped-tar layer from the files and directories described by . + /// + /// The local-to-container path mappings that populate the layer. + /// + /// The gzipped-layer media type to report (defaults to ). Pass + /// when overlaying onto a Docker-schema base image. + /// + /// A cancellation token. + /// The built , with both digests computed. + /// is . + /// The manifest is empty, or an entry has an invalid container path. + /// A mapped local file does not exist. + /// A mapped local directory does not exist. + public static async Task CreateAsync( + FileManifest manifest, + string mediaType = MediaTypes.OciLayerGzipV1, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(manifest); + if (manifest.Entries.Count == 0) + { + throw new ArgumentException("The file manifest is empty; a layer must contain at least one file.", nameof(manifest)); + } + + // Expand every manifest entry into concrete file additions (container path -> local file + mode), plus the + // set of directory paths implied by those files. Using a sorted dictionary keyed by the normalized + // container path gives us a deterministic emission order regardless of manifest insertion order. + var files = new SortedDictionary(StringComparer.Ordinal); + var directories = new SortedSet(StringComparer.Ordinal); + + foreach (FileManifestEntry entry in manifest.Entries) + { + CollectEntry(entry, files, directories); + } + + // diff_id is the hash of the uncompressed tar; digest is the hash of the gzipped tar. We compute the + // uncompressed hash by streaming the tar bytes through a hashing stream while they are written, then + // compress those same bytes. Buffering the tar in memory keeps the two passes consistent and lets us + // report an exact compressed size — image layers added by this library are app payloads, not whole OS + // images, so in-memory is appropriate. + byte[] uncompressedTar = await WriteTarAsync(files, directories, cancellationToken).ConfigureAwait(false); + + string diffId = ComputeDigest(uncompressedTar); + byte[] gzipped = Compress(uncompressedTar); + string digest = ComputeDigest(gzipped); + + return new Layer(gzipped, digest, diffId, gzipped.LongLength, mediaType); + } + + /// + /// Expands a single manifest entry into file additions and the directory paths they imply. + /// + private static void CollectEntry(FileManifestEntry entry, SortedDictionary files, SortedSet directories) + { + string containerPath = NormalizeContainerPath(entry.ContainerPath); + + if (File.Exists(entry.LocalPath)) + { + AddFile(entry.LocalPath, containerPath, entry.Mode, files, directories); + return; + } + + if (Directory.Exists(entry.LocalPath)) + { + // A directory maps recursively: each descendant file is placed under the container path, preserving + // its relative sub-path. The directory itself is also recorded so an empty directory still appears. + directories.Add(containerPath); + EnsureParentDirectories(containerPath, directories); + + string root = Path.GetFullPath(entry.LocalPath); + foreach (string localFile in Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)) + { + string relative = Path.GetRelativePath(root, localFile).Replace('\\', '/'); + string target = $"{containerPath}/{relative}"; + AddFile(localFile, target, entry.Mode, files, directories); + } + return; + } + + // Neither a file nor a directory: surface a precise error naming the missing source. + throw new FileNotFoundException( + $"The manifest maps '{entry.LocalPath}' to '{entry.ContainerPath}', but no such file or directory exists.", + entry.LocalPath); + } + + private static void AddFile(string localPath, string containerPath, UnixFileMode? explicitMode, SortedDictionary files, SortedSet directories) + { + if (files.ContainsKey(containerPath)) + { + throw new ArgumentException($"Two manifest entries map to the same container path '{containerPath}'."); + } + files[containerPath] = new FileSource(localPath, ResolveFileMode(localPath, explicitMode)); + EnsureParentDirectories(containerPath, directories); + } + + /// + /// Determines the Unix mode to stamp on a file entry: + /// + /// an explicit mode from the manifest entry wins; + /// otherwise, on Linux/macOS the source file's own mode is used, so an executable source stays + /// executable (a non-readable source is still forced readable so the layer is usable); + /// otherwise (Windows, which has no Unix mode bits) a standard 0644 is used. + /// + /// + private static UnixFileMode ResolveFileMode(string localPath, UnixFileMode? explicitMode) + { + if (explicitMode is { } mode) + { + return mode; + } + + if (!OperatingSystem.IsWindows()) + { + // On Unix the source carries real mode bits. Preserve them (notably the executable bits), but always + // ensure the owner can read the file so the resulting layer entry is never unreadable. + UnixFileMode sourceMode = File.GetUnixFileMode(localPath); + return sourceMode | UnixFileMode.UserRead; + } + + return DefaultFileMode; + } + + /// Records every ancestor directory of so the tar contains them. + private static void EnsureParentDirectories(string containerPath, SortedSet directories) + { + int slash = containerPath.LastIndexOf('/'); + while (slash > 0) + { + string parent = containerPath[..slash]; + if (!directories.Add(parent)) + { + // Already recorded — all its ancestors are too, so we can stop early. + break; + } + slash = parent.LastIndexOf('/'); + } + } + + /// + /// Writes the directory and file entries to an in-memory PAX tar archive, in a deterministic order, and + /// returns the raw (uncompressed) tar bytes. + /// + /// + /// PAX is the format used by container tooling (including Microsoft.NET.Build.Containers) and is what + /// registries and runtimes expect for image layers. PAX normally writes an extended-attribute header whose + /// contents (sub-second mtime/atime/ctime) would vary run-to-run, so this builder sets + /// a fixed extended-attribute set (see ) and a fixed + /// on every entry, making the archive bytes reproducible. + /// + private static async Task WriteTarAsync( + SortedDictionary files, + SortedSet directories, + CancellationToken cancellationToken) + { + using var tarBuffer = new MemoryStream(); + + // leaveOpen: true so we can read the buffer back after the writer is disposed/flushed. + await using (var tar = new TarWriter(tarBuffer, TarEntryFormat.Pax, leaveOpen: true)) + { + // Directories first, shallowest-to-deepest (the sorted set already yields a stable order), so every + // file's parent exists before the file entry. Both collections are ordinally sorted for determinism. + foreach (string dir in directories) + { + cancellationToken.ThrowIfCancellationRequested(); + var dirEntry = new PaxTarEntry(TarEntryType.Directory, ToTarEntryName(dir, isDirectory: true), ReproducibleExtendedAttributes()) + { + Mode = DefaultDirectoryMode, + ModificationTime = ReproducibleTimestamp, + }; + await tar.WriteEntryAsync(dirEntry, cancellationToken).ConfigureAwait(false); + } + + foreach ((string containerPath, FileSource file) in files) + { + cancellationToken.ThrowIfCancellationRequested(); + var fileEntry = new PaxTarEntry(TarEntryType.RegularFile, ToTarEntryName(containerPath, isDirectory: false), ReproducibleExtendedAttributes()) + { + Mode = file.Mode, + ModificationTime = ReproducibleTimestamp, + }; + + // Open each source file read-only and shared so concurrent readers (e.g. the build) aren't blocked. + await using FileStream source = new(file.LocalPath, FileMode.Open, FileAccess.Read, FileShare.Read); + fileEntry.DataStream = source; + await tar.WriteEntryAsync(fileEntry, cancellationToken).ConfigureAwait(false); + } + } + + return tarBuffer.ToArray(); + } + + /// A resolved file to place in the layer: its source path and the Unix mode to stamp on it. + private readonly record struct FileSource(string LocalPath, UnixFileMode Mode); + + /// + /// The PAX extended attributes applied to every entry: fixed mtime, atime and ctime. + /// + /// + /// On .NET 8 the PAX writer auto-populates atime and ctime in the extended header with the + /// current wall-clock time when they are not supplied, which makes the archive bytes (and thus the layer + /// digest) differ on every run. .NET 9+ stopped doing this. Pinning all three timestamps to a fixed epoch + /// here makes the output reproducible on every supported runtime. (Verified by byte-diffing two archives + /// built from identical input on .NET 8.) + /// + private static Dictionary ReproducibleExtendedAttributes() + { + string epoch = ReproducibleTimestamp.ToUnixTimeSeconds().ToString("D", System.Globalization.CultureInfo.InvariantCulture); + return new Dictionary + { + ["mtime"] = epoch, + ["atime"] = epoch, + ["ctime"] = epoch, + }; + } + + /// + /// Gzip-compresses the supplied bytes, then normalizes the gzip member header so the output is byte-for-byte + /// reproducible across runs and target frameworks. + /// + /// + /// A gzip member begins with a 10-byte header whose bytes 4–7 hold a modification time (MTIME) and whose + /// byte 9 holds the source OS. populates these with values that vary by run and by + /// runtime (for example net8.0 writes the current time into MTIME while net9.0+ zeroes it). Left as-is, the + /// compressed digest would change run-to-run even for identical input. We therefore zero MTIME and + /// set the OS byte to 0xFF ("unknown"), matching the convention used by reproducible-build tooling. + /// + private static byte[] Compress(byte[] data) + { + using var output = new MemoryStream(); + // SmallestSize keeps layer blobs compact; layer payloads here are modest, so the extra CPU is cheap. + using (var gzip = new GZipStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + { + gzip.Write(data, 0, data.Length); + } + + byte[] bytes = output.ToArray(); + + // Normalize the gzip header for deterministic output. The header is present whenever there is output; + // a valid gzip stream is always at least the 10-byte header plus the 8-byte trailer. + if (bytes.Length >= 10) + { + // Bytes 4–7: MTIME (little-endian) -> zero. + bytes[4] = 0; + bytes[5] = 0; + bytes[6] = 0; + bytes[7] = 0; + // Byte 9: OS -> 0xFF (unknown), so it does not vary by platform. + bytes[9] = 0xFF; + } + + return bytes; + } + + private static string ComputeDigest(byte[] data) + { + byte[] hash = SHA256.HashData(data); + // Convert.ToHexStringLower exists only on net9.0+, so lower-case explicitly for net8.0 compatibility. + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + /// + /// Normalizes a caller-supplied container path to an absolute, forward-slash path with no trailing slash + /// and no redundant segments. + /// + private static string NormalizeContainerPath(string containerPath) + { + if (string.IsNullOrWhiteSpace(containerPath)) + { + throw new ArgumentException("A container path must not be null or empty."); + } + + string normalized = containerPath.Replace('\\', '/').Trim(); + if (!normalized.StartsWith('/')) + { + throw new ArgumentException($"Container path '{containerPath}' must be absolute (start with '/')."); + } + + // Collapse '.' and empty segments; reject '..' which could escape the intended location. + var segments = new List(); + foreach (string segment in normalized.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + if (segment == ".") + { + continue; + } + if (segment == "..") + { + throw new ArgumentException($"Container path '{containerPath}' must not contain '..' segments."); + } + segments.Add(segment); + } + + if (segments.Count == 0) + { + throw new ArgumentException($"Container path '{containerPath}' does not name a file or directory."); + } + + return "/" + string.Join('/', segments); + } + + /// + /// Converts a normalized, rooted container path (for example /app/app.dll) to a tar entry name. Tar + /// entry names are relative (no leading slash) and directory entries carry a trailing slash by convention. + /// + private static string ToTarEntryName(string normalizedPath, bool isDirectory) + { + string name = normalizedPath.TrimStart('/'); + return isDirectory ? name + "/" : name; + } + + // Standard rootfs permissions: 0755 for directories, 0644 for files. + private const UnixFileMode DefaultDirectoryMode = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + + private const UnixFileMode DefaultFileMode = + UnixFileMode.UserRead | UnixFileMode.UserWrite | + UnixFileMode.GroupRead | + UnixFileMode.OtherRead; +} diff --git a/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs new file mode 100644 index 000000000..abaf4fa36 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs @@ -0,0 +1,171 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Amazon.OCI.Builder.Images; + +/// +/// An image manifest (OCI image manifest or Docker V2 Schema 2): the single-platform document that references the +/// image blob and the ordered set of that make up the image. +/// +/// +/// The manifest is serialized compactly and its property order is fixed (schema version, media type, config, +/// layers), because the image's digest/tag is computed over these exact bytes. +/// +public sealed class Manifest +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + /// The manifest schema version. Always 2 for OCI and Docker V2 Schema 2. + [JsonPropertyName("schemaVersion")] + [JsonPropertyOrder(0)] + public int SchemaVersion { get; set; } = 2; + + /// + /// The manifest media type (for example or + /// ). Some older Docker manifests omit this; it is written when set. + /// + [JsonPropertyName("mediaType")] + [JsonPropertyOrder(1)] + public string? MediaType { get; set; } + + /// The descriptor of the image config blob. + [JsonPropertyName("config")] + [JsonPropertyOrder(2)] + public Descriptor Config { get; set; } = null!; + + /// The ordered layer descriptors (base layers first, then any overlaid layers). + [JsonPropertyName("layers")] + [JsonPropertyOrder(3)] + public List Layers { get; set; } = new(); + + /// Parses an image manifest document. + /// The JSON is null/empty or cannot be parsed as a manifest. + public static Manifest Parse(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + throw new ArgumentException("The manifest JSON must not be null or empty.", nameof(json)); + } + + try + { + return JsonSerializer.Deserialize(json, SerializerOptions) + ?? throw new ArgumentException("The manifest JSON deserialized to null.", nameof(json)); + } + catch (JsonException ex) + { + throw new ArgumentException("The manifest is not valid JSON.", nameof(json), ex); + } + } + + /// Serializes the manifest to its canonical, compact UTF-8 bytes (what the image digest is computed over). + public byte[] ToJsonBytes() => JsonSerializer.SerializeToUtf8Bytes(this, SerializerOptions); + + /// Serializes the manifest to its canonical, compact JSON string. + public string ToJsonString() => JsonSerializer.Serialize(this, SerializerOptions); +} + +/// +/// An image index / manifest list: a multi-platform document whose each point at a +/// single-platform for a specific OS/architecture. Base images are frequently published as +/// an index, so the builder must select the right per-platform manifest from it. +/// +public sealed class ImageIndex +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + /// The schema version (always 2). + [JsonPropertyName("schemaVersion")] + public int SchemaVersion { get; set; } = 2; + + /// The index media type (OCI image index or Docker manifest list). + [JsonPropertyName("mediaType")] + public string? MediaType { get; set; } + + /// The per-platform manifest descriptors. + [JsonPropertyName("manifests")] + public List Manifests { get; set; } = new(); + + /// Parses an image index / manifest list document. + /// The JSON is null/empty or cannot be parsed as an index. + public static ImageIndex Parse(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + throw new ArgumentException("The image index JSON must not be null or empty.", nameof(json)); + } + + try + { + return JsonSerializer.Deserialize(json, SerializerOptions) + ?? throw new ArgumentException("The image index JSON deserialized to null.", nameof(json)); + } + catch (JsonException ex) + { + throw new ArgumentException("The image index is not valid JSON.", nameof(json), ex); + } + } + + /// + /// Selects the manifest descriptor matching and , + /// ignoring descriptors that carry no platform or name an "unknown" platform (attestation entries). + /// + /// The matching descriptor, or when the index has no such platform. + public PlatformManifestDescriptor? FindManifest(string os, string architecture) + { + ArgumentException.ThrowIfNullOrWhiteSpace(os); + ArgumentException.ThrowIfNullOrWhiteSpace(architecture); + + return Manifests.FirstOrDefault(m => + m.Platform is { } p && + string.Equals(p.Os, os, StringComparison.OrdinalIgnoreCase) && + string.Equals(p.Architecture, architecture, StringComparison.OrdinalIgnoreCase)); + } +} + +/// A manifest descriptor within an , carrying the target platform. +public sealed class PlatformManifestDescriptor +{ + /// The media type of the referenced manifest. + [JsonPropertyName("mediaType")] + public string MediaType { get; set; } = null!; + + /// The size, in bytes, of the referenced manifest. + [JsonPropertyName("size")] + public long Size { get; set; } + + /// The sha256:... digest of the referenced manifest. + [JsonPropertyName("digest")] + public string Digest { get; set; } = null!; + + /// The platform this manifest targets, if declared. + [JsonPropertyName("platform")] + public Platform? Platform { get; set; } +} + +/// An OS/architecture platform descriptor within an image index. +public sealed class Platform +{ + /// The operating system, for example linux. + [JsonPropertyName("os")] + public string Os { get; set; } = null!; + + /// The CPU architecture, for example amd64 or arm64. + [JsonPropertyName("architecture")] + public string Architecture { get; set; } = null!; + + /// An optional architecture variant, for example v8 for arm64. + [JsonPropertyName("variant")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Variant { get; set; } +} diff --git a/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs b/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs new file mode 100644 index 000000000..6ab112b43 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs @@ -0,0 +1,103 @@ +namespace Amazon.OCI.Builder; + +/// +/// The lifecycle phase a layer is in when a event is reported. +/// +/// +/// Every layer — whether uploaded or skipped — terminates on , so a consumer counting +/// finished layers increments on a single phase without branching on whether the layer was skipped. A skipped +/// layer is reported as a single event with set +/// to ; it does not emit a preceding because there is no upload to +/// announce. +/// +public enum LayerPushPhase +{ + /// The layer's upload is about to begin. is 0. + Started, + + /// + /// Bytes of the layer have been sent. rises toward + /// across successive events. + /// + Uploading, + + /// + /// The layer is finished. For an uploaded layer equals + /// ; for a skipped layer (already present in the registry) + /// is and + /// is 0. + /// + Completed, +} + +/// +/// Distinguishes a layer copied from the base image from one built locally by this library during the push. +/// +public enum LayerKind +{ + /// A layer of the base image, copied from the source registry into the target repository. + BaseLayer, + + /// A layer produced by this build (via ). + BuiltLayer, +} + +/// +/// A progress event reported while an image is being pushed to Amazon ECR, describing the state of one layer. +/// Subscribe by setting to an of this type. +/// +/// +/// +/// Each layer has a stable identity — , , +/// and — that is assigned once, before any upload begins, and never changes across the events +/// for that layer. Consumers should therefore key their state (for example a per-layer progress bar) by +/// or , not by the order events arrive. +/// +/// +/// This matters because the push may upload layers concurrently in a future revision of this library. When it +/// does, events for different layers will interleave and a layer may finish before an earlier-indexed one — but +/// because each event carries its layer's fixed index and digest, this contract is unchanged and consumers keyed +/// by index/digest continue to work without modification. +/// +/// +/// Only image layers are reported. The config blob and the manifest PUT are not layers and do not +/// produce progress events, so reflects exactly the number of layers (base + built). +/// +/// +public readonly record struct LayerPushProgress +{ + /// The 1-based position of this layer within the push, in the range 1... + public int LayerIndex { get; init; } + + /// The total number of layers in this push (base layers plus layers built by this library). + public int TotalLayers { get; init; } + + /// The layer's sha256:... digest — its stable identity across all events for this layer. + public string Digest { get; init; } + + /// + /// The number of bytes of this layer uploaded so far. 0 at , rising during + /// , and equal to at + /// for an uploaded layer (0 for a skipped one). + /// + public long BytesPushed { get; init; } + + /// + /// The total size of this layer, in bytes. Reported even for a skipped layer (so a UI can show "skipped — + /// 37 MB already present" rather than "0 of 0 bytes"). + /// + public long LayerSizeBytes { get; init; } + + /// + /// when the layer was not uploaded because the registry already has the blob. Such a + /// layer is reported as a single event with + /// of 0. + /// + public bool IsSkipped { get; init; } + + /// The lifecycle phase this event represents. + public LayerPushPhase Phase { get; init; } + + /// Whether this layer is a base-image layer or a layer built by this library. + public LayerKind Kind { get; init; } +} diff --git a/Libraries/src/AWS.OCI.Builder/MediaTypes.cs b/Libraries/src/AWS.OCI.Builder/MediaTypes.cs new file mode 100644 index 000000000..29235977a --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/MediaTypes.cs @@ -0,0 +1,37 @@ +namespace Amazon.OCI.Builder; + +/// +/// Well-known media type strings used by the OCI Image Specification and the legacy +/// Docker Image Manifest V2 Schema 2. A base image may use either family; the builder +/// preserves the base image's family when producing the output manifest. +/// +public static class MediaTypes +{ + // --- OCI (application/vnd.oci.*) --- + + /// OCI image manifest (a single-platform image). + public const string OciManifestV1 = "application/vnd.oci.image.manifest.v1+json"; + + /// OCI image index (a multi-platform manifest list). + public const string OciImageIndexV1 = "application/vnd.oci.image.index.v1+json"; + + /// OCI image config. + public const string OciImageConfigV1 = "application/vnd.oci.image.config.v1+json"; + + /// OCI gzipped layer. + public const string OciLayerGzipV1 = "application/vnd.oci.image.layer.v1.tar+gzip"; + + // --- Docker Image Manifest V2 Schema 2 (application/vnd.docker.*) --- + + /// Docker image manifest (a single-platform image). + public const string DockerManifestV2 = "application/vnd.docker.distribution.manifest.v2+json"; + + /// Docker manifest list (a multi-platform manifest list). + public const string DockerManifestListV2 = "application/vnd.docker.distribution.manifest.list.v2+json"; + + /// Docker image config. + public const string DockerImageConfigV1 = "application/vnd.docker.container.image.v1+json"; + + /// Docker gzipped layer. + public const string DockerLayerGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip"; +} diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs new file mode 100644 index 000000000..6e70da196 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -0,0 +1,607 @@ +using Amazon.OCI.Builder.Images; +using Amazon.OCI.Builder.Registry; + +namespace Amazon.OCI.Builder; + +/// +/// The programmatic entry point for building an OCI container image from a base image plus local files and +/// pushing it to Amazon ECR — no Docker daemon required. +/// +/// +/// +/// The flow mirrors what the .NET SDK's container tooling does internally, specialized for ECR as the push target: +/// +/// +/// Pull the base image's manifest from its source registry (selecting the platform from a manifest list +/// when the base is multi-platform), then its config blob. Base layer blobs are not downloaded unless +/// they must be copied to ECR. +/// Build one gzipped-tar layer per call, overlaying the local files at +/// their target container paths. +/// Apply any config customizations (entrypoint, env, working directory, ...). +/// Copy the base layers to the target ECR repository, upload the new layer(s) and the rewritten config, +/// then PUT the manifest under the destination tag. +/// +/// +/// Typical use: +/// +/// +/// var builder = await OciImageBuilder.FromBaseImageAsync( +/// "mcr.microsoft.com/dotnet/aspnet:8.0", platform: "linux/amd64"); +/// +/// await builder.AddLayerAsync(new FileManifest { { "bin/Release/net8.0/publish", "/app" } }); +/// builder.Config.SetWorkingDirectory("/app"); +/// builder.Config.SetEntrypoint("dotnet", "/app/MyApp.dll"); +/// +/// await builder.PushAsync("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest"); +/// +/// This type is not thread-safe; use one instance per image build. +/// +public sealed class OciImageBuilder : IDisposable +{ + private readonly ImageReference _baseReference; + private readonly RegistryClient _sourceClient; + private readonly ImageBuilder _imageBuilder; + private readonly IReadOnlyList _baseLayers; + private readonly List _pendingLayers = new(); + private bool _disposed; + + /// + /// Factory that produces the registry client used to push to the target. Defaults to an ECR-authenticating + /// client built from the push options; replaced in tests to inject a stubbed transport. The default ignores + /// the credential-source argument because builds the ECR source itself. + /// + private Func _targetClientFactory = + static (host, credentialSource) => RegistryClient.Create(host, credentialSource); + + private OciImageBuilder( + ImageReference baseReference, + RegistryClient sourceClient, + ImageBuilder imageBuilder, + IReadOnlyList baseLayers) + { + _baseReference = baseReference; + _sourceClient = sourceClient; + _imageBuilder = imageBuilder; + _baseLayers = baseLayers; + } + + /// + /// Test-only factory: builds an instance directly from pre-fetched base data and an injected source client, + /// bypassing the live base-image pull. Combine with to exercise + /// the full push flow against stubbed transports. + /// + internal static OciImageBuilder CreateForTest( + ImageReference baseReference, + RegistryClient sourceClient, + ImageBuilder imageBuilder, + IReadOnlyList baseLayers) + => new(baseReference, sourceClient, imageBuilder, baseLayers); + + /// Test-only: overrides how the push-target registry client is created. + internal void UseTargetClientFactoryForTest(Func factory) + => _targetClientFactory = factory; + + /// The config of the image being built, exposed for customization (entrypoint, env, labels, ...). + public ImageConfig Config => _imageBuilder.Config; + + /// + /// Fetches a base image and prepares a builder to overlay layers onto it. + /// + /// + /// The base image reference, for example mcr.microsoft.com/dotnet/aspnet:8.0 or + /// ubuntu:22.04. Public registries (anonymous or Bearer-token pull) are supported. + /// + /// + /// The target platform as os/architecture (for example linux/amd64 or linux/arm64), used + /// to select the right manifest when the base image is published as a multi-platform manifest list. Defaults to + /// linux/amd64. + /// + /// A cancellation token. + /// The base reference or platform string is malformed. + /// The base image or the requested platform could not be fetched. + public static async Task FromBaseImageAsync( + string baseImage, + string platform = "linux/amd64", + CancellationToken cancellationToken = default) + { + ImageReference baseReference = ImageReference.Parse(baseImage); + (string os, string architecture) = ParsePlatform(platform); + + // Public base images pull anonymously (the handshake still performs a Bearer-token exchange when the + // registry demands one, e.g. Docker Hub; the token is obtained without credentials). + RegistryClient sourceClient = RegistryClient.Create(baseReference.Registry, new AnonymousRegistryCredentialSource()); + + try + { + // 1. Fetch the base reference. It may be a single-platform manifest or a manifest list / image index. + ManifestResponse top = await sourceClient + .GetManifestAsync(baseReference.Repository, baseReference.Reference, cancellationToken) + .ConfigureAwait(false); + + // Verify the top-level document's integrity. When the caller pinned a digest (repo@sha256:...), that + // digest is authoritative and must match. Otherwise fall back to the registry-reported + // Docker-Content-Digest when present — cheap defense-in-depth against a cache/proxy corrupting the + // manifest in transit. (A tag fetch from a registry that omits the header is left unverified here; + // the per-platform manifest, config, and layers it points at are all verified by digest below.) + string? topExpectedDigest = baseReference.ReferenceIsDigest ? baseReference.Reference : top.Digest; + if (topExpectedDigest is not null) + { + RegistryClient.VerifyDigest("base image manifest", topExpectedDigest, top.RawBytes); + } + + ManifestResponse platformManifest = await ResolvePlatformManifestAsync( + sourceClient, baseReference, top, os, architecture, cancellationToken).ConfigureAwait(false); + + // 2. Fetch the config blob referenced by the single-platform manifest, verifying its digest. + Manifest manifest = Manifest.Parse(platformManifest.Json); + if (manifest.Config is null) + { + throw new RegistryException($"The base manifest for '{baseImage}' has no config descriptor."); + } + + byte[] configBytes = await sourceClient + .GetVerifiedBlobBytesAsync(baseReference.Repository, manifest.Config.Digest, "base image config", cancellationToken) + .ConfigureAwait(false); + string configJson = System.Text.Encoding.UTF8.GetString(configBytes); + + // 3. Prepare the in-memory overlay builder, preserving the base image's schema (OCI vs Docker). + ImageBuilder imageBuilder = ImageBuilder.FromBaseImage(configJson, platformManifest.Json, platformManifest.MediaType); + + // Capture the base image's layer descriptors now; PushAsync copies these to ECR before uploading the + // new layer(s). They are a snapshot of the base manifest, distinct from layers added by this build. + IReadOnlyList baseLayers = manifest.Layers.ToArray(); + + return new OciImageBuilder(baseReference, sourceClient, imageBuilder, baseLayers); + } + catch + { + sourceClient.Dispose(); + throw; + } + } + + /// + /// Adds the files described by as a new image layer overlaid on top of the base + /// (and any previously-added) layers. Each call produces one layer. Call this before . + /// + /// The local-to-container path mappings that populate the layer. + /// Optional free-text description recorded in the image history for this layer. + /// A cancellation token. + /// The manifest is empty or an entry has an invalid container path. + /// A mapped local file does not exist. + public async Task AddLayerAsync(FileManifest files, string? createdBy = null, CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + ArgumentNullException.ThrowIfNull(files); + + // Build the layer with the media type matching the image schema so the manifest stays consistent. + Layer layer = await Layer.CreateAsync(files, _imageBuilder.LayerMediaType, cancellationToken).ConfigureAwait(false); + _imageBuilder.AddLayer(layer, createdBy); + + // Spill the compressed layer to a temp file so it is not retained in memory for the builder's lifetime. + // Only the (small) DiffId is needed in memory after this — the config references it; the gzipped blob is + // streamed from disk at push time. The builder owns the file and deletes it on Dispose. + string layerFile = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"aws-oci-built-layer-{Guid.NewGuid():N}.tmp"); + try + { + await using (var file = new FileStream( + layerFile, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 81920, useAsync: true)) + { + await file.WriteAsync(layer.GzippedTarBytes, cancellationToken).ConfigureAwait(false); + } + } + catch + { + TryDeleteFile(layerFile); + throw; + } + + _pendingLayers.Add(new PendingLayer(layer.Digest, layerFile, layer.GzippedTarBytes.LongLength)); + } + + /// + /// Pushes the assembled image to the given Amazon ECR reference, authenticating with an ECR authorization + /// token the caller has already obtained. The target repository must already exist in ECR (this library does + /// not create repositories). The region is taken from the ECR host in . + /// + /// + /// + /// This method is named for its ECR-specific behavior: it expects to be + /// the token returned by the ECR GetAuthorizationToken API — a base64-encoded AWS:<password> + /// string — which the library decodes into the Basic username/password the registry handshake uses. + /// + /// + /// The library deliberately does not depend on the AWS SDK: obtaining the token (and refreshing it if + /// the push outlives its ~12-hour validity) is the caller's responsibility. A typical caller holds an + /// AmazonECRClient (or AWS Tools for PowerShell) already and passes the token straight through, e.g.: + /// + /// + /// var auth = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); + /// string token = auth.AuthorizationData[0].AuthorizationToken; // base64("AWS:<password>") + /// string digest = await builder.PushToEcrAsync(targetImage, token); + /// + /// + /// + /// The destination ECR reference, for example + /// 123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest. + /// + /// + /// The ECR authorization token — the base64-encoded AWS:<password> string from the ECR + /// GetAuthorizationToken API's AuthorizationData.AuthorizationToken. + /// + /// Optional push settings (for example the in-memory layer threshold). + /// A cancellation token. + /// The pushed image's manifest digest (sha256:...). + /// + /// The target reference is not a valid ECR reference, the authorization token is missing or malformed, or an + /// option value is invalid. + /// + /// A registry operation failed. + public async Task PushToEcrAsync( + string targetImage, + string authorizationToken, + EcrPushOptions? options = null, + CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + + ImageReference target = ImageReference.Parse(targetImage); + if (!target.IsEcr) + { + throw new ArgumentException( + $"The push target '{targetImage}' is not an Amazon ECR reference. This library only pushes to ECR " + + "(e.g. 123456789012.dkr.ecr.us-west-2.amazonaws.com/repo:tag).", + nameof(targetImage)); + } + + long maxInMemoryLayerBytes = options?.MaxInMemoryLayerBytes ?? EcrPushOptions.DefaultMaxInMemoryLayerBytes; + if (maxInMemoryLayerBytes < 0) + { + throw new ArgumentException( + $"{nameof(EcrPushOptions.MaxInMemoryLayerBytes)} must be non-negative (was {maxInMemoryLayerBytes}).", + nameof(options)); + } + + // Decode the ECR authorization token into the Basic username/password the registry handshake expects. + (string username, string password) = DecodeEcrAuthorizationToken(authorizationToken); + var credentialSource = new StaticRegistryCredentialSource(username, password); + + IProgress? progress = options?.Progress; + + using RegistryClient targetClient = _targetClientFactory(target.Registry, credentialSource); + + // 1. Finalize the config + manifest bytes and their digests. + BuiltImage built = _imageBuilder.Build(); + + // Assign each layer its stable identity (index, total, digest, size, kind) BEFORE any upload begins. Base + // layers (in manifest order) come first, then the locally-built layers (in add order). Fixing identity up + // front is what makes the progress contract survive a future parallel push: an event always carries its + // layer's own index/digest, never a position derived from the order work happens to finish in. + int totalLayers = _baseLayers.Count + _pendingLayers.Count; + + // 2. Copy the base image's layers from the source registry to the ECR target. Cross-host, so each base + // layer that ECR does not already have is streamed from the source and uploaded. Base layers take the + // first index slots; the call returns the next free index for the locally-built layers. + int nextIndex = await CopyBaseLayersAsync(target.Repository, targetClient, maxInMemoryLayerBytes, progress, totalLayers, cancellationToken) + .ConfigureAwait(false); + + // 3. Upload the new layer blob(s) produced locally. Each was spilled to a temp file by AddLayerAsync, so + // it is streamed from disk here rather than held in memory. + foreach (PendingLayer pending in _pendingLayers) + { + int layerIndex = nextIndex++; + long size = pending.Size; + + if (await targetClient.BlobExistsAsync(target.Repository, pending.Digest, cancellationToken).ConfigureAwait(false)) + { + ReportSkipped(progress, layerIndex, totalLayers, pending.Digest, size, LayerKind.BuiltLayer); + continue; + } + + ReportStarted(progress, layerIndex, totalLayers, pending.Digest, size, LayerKind.BuiltLayer); + + await using (var layerStream = new FileStream( + pending.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, useAsync: true)) + { + await targetClient.UploadBlobAsync( + target.Repository, + pending.Digest, + layerStream, + cancellationToken, + onBytesSent: sent => ReportUploading(progress, layerIndex, totalLayers, pending.Digest, sent, size, LayerKind.BuiltLayer), + contentLength: size) + .ConfigureAwait(false); + } + + ReportCompleted(progress, layerIndex, totalLayers, pending.Digest, size, LayerKind.BuiltLayer); + } + + // 4. Upload the config blob. + using (var configStream = new MemoryStream(built.ConfigBytes, writable: false)) + { + await targetClient.UploadBlobAsync( + target.Repository, built.ConfigDigest, configStream, cancellationToken, contentLength: built.ConfigBytes.LongLength) + .ConfigureAwait(false); + } + + // 5. PUT the manifest under the destination tag/digest. All referenced blobs are now present. + string manifestJson = System.Text.Encoding.UTF8.GetString(built.ManifestBytes); + await targetClient.PutManifestAsync(target.Repository, target.Reference, manifestJson, built.ManifestMediaType, cancellationToken) + .ConfigureAwait(false); + + return built.ManifestDigest; + } + + /// + /// Copies every base-image layer from the source registry into the target ECR repository, skipping any blob + /// ECR already has. Base layers come from a different host than ECR, so cross-repo mount cannot be used — the + /// blob is streamed from the source and re-uploaded. + /// + /// + /// A layer whose declared size is at or below is buffered in memory; + /// a larger one is streamed through a temporary file on disk to bound peak memory. Both paths verify the + /// layer's digest before upload (the disk path verifies twice — see + /// ), so data integrity holds regardless of which path + /// a layer takes. + /// + private async Task CopyBaseLayersAsync( + string targetRepository, + RegistryClient targetClient, + long maxInMemoryLayerBytes, + IProgress? progress, + int totalLayers, + CancellationToken cancellationToken) + { + // Base layers occupy the first index slots (1..baseCount). The local counter advances per layer and its + // final value — the next free index — is returned for the locally-built layers that follow. + int index = 1; + foreach (Descriptor layer in _baseLayers) + { + int layerIndex = index++; + long size = layer.Size; + + if (await targetClient.BlobExistsAsync(targetRepository, layer.Digest, cancellationToken).ConfigureAwait(false)) + { + ReportSkipped(progress, layerIndex, totalLayers, layer.Digest, size, LayerKind.BaseLayer); + continue; + } + + ReportStarted(progress, layerIndex, totalLayers, layer.Digest, size, LayerKind.BaseLayer); + + if (layer.Size > maxInMemoryLayerBytes) + { + // Large layer: stream to a temp file (verified twice) and upload from disk, bounding memory. + await using VerifiedBlobFile spilled = await _sourceClient + .GetVerifiedBlobToFileAsync(_baseReference.Repository, layer.Digest, "base layer", cancellationToken) + .ConfigureAwait(false); + + await targetClient.UploadBlobAsync( + targetRepository, + layer.Digest, + spilled.Stream, + cancellationToken, + onBytesSent: sent => ReportUploading(progress, layerIndex, totalLayers, layer.Digest, sent, size, LayerKind.BaseLayer), + contentLength: size) + .ConfigureAwait(false); + } + else + { + // Small layer: download, verify, and re-upload from memory. + byte[] blob = await _sourceClient + .GetVerifiedBlobBytesAsync(_baseReference.Repository, layer.Digest, "base layer", cancellationToken) + .ConfigureAwait(false); + + using var upload = new MemoryStream(blob, writable: false); + await targetClient.UploadBlobAsync( + targetRepository, + layer.Digest, + upload, + cancellationToken, + onBytesSent: sent => ReportUploading(progress, layerIndex, totalLayers, layer.Digest, sent, size, LayerKind.BaseLayer), + contentLength: blob.LongLength) + .ConfigureAwait(false); + } + + ReportCompleted(progress, layerIndex, totalLayers, layer.Digest, size, LayerKind.BaseLayer); + } + + return index; + } + + // ---- Progress reporting helpers ------------------------------------------------------------ + // Each helper builds one LayerPushProgress event and hands it to the sink (when one is configured). Pulling + // these out keeps the push methods readable and guarantees every event sets the same stable-identity fields. + + private static void ReportStarted( + IProgress? progress, int index, int total, string digest, long size, LayerKind kind) + => progress?.Report(new LayerPushProgress + { + LayerIndex = index, + TotalLayers = total, + Digest = digest, + BytesPushed = 0, + LayerSizeBytes = size, + IsSkipped = false, + Phase = LayerPushPhase.Started, + Kind = kind, + }); + + private static void ReportUploading( + IProgress? progress, int index, int total, string digest, long bytesPushed, long size, LayerKind kind) + => progress?.Report(new LayerPushProgress + { + LayerIndex = index, + TotalLayers = total, + Digest = digest, + BytesPushed = bytesPushed, + LayerSizeBytes = size, + IsSkipped = false, + Phase = LayerPushPhase.Uploading, + Kind = kind, + }); + + private static void ReportCompleted( + IProgress? progress, int index, int total, string digest, long size, LayerKind kind) + => progress?.Report(new LayerPushProgress + { + LayerIndex = index, + TotalLayers = total, + Digest = digest, + BytesPushed = size, + LayerSizeBytes = size, + IsSkipped = false, + Phase = LayerPushPhase.Completed, + Kind = kind, + }); + + private static void ReportSkipped( + IProgress? progress, int index, int total, string digest, long size, LayerKind kind) + => progress?.Report(new LayerPushProgress + { + LayerIndex = index, + TotalLayers = total, + Digest = digest, + BytesPushed = 0, + LayerSizeBytes = size, + IsSkipped = true, + Phase = LayerPushPhase.Completed, + Kind = kind, + }); + + private static async Task ResolvePlatformManifestAsync( + RegistryClient client, + ImageReference baseReference, + ManifestResponse top, + string os, + string architecture, + CancellationToken cancellationToken) + { + // Single-platform manifest — use it directly. + if (!IsIndexMediaType(top.MediaType)) + { + return top; + } + + // Manifest list / image index — pick the matching platform and fetch that manifest by digest, then verify + // the returned bytes hash to the digest we requested it under. + ImageIndex index = ImageIndex.Parse(top.Json); + PlatformManifestDescriptor? selected = index.FindManifest(os, architecture) + ?? throw new RegistryException( + $"The base image '{baseReference}' has no manifest for platform '{os}/{architecture}'. " + + $"Available: {DescribePlatforms(index)}."); + + ManifestResponse platformManifest = + await client.GetManifestAsync(baseReference.Repository, selected.Digest, cancellationToken).ConfigureAwait(false); + + RegistryClient.VerifyDigest( + $"platform manifest {os}/{architecture}", + selected.Digest, + platformManifest.RawBytes); + + return platformManifest; + } + + private static string DescribePlatforms(ImageIndex index) + { + IEnumerable platforms = index.Manifests + .Where(m => m.Platform is not null && !string.Equals(m.Platform!.Os, "unknown", StringComparison.OrdinalIgnoreCase)) + .Select(m => $"{m.Platform!.Os}/{m.Platform.Architecture}"); + string joined = string.Join(", ", platforms); + return joined.Length == 0 ? "(none)" : joined; + } + + private static bool IsIndexMediaType(string mediaType) + => string.Equals(mediaType, MediaTypes.OciImageIndexV1, StringComparison.OrdinalIgnoreCase) + || string.Equals(mediaType, MediaTypes.DockerManifestListV2, StringComparison.OrdinalIgnoreCase); + + private static (string Os, string Architecture) ParsePlatform(string platform) + { + ArgumentException.ThrowIfNullOrWhiteSpace(platform); + string[] parts = platform.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + throw new ArgumentException( + $"The platform '{platform}' must be in 'os/architecture' form, for example 'linux/amd64'.", + nameof(platform)); + } + return (parts[0], parts[1]); + } + + private void ThrowIfDisposed() + => ObjectDisposedException.ThrowIf(_disposed, this); + + /// + /// Decodes an ECR authorization token (base64 of username:password, where the username is AWS) + /// into its username/password parts. + /// + /// The token is null/empty, not valid base64, or not in the expected form. + private static (string Username, string Password) DecodeEcrAuthorizationToken(string authorizationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(authorizationToken); + + byte[] decodedBytes; + try + { + decodedBytes = Convert.FromBase64String(authorizationToken.Trim()); + } + catch (FormatException ex) + { + throw new ArgumentException( + "The ECR authorization token is not valid base64. Pass the AuthorizationData.AuthorizationToken " + + "value returned by the ECR GetAuthorizationToken API verbatim.", + nameof(authorizationToken), ex); + } + + string decoded = System.Text.Encoding.UTF8.GetString(decodedBytes); + int separator = decoded.IndexOf(':'); + if (separator < 0) + { + throw new ArgumentException( + "The ECR authorization token did not decode to the expected 'username:password' form.", + nameof(authorizationToken)); + } + + return (decoded[..separator], decoded[(separator + 1)..]); + } + + /// Disposes the source registry client and deletes any temp files holding spilled built layers. + public void Dispose() + { + if (_disposed) + { + return; + } + _disposed = true; + + foreach (PendingLayer pending in _pendingLayers) + { + TryDeleteFile(pending.FilePath); + } + _pendingLayers.Clear(); + + _sourceClient.Dispose(); + } + + /// Best-effort delete of a temp file; a leftover must never mask a more important error. + private static void TryDeleteFile(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch + { + // Best-effort cleanup only. + } + } + + /// + /// A layer built locally during this session. The compressed blob is spilled to (not + /// kept in memory) and streamed from there by ; the builder deletes the file on + /// . + /// + private readonly record struct PendingLayer(string Digest, string FilePath, long Size); +} diff --git a/Libraries/src/AWS.OCI.Builder/README.md b/Libraries/src/AWS.OCI.Builder/README.md new file mode 100644 index 000000000..d7e52a499 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/README.md @@ -0,0 +1,135 @@ +# AWS.OCI.Builder + +A .NET library for building [OCI](https://opencontainers.org/) container images +**programmatically**, without a Docker daemon. It pulls a base image's manifest over the +registry HTTP API, overlays a new layer containing local files at specified container paths, +and pushes the resulting image to **Amazon ECR**. + +## Status + +This library is designed and supported for use by other AWS libraries and tools. +The version of the library is kept below 1.0.0 to indicate that the API is subject to change base on the needs of AWS libraries and tools. +If you are using this library in a non AWS context, please be aware that the API may change in future releases. + +## Authentication + +This library has **no AWS SDK dependency**. Instead of resolving AWS credentials itself, it takes an ECR +authorization token that you obtain and pass to `PushToEcrAsync`. This keeps the library free of +`AWSSDK.Core`, so it will not conflict with the AWS SDK versions a host (for example an AWS Tools for +PowerShell cmdlet) already loads. + +The token is the value returned by the ECR +[`GetAuthorizationToken`](https://docs.aws.amazon.com/AmazonECR/latest/APIReference/API_GetAuthorizationToken.html) +API — a base64-encoded `AWS:` string. The library decodes it into the `Basic` username/password the +registry handshake needs. Obtaining and refreshing the token (it is valid for ~12 hours) is the caller's +responsibility; the target region is taken from the ECR host in the target reference. + +```csharp +using Amazon.ECR; +using Amazon.ECR.Model; + +// Caller obtains the token however it likes — directly via the SDK, or from an AWS Tools for PowerShell cmdlet. +using var ecr = new AmazonECRClient(); // region/credentials from the SDK default chain +var auth = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); +string authorizationToken = auth.AuthorizationData[0].AuthorizationToken; // base64("AWS:") +``` + +## Usage + +```csharp +using Amazon.OCI.Builder; + +// Pull the base image (selects the right platform from a manifest list). +using var builder = await OciImageBuilder.FromBaseImageAsync( + "mcr.microsoft.com/dotnet/aspnet:8.0", + platform: "linux/amd64"); + +// Overlay local files as a new layer. Each call adds one layer. +await builder.AddLayerAsync(new FileManifest +{ + { "/local/publish/MyApp.dll", "/app/MyApp.dll" }, + { "/local/publish/config", "/app/config" }, +}); + +// Setting Linux file permissions (see "File permissions" below): +var files = new FileManifest(); +files.Add("/local/publish/MyApp.dll", "/app/MyApp.dll"); // default 0644 +files.AddExecutable("/local/scripts/entrypoint.sh", "/app/entrypoint.sh"); // 0755 +files.Add("/local/secrets/key", "/app/key", UnixPermissions.FromOctal("600")); // explicit 0600 +await builder.AddLayerAsync(files); + +// Customize the image config. +builder.Config.SetWorkingDirectory("/app"); +builder.Config.SetEntrypoint("dotnet", "/app/MyApp.dll"); + +// Environment variables are merged into the base image's existing config.Env. Setting a key the +// base image already defines (e.g. PATH) overrides it in place rather than duplicating it. +builder.Config.SetEnvironmentVariable("ASPNETCORE_URLS", "http://+:8080"); +builder.Config.SetEnvironmentVariable("DOTNET_EnableDiagnostics", "0"); + +// Obtain an ECR authorization token (the library has no AWS SDK dependency — getting the token is the +// caller's job; see "Authentication" above), then push. The region is taken from the ECR host in the target. +string digest = await builder.PushToEcrAsync( + "123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest", + authorizationToken); +``` + +### Tracking push progress + +Pass an `IProgress` via `EcrPushOptions.Progress` to observe the push layer by layer — for +example to render a progress bar. It is entirely opt-in; omit it and the push reports nothing. + +```csharp +var progress = new Progress(p => +{ + switch (p.Phase) + { + case LayerPushPhase.Started: + Console.WriteLine($"[{p.LayerIndex}/{p.TotalLayers}] {p.Kind} {p.Digest} — uploading {p.LayerSizeBytes:N0} bytes"); + break; + case LayerPushPhase.Uploading: + Console.WriteLine($"[{p.LayerIndex}/{p.TotalLayers}] {p.BytesPushed:N0} / {p.LayerSizeBytes:N0} bytes"); + break; + case LayerPushPhase.Completed when p.IsSkipped: + Console.WriteLine($"[{p.LayerIndex}/{p.TotalLayers}] {p.Digest} — already present, skipped"); + break; + case LayerPushPhase.Completed: + Console.WriteLine($"[{p.LayerIndex}/{p.TotalLayers}] {p.Digest} — done"); + break; + } +}); + +string digest = await builder.PushToEcrAsync( + "123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest", + authorizationToken, + new EcrPushOptions { Progress = progress }); +``` + +Each layer reports a `Started` event, zero or more `Uploading` events as its bytes are sent, and a terminal +`Completed` event. A layer the registry already has reports a **single** `Completed` event with `IsSkipped = true` +and `BytesPushed = 0` — so "layer done" is always the one `Completed` phase, whether the layer was uploaded or +skipped. Only image layers are reported (the config blob and manifest are not layers), so `TotalLayers` is exactly +the number of layers. + +Every event carries the layer's stable `LayerIndex` (1-based), `TotalLayers`, `Digest` and `Kind` +(`BaseLayer` vs `BuiltLayer`), all fixed before any upload begins. Key your state by `LayerIndex` or `Digest` +rather than the order events arrive: the contract is designed so that if the library later uploads layers in +parallel — interleaving their events — consumers keep working unchanged. A custom `IProgress` implementation +must be thread-safe for that reason; `System.Progress` already marshals callbacks to the captured +synchronization context. + +### File permissions + +Each file in a layer carries a Unix mode. When you don't specify one: + +- On **Linux/macOS** the source file's own mode is preserved, so an already-executable file stays executable. +- On **Windows** — which has no Unix mode bits to read — files default to `0644` and directories to `0755`. + +Because Windows can't carry an executable bit, set the mode explicitly when you need a script or binary to be +runnable in the container regardless of the build host: + +- `FileManifest.AddExecutable(local, container)` — marks the entry `0755`. +- `FileManifest.Add(local, container, mode)` — any `System.IO.UnixFileMode`. +- `UnixPermissions.FromOctal("755")` / `FromOctal(755)` — express a mode in `chmod`-style octal (C# has no octal + literal, so `FromOctal` is how you write `0755`). For a directory mapping, an explicit mode applies to every file + it contains; directory entries themselves are always `0755`. diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs new file mode 100644 index 000000000..061aa46f8 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -0,0 +1,296 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using System.Web; + +namespace Amazon.OCI.Builder.Registry; + +/// +/// A that performs the Docker/OCI registry authentication handshake. +/// +/// +/// +/// When a registry responds 401 Unauthorized with a WWW-Authenticate challenge, this handler: +/// +/// +/// parses the challenge scheme (Bearer or Basic); +/// +/// for Bearer, validates the realm endpoint (see ) and performs a +/// token GET against it, sending the resolved credentials as Basic auth and reading back a token; +/// +/// for Basic, encodes the resolved credentials directly; +/// caches the resulting authorization header per registry host (with expiry for Bearer tokens) and retries the request. +/// +/// +/// Credentials come from an . Anonymous access is attempted when the +/// source returns (public base-image pulls do not need credentials to fetch a token). +/// +/// +public sealed partial class AuthHandshakeHandler : DelegatingHandler +{ + private const string BasicScheme = "Basic"; + private const string BearerScheme = "Bearer"; + + private readonly IRegistryCredentialSource _credentialSource; + + // Cached authorization header per registry host, with the time it stops being valid. + private readonly ConcurrentDictionary _cache = new(StringComparer.OrdinalIgnoreCase); + + /// Creates the handler. + /// Supplies credentials for each registry host. + /// The inner handler (typically an ). + public AuthHandshakeHandler(IRegistryCredentialSource credentialSource, HttpMessageHandler innerHandler) + : base(innerHandler) + => _credentialSource = credentialSource ?? throw new ArgumentNullException(nameof(credentialSource)); + + private readonly record struct CachedAuth(AuthenticationHeaderValue Header, DateTimeOffset Expiry); + + /// + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is null) + { + throw new ArgumentException("The request must have an absolute RequestUri.", nameof(request)); + } + + string registryHost = request.RequestUri.Host; + + // Apply a cached, still-valid authorization header if we have one. + if (_cache.TryGetValue(registryHost, out CachedAuth cached) && cached.Expiry > DateTimeOffset.UtcNow) + { + request.Headers.Authorization = cached.Header; + } + + HttpResponseMessage response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.Unauthorized) + { + return response; + } + + // 401 -> parse the challenge and perform the handshake. + if (!TryParseChallenge(response, out string? scheme, out AuthInfo? authInfo)) + { + return response; // No usable challenge; let the caller handle the 401. + } + + // Drain the 401 body so the connection is freed for the token request. + _ = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + response.Dispose(); + + (AuthenticationHeaderValue header, DateTimeOffset expiry) = + await AuthenticateAsync(registryHost, scheme!, authInfo, cancellationToken).ConfigureAwait(false); + + _cache[registryHost] = new CachedAuth(header, expiry); + + // Retry the original request with the authorization header. + HttpRequestMessage retry = await CloneAsync(request).ConfigureAwait(false); + retry.Headers.Authorization = header; + return await base.SendAsync(retry, cancellationToken).ConfigureAwait(false); + } + + private async Task<(AuthenticationHeaderValue, DateTimeOffset)> AuthenticateAsync( + string registryHost, string scheme, AuthInfo? authInfo, CancellationToken cancellationToken) + { + BasicRegistryCredentials? creds = + await _credentialSource.GetCredentialsAsync(registryHost, cancellationToken).ConfigureAwait(false); + + if (scheme.Equals(BasicScheme, StringComparison.OrdinalIgnoreCase)) + { + if (creds is not { } basic) + { + throw new RegistryAuthenticationException(registryHost, "the registry requires Basic auth but no credentials were available."); + } + return (BasicHeader(basic), DateTimeOffset.MaxValue); + } + + if (scheme.Equals(BearerScheme, StringComparison.OrdinalIgnoreCase)) + { + if (authInfo is not { } info) + { + throw new InvalidAuthResponseException(registryHost, "the Bearer challenge did not include a realm."); + } + return await GetBearerTokenAsync(registryHost, info, creds, cancellationToken).ConfigureAwait(false); + } + + throw new RegistryAuthenticationException(registryHost, $"unsupported authentication scheme '{scheme}'."); + } + + private async Task<(AuthenticationHeaderValue, DateTimeOffset)> GetBearerTokenAsync( + string registryHost, AuthInfo info, BasicRegistryCredentials? creds, CancellationToken cancellationToken) + { + // SSRF guard: never send credentials to an unvalidated realm. + Uri realm = RealmValidator.Validate(info.Realm, registryHost); + + var builder = new UriBuilder(realm); + System.Collections.Specialized.NameValueCollection query = HttpUtility.ParseQueryString(realm.Query); + if (info.Service is { } svc) + { + query["service"] = svc; + } + if (info.Scope is { } scope) + { + query["scope"] = scope; + } + builder.Query = query.ToString(); + + using var tokenRequest = new HttpRequestMessage(HttpMethod.Get, builder.Uri); + if (creds is { } c) + { + // Even for a Bearer scheme, username/password are presented to the token endpoint as Basic auth. + tokenRequest.Headers.Authorization = BasicHeader(c); + } + + using HttpResponseMessage tokenResponse = + await base.SendAsync(tokenRequest, cancellationToken).ConfigureAwait(false); + + if (!tokenResponse.IsSuccessStatusCode) + { + throw new RegistryAuthenticationException( + registryHost, $"the token endpoint returned {(int)tokenResponse.StatusCode} {tokenResponse.ReasonPhrase}."); + } + + TokenResponse? token; + await using (Stream s = await tokenResponse.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) + { + token = await JsonSerializer.DeserializeAsync(s, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + if (token is null || token.ResolvedToken is null) + { + throw new InvalidAuthResponseException(registryHost, "the token endpoint response could not be parsed into a token."); + } + + return (new AuthenticationHeaderValue(BearerScheme, token.ResolvedToken), token.ResolvedExpiration); + } + + private static AuthenticationHeaderValue BasicHeader(BasicRegistryCredentials c) + => new(BasicScheme, Convert.ToBase64String(Encoding.ASCII.GetBytes($"{c.Username}:{c.Password}"))); + + private static bool TryParseChallenge(HttpResponseMessage response, out string? scheme, out AuthInfo? authInfo) + { + scheme = null; + authInfo = null; + + HttpHeaderValueCollection challenges = response.Headers.WwwAuthenticate; + if (!challenges.Any()) + { + return false; + } + + AuthenticationHeaderValue header = challenges.First(); + if (header.Scheme is null) + { + return false; + } + + scheme = header.Scheme; + + if (header.Scheme.Equals(BasicScheme, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (header.Scheme.Equals(BearerScheme, StringComparison.OrdinalIgnoreCase)) + { + Dictionary parsed = ParseBearerParameters(header.Parameter); + if (!parsed.TryGetValue("realm", out string? realm)) + { + return false; + } + parsed.TryGetValue("service", out string? service); + parsed.TryGetValue("scope", out string? scopeValue); + authInfo = new AuthInfo(realm, service, scopeValue); + return true; + } + + return false; + } + + private static Dictionary ParseBearerParameters(string? parameters) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (parameters is null) + { + return result; + } + + foreach (Match match in BearerParameterSplitter().Matches(parameters)) + { + result[match.Groups["key"].Value] = match.Groups["value"].Value; + } + return result; + } + + private static async Task CloneAsync(HttpRequestMessage original) + { + var clone = new HttpRequestMessage(original.Method, original.RequestUri) + { + Version = original.Version, + }; + + foreach (KeyValuePair> header in original.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (original.Content is not null) + { + // Buffer the content so it can be sent again on the authenticated retry. + byte[] bytes = await original.Content.ReadAsByteArrayAsync().ConfigureAwait(false); + var content = new ByteArrayContent(bytes); + foreach (KeyValuePair> header in original.Content.Headers) + { + content.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + clone.Content = content; + } + + return clone; + } + + [GeneratedRegex("(?\\w+)=\"(?[^\"]*)\"(?:,|$)")] + private static partial Regex BearerParameterSplitter(); + + /// Parsed Bearer challenge parameters. + private sealed record AuthInfo(string Realm, string? Service, string? Scope); + + /// The JSON body returned by a registry token endpoint. + private sealed record TokenResponse + { + [JsonPropertyName("token")] + public string? Token { get; init; } + + [JsonPropertyName("access_token")] + public string? AccessToken { get; init; } + + [JsonPropertyName("expires_in")] + public int? ExpiresIn { get; init; } + + [JsonPropertyName("issued_at")] + public DateTimeOffset? IssuedAt { get; init; } + + /// The usable token (registries populate either token or access_token). + public string? ResolvedToken => Token ?? AccessToken; + + /// + /// The computed expiry. Per the token spec, default to 60 seconds of validity when the endpoint + /// omits expires_in; subtract a small safety margin so a cached token is refreshed before it + /// actually expires. + /// + public DateTimeOffset ResolvedExpiration + { + get + { + DateTimeOffset issued = IssuedAt ?? DateTimeOffset.UtcNow; + int validity = ExpiresIn ?? 60; + // 10-second safety margin, floored so very short tokens still move forward in time. + int margin = Math.Min(10, Math.Max(0, validity - 1)); + return issued.AddSeconds(validity - margin); + } + } + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs new file mode 100644 index 000000000..d072f0a8d --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs @@ -0,0 +1,60 @@ +namespace Amazon.OCI.Builder.Registry; + +/// +/// Supplies the username/password a registry's authentication handshake needs for a given registry host. +/// +/// +/// This decouples the from where credentials come from: +/// pushes to Amazon ECR use (holding the username/password decoded +/// from a caller-supplied ECR authorization token), while pulls of public base images use +/// . +/// +public interface IRegistryCredentialSource +{ + /// + /// Returns the credentials for , or to attempt + /// anonymous access. + /// + /// The registry host, for example 123456789012.dkr.ecr.us-west-2.amazonaws.com. + /// A cancellation token. + Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default); +} + +/// A plain username/password pair used for registry authentication. +/// The registry username. +/// The registry password or token. +public readonly record struct BasicRegistryCredentials(string Username, string Password); + +/// +/// A credential source that always returns , causing the handshake to attempt +/// anonymous access. Suitable for pulling public base images (for example from +/// mcr.microsoft.com or Docker Hub). +/// +public sealed class AnonymousRegistryCredentialSource : IRegistryCredentialSource +{ + /// + public Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default) + => Task.FromResult(null); +} + +/// +/// A credential source backed by a fixed username/password pair known up front (for example one decoded from an +/// Amazon ECR authorization token). It performs no network or credential resolution and has no dependency on the +/// AWS SDK; the caller is responsible for obtaining the token and keeping it fresh for the push's duration. +/// +public sealed class StaticRegistryCredentialSource : IRegistryCredentialSource +{ + private readonly BasicRegistryCredentials _credentials; + + /// Creates a source that returns the given username/password for every host. + public StaticRegistryCredentialSource(string username, string password) + { + ArgumentNullException.ThrowIfNull(username); + ArgumentNullException.ThrowIfNull(password); + _credentials = new BasicRegistryCredentials(username, password); + } + + /// + public Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default) + => Task.FromResult(_credentials); +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs b/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs new file mode 100644 index 000000000..bd72b6e52 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs @@ -0,0 +1,108 @@ +using System.Net; +using System.Net.Sockets; + +namespace Amazon.OCI.Builder.Registry; + +/// +/// Validates the realm (token endpoint) URL returned in a registry's WWW-Authenticate +/// challenge before the client sends credentials to it. +/// +/// +/// A compromised or malicious registry could return a realm that points at an internal or loopback +/// address in order to exfiltrate credentials or probe the host's private network (an SSRF vector). This +/// validator rejects any realm that is not HTTPS or whose host resolves to a loopback, link-local, private, +/// or unspecified address. The approach mirrors Microsoft.NET.Build.Containers. Because we only +/// target public registries (ECR, MCR, Docker Hub), there is no insecure/on-prem exception. +/// +internal static class RealmValidator +{ + private static readonly IPNetwork[] BlockedV4Networks = + [ + IPNetwork.Parse("0.0.0.0/8"), // "this network" / unspecified + IPNetwork.Parse("10.0.0.0/8"), // private (RFC 1918) + IPNetwork.Parse("172.16.0.0/12"), // private (RFC 1918) + IPNetwork.Parse("192.168.0.0/16"), // private (RFC 1918) + IPNetwork.Parse("169.254.0.0/16"), // link-local + IPNetwork.Parse("224.0.0.0/24"), // link-local multicast + ]; + + /// + /// Validates and returns it as a . + /// + /// + /// Thrown when the realm is unparseable, not HTTPS, or points at a blocked (loopback/private/link-local) host. + /// + public static Uri Validate(string realm, string registryName) + { + if (!Uri.TryCreate(realm, UriKind.Absolute, out Uri? realmUri)) + { + throw new InvalidAuthResponseException( + registryName, $"The authentication realm '{realm}' is relative or could not be parsed."); + } + + // Only HTTPS is permitted (we target public registries exclusively). + if (!realmUri.Scheme.Equals("https", StringComparison.Ordinal)) + { + throw new InvalidAuthResponseException( + registryName, $"The authentication realm '{realm}' uses disallowed scheme '{realmUri.Scheme}'; only https is allowed."); + } + + // Use IdnHost (the ASCII/canonical host) so Unicode-dot forms that the runtime canonicalizes + // back to a loopback/private literal cannot bypass the check by appearing as a DNS name. + string realmHost = TrimTrailingDot(realmUri.IdnHost); + + if (IPAddress.TryParse(realmHost, out IPAddress? realmIp) && IsBlockedIpLiteral(realmIp)) + { + throw new InvalidAuthResponseException( + registryName, $"The authentication realm '{realm}' resolves to a blocked (loopback/private/link-local) address '{realmHost}'."); + } + + if (IsLoopbackDnsName(realmHost)) + { + // RFC 6761 reserves "localhost"/"*.localhost" for loopback resolution regardless of /etc/hosts. + throw new InvalidAuthResponseException( + registryName, $"The authentication realm '{realm}' uses a loopback host name '{realmHost}'."); + } + + return realmUri; + } + + private static bool IsLoopbackDnsName(string host) + => host.Equals("localhost", StringComparison.OrdinalIgnoreCase) + || host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase); + + private static string TrimTrailingDot(string host) + => host.Length > 1 && host[^1] == '.' ? host[..^1] : host; + + private static bool IsBlockedIpLiteral(IPAddress ip) + { + if (IPAddress.IsLoopback(ip)) + { + return true; + } + + // IPv4-mapped IPv6: unwrap so we don't need a parallel set of IPv4-in-IPv6 CIDRs. + if (ip.IsIPv4MappedToIPv6) + { + return IsBlockedIpLiteral(ip.MapToIPv4()); + } + + if (ip.AddressFamily == AddressFamily.InterNetwork) + { + foreach (IPNetwork net in BlockedV4Networks) + { + if (net.Contains(ip)) + { + return true; + } + } + } + + // IPv6-only properties; all return false for IPv4 so the family gate is implicit. + return ip.Equals(IPAddress.IPv6Any) + || ip.IsIPv6LinkLocal + || ip.IsIPv6SiteLocal + || ip.IsIPv6UniqueLocal + || (ip.IsIPv6Multicast && (ip.GetAddressBytes()[1] & 0x0f) == 0x02); + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs new file mode 100644 index 000000000..825749373 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -0,0 +1,682 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Security.Cryptography; + +namespace Amazon.OCI.Builder.Registry; + +/// +/// A client for the OCI Distribution / Docker Registry HTTP API (v2): pull and push of manifests and blobs. +/// +/// +/// +/// One instance targets one registry host. Authentication is handled transparently by an +/// wrapped around the supplied , so callers do not +/// deal with tokens directly. +/// +/// +/// Blob uploads send the whole blob in a single PATCH (see ). Amazon ECR does +/// not honor Content-Range offsets across multiple PATCHes to one upload session, so chunked uploads +/// are deliberately avoided. +/// +/// +public sealed class RegistryClient : IDisposable +{ + private readonly HttpClient _http; + private readonly string _registryHost; + private readonly bool _ownsHttpClient; + + /// Creates a client for using the supplied authenticating client. + /// The registry host, for example registry-1.docker.io or an ECR endpoint. + /// + /// An whose handler chain includes an . + /// Use to build one correctly. The caller owns and must dispose a client passed here. + /// + public RegistryClient(string registryHost, HttpClient httpClient) + { + _registryHost = registryHost ?? throw new ArgumentNullException(nameof(registryHost)); + _http = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _ownsHttpClient = false; + } + + private RegistryClient(string registryHost, HttpClient httpClient, bool ownsHttpClient) + { + _registryHost = registryHost; + _http = httpClient; + _ownsHttpClient = ownsHttpClient; + } + + /// The registry host this client targets. + public string RegistryHost => _registryHost; + + /// + /// Builds a whose HTTP pipeline performs the authentication handshake using + /// . + /// + public static RegistryClient Create(string registryHost, IRegistryCredentialSource credentialSource) + { + var inner = new SocketsHttpHandler + { + AutomaticDecompression = DecompressionMethods.None, + // Registry endpoints can redirect blob downloads to object storage; allow it. + AllowAutoRedirect = true, + }; + var authHandler = new AuthHandshakeHandler(credentialSource, inner); + var http = new HttpClient(authHandler); + return new RegistryClient(registryHost, http, ownsHttpClient: true); + } + + private Uri Base => new($"https://{_registryHost}/v2/"); + + // ---- Manifests ---------------------------------------------------------------------------- + + /// + /// Fetches a manifest (or manifest list / image index) by tag or digest reference. The returned + /// carries the raw JSON, its media type, and the server-reported digest. + /// + public async Task GetManifestAsync(string repository, string reference, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(Base, $"{repository}/manifests/{reference}")); + AddManifestAcceptHeaders(request); + + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(response, $"GET manifest {repository}:{reference}", cancellationToken).ConfigureAwait(false); + + // Read the exact bytes on the wire. Digests must be computed over these, not over a string that has been + // decoded and re-encoded (which could differ by BOM/charset), so the raw bytes are exposed for verification. + byte[] raw = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + string json = System.Text.Encoding.UTF8.GetString(raw); + string mediaType = response.Content.Headers.ContentType?.MediaType + ?? throw new RegistryRequestException((int)response.StatusCode, "manifest response had no Content-Type."); + string? digest = response.Headers.TryGetValues("Docker-Content-Digest", out IEnumerable? values) + ? values.FirstOrDefault() + : null; + + return new ManifestResponse(json, raw, mediaType, digest); + } + + /// Checks whether a manifest exists for the given reference without downloading it. + public async Task ManifestExistsAsync(string repository, string reference, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Head, new Uri(Base, $"{repository}/manifests/{reference}")); + AddManifestAcceptHeaders(request); + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + return response.StatusCode == HttpStatusCode.OK; + } + + /// Uploads a manifest under the given tag or digest reference. + public async Task PutManifestAsync(string repository, string reference, string manifestJson, string mediaType, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Put, new Uri(Base, $"{repository}/manifests/{reference}")) + { + Content = new StringContent(manifestJson), + }; + request.Content.Headers.ContentType = new MediaTypeHeaderValue(mediaType); + + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(response, $"PUT manifest {repository}:{reference}", cancellationToken).ConfigureAwait(false); + } + + // ---- Blobs -------------------------------------------------------------------------------- + + /// Checks whether a blob with the given digest already exists in the repository. + public async Task BlobExistsAsync(string repository, string digest, CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Head, new Uri(Base, $"{repository}/blobs/{digest}")); + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + return response.StatusCode == HttpStatusCode.OK; + } + + /// Downloads a blob by digest as a stream. The caller owns and must dispose the returned stream's response. + public async Task GetBlobAsync(string repository, string digest, CancellationToken cancellationToken = default) + { + var request = new HttpRequestMessage(HttpMethod.Get, new Uri(Base, $"{repository}/blobs/{digest}")); + HttpResponseMessage response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(response, $"GET blob {repository} {digest}", cancellationToken).ConfigureAwait(false); + return await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Attempts a cross-repository blob mount so an already-present base-image layer is not re-uploaded. + /// Returns when the registry mounted the blob (HTTP 201), + /// when it requires a full upload instead (HTTP 202). + /// + public async Task TryMountBlobAsync(string repository, string digest, string fromRepository, CancellationToken cancellationToken = default) + { + var uri = new Uri(Base, $"{repository}/blobs/uploads/?mount={Uri.EscapeDataString(digest)}&from={Uri.EscapeDataString(fromRepository)}"); + using var request = new HttpRequestMessage(HttpMethod.Post, uri); + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.Created) + { + return true; // Mounted — no upload needed. + } + if (response.StatusCode == HttpStatusCode.Accepted) + { + return false; // Not mounted; caller must upload. + } + + await EnsureSuccessAsync(response, $"POST blob mount {repository} {digest}", cancellationToken).ConfigureAwait(false); + return false; + } + + /// + /// Uploads a blob using the POST/PATCH/PUT flow, sending the whole blob in a single + /// PATCH. Skips the upload entirely if the blob already exists. + /// + /// + /// + /// The blob is sent as one PATCH rather than multiple 5 MiB chunks. Amazon ECR does not honor the + /// Content-Range offset across successive PATCHes to the same upload session — each is treated as + /// writing from offset 0, so only the final chunk would survive and the commit would fail the digest check. + /// A single PATCH of the full content is what ECR expects and is what registries accept generally. + /// + /// + /// The request advertises a Content-Length (from , or the stream's + /// when it is seekable). This is important for memory: when the length is known + /// streams straight onto the socket, whereas without a + /// known length it must buffer the entire body into memory first to compute the length — which would + /// defeat the spill-to-disk download path by re-materializing a large layer in memory on the upload side. + /// + /// + /// The destination repository (for example myapp). + /// The sha256:... digest of the (compressed) blob content. + /// The blob bytes. + /// A cancellation token. + /// + /// Optional callback invoked as bytes are read from and sent, receiving the + /// cumulative number of bytes sent so far. Called on the thread driving the upload; keep it cheap and + /// non-throwing. When the blob already exists no bytes are sent and this is not called. + /// + /// + /// The exact length of in bytes, when the caller knows it. Supplying it lets the + /// upload stream without buffering the whole body in memory. When , a seekable stream's + /// is used; a non-seekable stream of unknown length falls back to buffering. + /// + public async Task UploadBlobAsync( + string repository, + string digest, + Stream content, + CancellationToken cancellationToken = default, + Action? onBytesSent = null, + long? contentLength = null) + { + if (await BlobExistsAsync(repository, digest, cancellationToken).ConfigureAwait(false)) + { + return; + } + + // 1. Start an upload session; the registry returns the upload URL in the Location header. + Uri uploadUri = await StartUploadAsync(repository, cancellationToken).ConfigureAwait(false); + + // 2. PATCH the entire blob in one request. (See remarks: ECR mishandles multi-PATCH chunking.) When a + // progress callback is supplied, wrap the source so bytes are reported as the HTTP stack reads them. + Stream patchContent = onBytesSent is null ? content : new ProgressReportingStream(content, onBytesSent); + + // Resolve the body length: prefer the caller-supplied value, else a seekable stream's own length. A known + // length lets HttpClient stream the body onto the socket instead of first buffering it all into memory. + long? bodyLength = contentLength ?? (content.CanSeek ? content.Length : null); + + using (var patch = new HttpRequestMessage(HttpMethod.Patch, uploadUri) + { + Content = new StreamContent(patchContent), + }) + { + patch.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + if (bodyLength is { } length) + { + // Advertising the length switches HttpClient from "buffer-to-measure" to streaming. + patch.Content.Headers.ContentLength = length; + } + + using HttpResponseMessage patchResponse = await _http.SendAsync(patch, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(patchResponse, $"PATCH blob {repository}", cancellationToken).ConfigureAwait(false); + + // The registry hands back the upload URL to use for the commit. + uploadUri = LocationFrom(patchResponse) ?? uploadUri; + } + + // 3. Close the session with PUT ?digest=..., which commits the blob. + Uri commitUri = AppendQuery(uploadUri, "digest", digest); + using var put = new HttpRequestMessage(HttpMethod.Put, commitUri) + { + Content = new ByteArrayContent(Array.Empty()), + }; + put.Content.Headers.ContentLength = 0; + using HttpResponseMessage putResponse = await _http.SendAsync(put, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(putResponse, $"PUT blob commit {repository} {digest}", cancellationToken).ConfigureAwait(false); + } + + private async Task StartUploadAsync(string repository, CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(HttpMethod.Post, new Uri(Base, $"{repository}/blobs/uploads/")); + using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + await EnsureSuccessAsync(response, $"POST blob upload start {repository}", cancellationToken).ConfigureAwait(false); + + return LocationFrom(response) + ?? throw new RegistryRequestException((int)response.StatusCode, "the registry did not return an upload Location header."); + } + + // ---- Helpers ------------------------------------------------------------------------------ + + private static void AddManifestAcceptHeaders(HttpRequestMessage request) + { + foreach (string mediaType in new[] + { + MediaTypes.OciManifestV1, + MediaTypes.OciImageIndexV1, + MediaTypes.DockerManifestV2, + MediaTypes.DockerManifestListV2, + }) + { + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(mediaType)); + } + } + + private Uri? LocationFrom(HttpResponseMessage response) + { + if (response.Headers.Location is not { } location) + { + return null; + } + // A registry may return a relative Location; resolve it against the registry base. + return location.IsAbsoluteUri ? location : new Uri(new Uri($"https://{_registryHost}"), location); + } + + private static Uri AppendQuery(Uri uri, string key, string value) + { + var builder = new UriBuilder(uri); + string encoded = $"{Uri.EscapeDataString(key)}={Uri.EscapeDataString(value)}"; + builder.Query = string.IsNullOrEmpty(builder.Query) ? encoded : builder.Query.TrimStart('?') + "&" + encoded; + return builder.Uri; + } + + private static async Task EnsureSuccessAsync(HttpResponseMessage response, string operation, CancellationToken cancellationToken) + { + if (response.IsSuccessStatusCode) + { + return; + } + + string body = string.Empty; + try + { + body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + // Ignore body read failures; the status code is the primary signal. + } + + throw new RegistryRequestException( + (int)response.StatusCode, + $"{operation} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {Truncate(body, 512)}"); + } + + private static string Truncate(string value, int max) + => value.Length <= max ? value : value[..max] + "…"; + + /// Computes the sha256:... digest of a stream (used by callers constructing blobs). + public static string ComputeDigest(Stream content) + { + byte[] hash = SHA256.HashData(content); + // Convert.ToHexStringLower exists only on net9.0+, so lower-case explicitly for net8.0 compatibility. + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + /// Computes the sha256:... digest of a byte buffer. + public static string ComputeDigest(ReadOnlySpan content) + { + Span hash = stackalloc byte[SHA256.HashSizeInBytes]; + SHA256.HashData(content, hash); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + /// + /// Verifies that hashes to , throwing + /// otherwise. The comparison is case-insensitive on the hex and + /// tolerant of an absent sha256: prefix on the expected value. Only the sha256 algorithm is + /// supported; an expected digest naming a different algorithm is rejected. + /// + /// A short description of the content, used in the error message (e.g. "config blob"). + /// The digest the content is expected to have (sha256:<hex>). + /// The downloaded bytes to verify. + public static void VerifyDigest(string what, string expectedDigest, ReadOnlySpan content) + { + ArgumentException.ThrowIfNullOrWhiteSpace(expectedDigest); + + int colon = expectedDigest.IndexOf(':'); + if (colon >= 0) + { + string algorithm = expectedDigest[..colon]; + if (!algorithm.Equals("sha256", StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryException( + $"Cannot verify {what}: unsupported digest algorithm '{algorithm}' (only sha256 is supported)."); + } + } + + string actual = ComputeDigest(content); + // Normalize both to "sha256:" for comparison. + string expectedNormalized = colon >= 0 + ? "sha256:" + expectedDigest[(colon + 1)..].ToLowerInvariant() + : "sha256:" + expectedDigest.ToLowerInvariant(); + + if (!actual.Equals(expectedNormalized, StringComparison.Ordinal)) + { + throw new DigestMismatchException(what, expectedNormalized, actual); + } + } + + /// + /// Downloads a blob by digest, buffers it fully, and verifies that the bytes hash to the requested digest + /// before returning them. Use this (rather than ) whenever the expected digest is + /// known, so corrupted or tampered content is rejected rather than trusted. + /// + /// The repository to read from. + /// The expected sha256:... digest; the download is verified against it. + /// A short description used in any integrity error (e.g. "base layer"). + /// A cancellation token. + /// The downloaded bytes do not hash to . + public async Task GetVerifiedBlobBytesAsync(string repository, string digest, string what, CancellationToken cancellationToken = default) + { + byte[] bytes; + await using (Stream stream = await GetBlobAsync(repository, digest, cancellationToken).ConfigureAwait(false)) + { + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + bytes = buffer.ToArray(); + } + + VerifyDigest(what, digest, bytes); + return bytes; + } + + /// + /// Downloads a blob by digest to a temporary file on disk (rather than buffering it in memory), verifying its + /// integrity twice: once against the bytes as they stream off the network, and again by re-reading the file + /// from disk before it is handed back. Use this for large layers where an in-memory buffer would be a memory + /// problem; small blobs should use . + /// + /// + /// + /// The second (re-read) verification is deliberate: the on-the-wire hash proves the download was not + /// corrupted, but the bytes could still be corrupted after that by a faulty disk write (a bit flip, + /// a torn or truncated write). Re-hashing the file as read back from disk — which is what will actually be + /// uploaded — closes that window, so a corrupt spilled layer is rejected here rather than committed to the + /// registry. + /// + /// + /// The returned handle owns the temporary file and an open read-only stream positioned at the start, ready to + /// pass to . Disposing the handle closes the stream and deletes the temp file; + /// the file is also cleaned up if verification fails. + /// + /// + /// The repository to read from. + /// The expected sha256:... digest; both verifications check against it. + /// A short description used in any integrity error (e.g. "base layer"). + /// A cancellation token. + /// + /// The downloaded bytes, or the bytes as read back from disk, do not hash to . + /// + public async Task GetVerifiedBlobToFileAsync(string repository, string digest, string what, CancellationToken cancellationToken = default) + { + // A unique temp path under the system temp dir (honors TMPDIR). Created now so we can always clean it up. + string tempPath = Path.Combine(Path.GetTempPath(), $"aws-oci-layer-{Guid.NewGuid():N}.tmp"); + + try + { + // 1. Stream the blob to disk, hashing the exact bytes written as we go (download-side integrity). + string downloadedDigest; + await using (Stream source = await GetBlobAsync(repository, digest, cancellationToken).ConfigureAwait(false)) + await using (var file = new FileStream( + tempPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 81920, useAsync: true)) + using (var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256)) + { + byte[] rented = System.Buffers.ArrayPool.Shared.Rent(81920); + try + { + int read; + while ((read = await source.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0) + { + hasher.AppendData(rented, 0, read); + await file.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + } + } + finally + { + System.Buffers.ArrayPool.Shared.Return(rented); + } + + // Push the OS buffers to disk before we re-read, so the re-read actually exercises the stored bytes. + await file.FlushAsync(cancellationToken).ConfigureAwait(false); + file.Flush(flushToDisk: true); + + downloadedDigest = "sha256:" + Convert.ToHexString(hasher.GetHashAndReset()).ToLowerInvariant(); + } + + VerifyDigest(what, digest, downloadedDigest); + + // 2. Re-read the file from disk and verify again — catches a faulty disk write that corrupted the + // spilled bytes after the download-side hash. + await VerifyFileDigestAsync(what + " (spilled to disk)", digest, tempPath, cancellationToken).ConfigureAwait(false); + + // 3. Hand back an open read stream over the verified file. The handle owns deletion. + var readStream = new FileStream( + tempPath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, useAsync: true); + return new VerifiedBlobFile(tempPath, readStream); + } + catch + { + TryDeleteFile(tempPath); + throw; + } + } + + /// + /// Verifies that an expected digest matches an already-computed digest string (both normalized), throwing + /// on mismatch. Used when the content was hashed incrementally while + /// streaming, so there is no in-memory buffer to re-hash. + /// + private static void VerifyDigest(string what, string expectedDigest, string actualDigest) + { + ArgumentException.ThrowIfNullOrWhiteSpace(expectedDigest); + + int colon = expectedDigest.IndexOf(':'); + if (colon >= 0 && !expectedDigest[..colon].Equals("sha256", StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryException( + $"Cannot verify {what}: unsupported digest algorithm '{expectedDigest[..colon]}' (only sha256 is supported)."); + } + + string expectedNormalized = colon >= 0 + ? "sha256:" + expectedDigest[(colon + 1)..].ToLowerInvariant() + : "sha256:" + expectedDigest.ToLowerInvariant(); + + if (!actualDigest.Equals(expectedNormalized, StringComparison.Ordinal)) + { + throw new DigestMismatchException(what, expectedNormalized, actualDigest); + } + } + + /// Hashes a file's contents from disk and verifies the digest, without buffering the whole file. + private static async Task VerifyFileDigestAsync(string what, string expectedDigest, string path, CancellationToken cancellationToken) + { + string actual; + await using (var file = new FileStream( + path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, useAsync: true)) + { + byte[] hash = await SHA256.HashDataAsync(file, cancellationToken).ConfigureAwait(false); + actual = "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + VerifyDigest(what, expectedDigest, actual); + } + + private static void TryDeleteFile(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch + { + // Best-effort cleanup; a leftover temp file is not worth masking the original error. + } + } + + /// + /// Disposes the underlying only when this instance created it (via + /// ). A client supplied through the public constructor is left for its owner to dispose. + /// + public void Dispose() + { + if (_ownsHttpClient) + { + _http.Dispose(); + } + } +} + +/// The result of fetching a manifest: its raw JSON, media type, and server-reported digest. +/// The manifest JSON decoded as UTF-8 (convenient for parsing). +/// The exact bytes received on the wire — digests must be computed over these. +/// The manifest media type (OCI or Docker; single image or index). +/// The server-reported Docker-Content-Digest, if present. +public readonly record struct ManifestResponse(string Json, byte[] RawBytes, string MediaType, string? Digest); + +/// +/// A verified blob that has been spilled to a temporary file on disk. Owns both the open read stream (positioned +/// at the start, ready to upload) and the temp file; disposing it closes the stream and deletes the file. +/// +public sealed class VerifiedBlobFile : IAsyncDisposable, IDisposable +{ + private readonly string _path; + private bool _disposed; + + internal VerifiedBlobFile(string path, FileStream stream) + { + _path = path; + Stream = stream; + } + + /// The open, read-only stream over the verified file, positioned at offset 0. + public FileStream Stream { get; } + + /// The full path of the temporary file backing . + public string Path => _path; + + /// Closes the stream and deletes the temporary file. + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + _disposed = true; + await Stream.DisposeAsync().ConfigureAwait(false); + Delete(); + } + + /// Closes the stream and deletes the temporary file. + public void Dispose() + { + if (_disposed) + { + return; + } + _disposed = true; + Stream.Dispose(); + Delete(); + } + + private void Delete() + { + try + { + if (File.Exists(_path)) + { + File.Delete(_path); + } + } + catch + { + // Best-effort cleanup. + } + } +} + +/// +/// A read-only pass-through stream that reports cumulative bytes read to a callback. Wrapping the upload source +/// in this lets the registry client surface byte-level upload progress: the HTTP stack reads from it to send the +/// PATCH body, and each read advances the reported total. The wrapped stream is not disposed by this wrapper — +/// its owner retains responsibility for it. +/// +internal sealed class ProgressReportingStream : Stream +{ + private readonly Stream _inner; + private readonly Action _onBytesRead; + private long _totalRead; + + public ProgressReportingStream(Stream inner, Action onBytesRead) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _onBytesRead = onBytesRead ?? throw new ArgumentNullException(nameof(onBytesRead)); + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + + public override long Position + { + get => _inner.Position; + set => _inner.Position = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + int read = _inner.Read(buffer, offset, count); + Report(read); + return read; + } + + public override int Read(Span buffer) + { + int read = _inner.Read(buffer); + Report(read); + return read; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int read = await _inner.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + Report(read); + return read; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int read = await _inner.ReadAsync(buffer.AsMemory(offset, count), cancellationToken).ConfigureAwait(false); + Report(read); + return read; + } + + private void Report(int read) + { + if (read <= 0) + { + return; + } + _totalRead += read; + _onBytesRead(_totalRead); + } + + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Flush() => _inner.Flush(); +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs new file mode 100644 index 000000000..1c4ec980a --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs @@ -0,0 +1,72 @@ +namespace Amazon.OCI.Builder.Registry; + +/// +/// The base type for errors raised while interacting with a container registry — authentication, +/// malformed responses, failed requests, or higher-level build problems such as a missing platform. +/// +public class RegistryException : Exception +{ + /// Creates the exception with a message. + public RegistryException(string message) : base(message) { } + + /// Creates the exception with a message and an inner cause. + public RegistryException(string message, Exception innerException) : base(message, innerException) { } +} + +/// Thrown when a registry's authentication challenge or token response is malformed or unsafe. +public sealed class InvalidAuthResponseException : RegistryException +{ + /// The registry that produced the invalid response. + public string RegistryName { get; } + + /// Creates the exception. + public InvalidAuthResponseException(string registryName, string message) + : base($"Invalid authentication response from registry '{registryName}': {message}") + => RegistryName = registryName; +} + +/// Thrown when the client cannot authenticate to, or is denied access by, a registry. +public sealed class RegistryAuthenticationException : RegistryException +{ + /// The registry that denied access. + public string RegistryName { get; } + + /// Creates the exception. + public RegistryAuthenticationException(string registryName, string message) + : base($"Unable to authenticate to registry '{registryName}': {message}") + => RegistryName = registryName; +} + +/// Thrown when a registry request fails with an unexpected HTTP status. +public sealed class RegistryRequestException : RegistryException +{ + /// The HTTP status code returned by the registry. + public int StatusCode { get; } + + /// Creates the exception. + public RegistryRequestException(int statusCode, string message) + : base(message) + => StatusCode = statusCode; +} + +/// +/// Thrown when downloaded content does not hash to its expected digest — i.e. a content-integrity check failed. +/// This indicates the registry, a cache/CDN, or the transport returned bytes that do not match the +/// content-addressable digest they were requested under, and the content must not be trusted. +/// +public sealed class DigestMismatchException : RegistryException +{ + /// The digest the content was expected to have. + public string ExpectedDigest { get; } + + /// The digest actually computed over the downloaded bytes. + public string ActualDigest { get; } + + /// Creates the exception. + public DigestMismatchException(string what, string expectedDigest, string actualDigest) + : base($"Integrity check failed for {what}: expected {expectedDigest} but the downloaded content hashed to {actualDigest}.") + { + ExpectedDigest = expectedDigest; + ActualDigest = actualDigest; + } +} diff --git a/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs b/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs new file mode 100644 index 000000000..f80c60022 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs @@ -0,0 +1,79 @@ +namespace Amazon.OCI.Builder; + +/// +/// Helpers for expressing Unix file permissions the way chmod does — in octal — and converting them to the +/// flags the layer builder uses. +/// +/// +/// C# has no octal integer literal, so the familiar 0755 cannot be written directly (and 0755 in C# +/// would be the decimal 755, the wrong bits). Use with "755", or +/// where each decimal digit is read as one octal permission digit (so the C# literal +/// 755 means octal 0755). Each of the three permission digits is a 0–7 value whose bits are +/// read(4)/write(2)/execute(1) for owner, group, and other respectively. +/// +public static class UnixPermissions +{ + /// + /// Parses an octal permission string such as "755", "0644", or "0o700" into a + /// . A leading 0 or 0o prefix is accepted and ignored. Only the nine + /// standard permission bits are supported; set-uid/set-gid/sticky bits (a fourth leading digit other than 0) + /// are rejected. + /// + /// Three octal digits, optionally prefixed with 0 or 0o. + /// The value is null/empty or not valid octal permission notation. + public static UnixFileMode FromOctal(string octal) + { + ArgumentException.ThrowIfNullOrWhiteSpace(octal); + + string digits = octal.Trim(); + // Accept a leading "0o"/"0O" prefix, or a single leading "0". + if (digits.StartsWith("0o", StringComparison.OrdinalIgnoreCase)) + { + digits = digits[2..]; + } + else if (digits.Length == 4 && digits[0] == '0') + { + digits = digits[1..]; + } + + if (digits.Length != 3) + { + throw new ArgumentException( + $"'{octal}' is not a valid octal permission. Use three octal digits, e.g. \"755\" or \"0644\" " + + "(set-uid/set-gid/sticky bits are not supported).", + nameof(octal)); + } + + int value = 0; + foreach (char c in digits) + { + if (c is < '0' or > '7') + { + throw new ArgumentException( + $"'{octal}' contains '{c}', which is not an octal digit (0–7).", nameof(octal)); + } + value = (value << 3) | (c - '0'); + } + + // The nine low bits map directly onto UnixFileMode, which uses the same rwx-per-triad layout. + return (UnixFileMode)value; + } + + /// + /// Parses a permission value whose decimal digits are read as octal — so the C# literal 755 + /// means octal 0755, and 644 means 0644. This lets callers write the number they would + /// type into chmod without a string. Equivalent to FromOctal(value.ToString()). + /// + /// A value in the range 0–777 whose decimal digits are each 0–7. + /// The value is out of range or has a digit outside 0–7. + public static UnixFileMode FromOctal(int octalDigits) + { + if (octalDigits is < 0 or > 777) + { + throw new ArgumentException( + $"{octalDigits} is out of range; pass the chmod-style digits, e.g. 755 or 644 (0–777).", + nameof(octalDigits)); + } + return FromOctal(octalDigits.ToString("D3", System.Globalization.CultureInfo.InvariantCulture)); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.IntegrationTests/AWS.OCI.Builder.IntegrationTests.csproj b/Libraries/test/AWS.OCI.Builder.IntegrationTests/AWS.OCI.Builder.IntegrationTests.csproj new file mode 100644 index 000000000..2578b3bb6 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.IntegrationTests/AWS.OCI.Builder.IntegrationTests.csproj @@ -0,0 +1,27 @@ + + + + + + $(DefaultPackageTargets) + false + + false + false + enable + enable + + + + + + + + + + + + + + + diff --git a/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs b/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs new file mode 100644 index 000000000..6369e77a8 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs @@ -0,0 +1,365 @@ +using Amazon; +using Amazon.ECR; +using Amazon.ECR.Model; +using Amazon.OCI.Builder; +using Xunit; +using Xunit.Abstractions; + +namespace AWS.OCI.Builder.IntegrationTests; + +/// +/// End-to-end integration test that pushes a real image to Amazon ECR using the library's public API, then +/// verifies the image landed by querying ECR. AWS credentials and region are taken from the environment (the AWS SDK default chain); set +/// AWS_SHARED_CREDENTIALS_FILE / AWS_PROFILE / AWS_REGION as needed before running. +/// +/// +/// What it does, end to end: +/// +/// Ensures a dedicated ECR repository exists (creates it if missing). +/// Pulls a small public base image, overlays a generated file as a new layer, and pushes to the repo +/// under a unique tag via . +/// Calls ECR DescribeImages to confirm the pushed tag exists and its digest matches what +/// returned. +/// Cleans up by deleting just the tag it pushed (the repository is left in place). +/// +/// +/// Overridable via environment variables: OCI_ECR_REPOSITORY (default aws-oci-builder-itest), +/// OCI_ECR_REGION (default us-west-2), OCI_ECR_BASE_IMAGE +/// (default mcr.microsoft.com/dotnet/runtime-deps:8.0). +/// +/// +[Trait("Category", "Integration")] +public class EcrPushIntegrationTests +{ + private const string DefaultRepository = "aws-oci-builder-itest"; + private const string DefaultRegion = "us-west-2"; + private const string DefaultBaseImage = "mcr.microsoft.com/dotnet/runtime-deps:8.0"; + private const string Platform = "linux/amd64"; + + private readonly ITestOutputHelper _output; + + public EcrPushIntegrationTests(ITestOutputHelper output) => _output = output; + + [Fact] + public async Task Pushes_image_to_real_ecr_and_verifies_it() + { + string repository = EnvOr("OCI_ECR_REPOSITORY", DefaultRepository); + string regionName = EnvOr("OCI_ECR_REGION", DefaultRegion); + string baseImage = EnvOr("OCI_ECR_BASE_IMAGE", DefaultBaseImage); + RegionEndpoint region = RegionEndpoint.GetBySystemName(regionName); + string tag = "itest-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss") + "-" + Guid.NewGuid().ToString("N")[..6]; + + using var ecr = new AmazonECRClient(region); + + // ---- 1. Resolve the registry host and ensure the repository exists. ---- + string accountId = await GetAccountIdAsync(ecr); + string registryHost = $"{accountId}.dkr.ecr.{regionName}.amazonaws.com"; + string targetImage = $"{registryHost}/{repository}:{tag}"; + _output.WriteLine($"Target: {targetImage}"); + + await EnsureRepositoryAsync(ecr, repository); + + string? pushedDigest = null; + try + { + // ---- 2. Build from the base image + a generated layer, and push. ---- + string payloadFile = Path.Combine(Path.GetTempPath(), $"oci-itest-{Guid.NewGuid():N}.txt"); + await File.WriteAllTextAsync(payloadFile, "Hello from the AWS.OCI.Builder integration test.\n"); + + try + { + using OciImageBuilder builder = await OciImageBuilder.FromBaseImageAsync(baseImage, Platform); + await builder.AddLayerAsync( + new FileManifest { { payloadFile, "/app/hello.txt" } }, + createdBy: "AWS.OCI.Builder integration test"); + builder.Config.SetWorkingDirectory("/app"); + builder.Config.SetLabel("com.amazonaws.oci-builder.itest", tag); + + // Region is inferred from the ECR host in the target; the auth token is fetched via the SDK. + string authToken = await GetAuthorizationTokenStringAsync(ecr); + + // Collect push progress so we can assert the event contract against real ECR byte streaming. + var events = new List(); + var options = new EcrPushOptions + { + Progress = new Progress(e => + { + lock (events) { events.Add(e); } + }), + }; + pushedDigest = await builder.PushToEcrAsync(targetImage, authToken, options); + + // Verify the progress contract end to end. (Progress marshals callbacks, so snapshot under lock.) + List seen; + lock (events) { seen = events.ToList(); } + + Assert.NotEmpty(seen); + int totalLayers = seen[0].TotalLayers; + Assert.All(seen, e => Assert.Equal(totalLayers, e.TotalLayers)); + Assert.All(seen, e => Assert.InRange(e.LayerIndex, 1, totalLayers)); + + // Every layer (1..totalLayers) terminates on exactly one Completed event. + for (int index = 1; index <= totalLayers; index++) + { + LayerPushProgress completed = Assert.Single( + seen, e => e.LayerIndex == index && e.Phase == LayerPushPhase.Completed); + // A non-skipped layer's byte count reaches its size; a skipped one reports 0 of a known size. + if (completed.IsSkipped) + { + Assert.Equal(0, completed.BytesPushed); + } + else + { + Assert.Equal(completed.LayerSizeBytes, completed.BytesPushed); + } + } + _output.WriteLine($"Progress: {seen.Count} events across {totalLayers} layers."); + } + finally + { + File.Delete(payloadFile); + } + + _output.WriteLine($"Pushed digest: {pushedDigest}"); + Assert.StartsWith("sha256:", pushedDigest); + + // ---- 3. Verify via ECR that the tag exists and the digest matches. ---- + DescribeImagesResponse described = await ecr.DescribeImagesAsync(new DescribeImagesRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + + ImageDetail detail = Assert.Single(described.ImageDetails); + Assert.Contains(tag, detail.ImageTags); + Assert.Equal(pushedDigest, detail.ImageDigest); + _output.WriteLine($"Verified image {detail.ImageDigest} ({detail.ImageSizeInBytes} bytes) in ECR."); + } + finally + { + // ---- 4. Best-effort cleanup: delete only the tag this test pushed. ---- + try + { + await ecr.BatchDeleteImageAsync(new BatchDeleteImageRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + _output.WriteLine($"Cleaned up tag {tag}."); + } + catch (Exception ex) + { + _output.WriteLine($"Cleanup of tag {tag} failed (non-fatal): {ex.Message}"); + } + } + } + + [Fact] + public async Task Pushes_image_to_real_ecr_using_the_spill_to_disk_path_for_large_base_layers() + { + string repository = EnvOr("OCI_ECR_REPOSITORY", DefaultRepository); + string regionName = EnvOr("OCI_ECR_REGION", DefaultRegion); + string baseImage = EnvOr("OCI_ECR_BASE_IMAGE", DefaultBaseImage); + RegionEndpoint region = RegionEndpoint.GetBySystemName(regionName); + string tag = "itest-spill-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss") + "-" + Guid.NewGuid().ToString("N")[..6]; + + using var ecr = new AmazonECRClient(region); + string accountId = await GetAccountIdAsync(ecr); + string registryHost = $"{accountId}.dkr.ecr.{regionName}.amazonaws.com"; + string targetImage = $"{registryHost}/{repository}:{tag}"; + _output.WriteLine($"Target: {targetImage}"); + + await EnsureRepositoryAsync(ecr, repository); + + string? pushedDigest = null; + try + { + string payloadFile = Path.Combine(Path.GetTempPath(), $"oci-itest-{Guid.NewGuid():N}.txt"); + await File.WriteAllTextAsync(payloadFile, "Hello from the AWS.OCI.Builder spill-to-disk integration test.\n"); + + try + { + using OciImageBuilder builder = await OciImageBuilder.FromBaseImageAsync(baseImage, Platform); + await builder.AddLayerAsync( + new FileManifest { { payloadFile, "/app/hello.txt" } }, + createdBy: "AWS.OCI.Builder spill-to-disk integration test"); + builder.Config.SetWorkingDirectory("/app"); + + // Force the spill-to-disk copy path: a 1 MiB threshold is well below the base image's real layer + // sizes (its largest runtime-deps layer is tens of MiB), so those layers are streamed through a + // temp file, re-verified from disk, and uploaded from the file rather than buffered in memory. + var options = new EcrPushOptions { MaxInMemoryLayerBytes = 1L * 1024 * 1024 }; + string authToken = await GetAuthorizationTokenStringAsync(ecr); + pushedDigest = await builder.PushToEcrAsync(targetImage, authToken, options); + } + finally + { + File.Delete(payloadFile); + } + + _output.WriteLine($"Pushed digest (spill path): {pushedDigest}"); + Assert.StartsWith("sha256:", pushedDigest); + + // Verify via ECR that the tag exists and the digest matches — proving the disk-spilled layers produced + // a valid, consistent image in ECR (every blob committed its digest check). + DescribeImagesResponse described = await ecr.DescribeImagesAsync(new DescribeImagesRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + + ImageDetail detail = Assert.Single(described.ImageDetails); + Assert.Contains(tag, detail.ImageTags); + Assert.Equal(pushedDigest, detail.ImageDigest); + _output.WriteLine($"Verified spill-path image {detail.ImageDigest} ({detail.ImageSizeInBytes} bytes) in ECR."); + } + finally + { + try + { + await ecr.BatchDeleteImageAsync(new BatchDeleteImageRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + _output.WriteLine($"Cleaned up tag {tag}."); + } + catch (Exception ex) + { + _output.WriteLine($"Cleanup of tag {tag} failed (non-fatal): {ex.Message}"); + } + } + } + + [Fact] + public async Task Pushed_executable_file_keeps_its_0755_mode_in_ecr() + { + string repository = EnvOr("OCI_ECR_REPOSITORY", DefaultRepository); + string regionName = EnvOr("OCI_ECR_REGION", DefaultRegion); + string baseImage = EnvOr("OCI_ECR_BASE_IMAGE", DefaultBaseImage); + RegionEndpoint region = RegionEndpoint.GetBySystemName(regionName); + string tag = "itest-exec-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss") + "-" + Guid.NewGuid().ToString("N")[..6]; + + using var ecr = new AmazonECRClient(region); + string accountId = await GetAccountIdAsync(ecr); + string registryHost = $"{accountId}.dkr.ecr.{regionName}.amazonaws.com"; + string targetImage = $"{registryHost}/{repository}:{tag}"; + await EnsureRepositoryAsync(ecr, repository); + + string? newLayerDigest = null; + try + { + string scriptFile = Path.Combine(Path.GetTempPath(), $"oci-itest-{Guid.NewGuid():N}.sh"); + await File.WriteAllTextAsync(scriptFile, "#!/bin/sh\necho hi\n"); + try + { + using OciImageBuilder builder = await OciImageBuilder.FromBaseImageAsync(baseImage, Platform); + // Mark the script executable regardless of host — this is the behavior that must survive to ECR, + // and on Windows there would be no source executable bit to rely on. + var files = new FileManifest(); + files.AddExecutable(scriptFile, "/app/run.sh"); + await builder.AddLayerAsync(files, createdBy: "exec-bit itest"); + string authToken = await GetAuthorizationTokenStringAsync(ecr); + await builder.PushToEcrAsync(targetImage, authToken); + } + finally + { + File.Delete(scriptFile); + } + + // Pull the pushed manifest, find the LAST layer (the one we added), download its blob, and inspect the + // mode of our file entry inside the gzipped tar. + var auth = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); + string basic = auth.AuthorizationData[0].AuthorizationToken; + + using var http = new HttpClient(); + http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", basic); + foreach (string accept in new[] + { + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + }) + { + http.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue(accept)); + } + + string manifestJson = await http.GetStringAsync($"https://{registryHost}/v2/{repository}/manifests/{tag}"); + using var manifestDoc = System.Text.Json.JsonDocument.Parse(manifestJson); + var layers = manifestDoc.RootElement.GetProperty("layers"); + newLayerDigest = layers[layers.GetArrayLength() - 1].GetProperty("digest").GetString(); + _output.WriteLine($"New layer blob: {newLayerDigest}"); + + byte[] gzipped = await http.GetByteArrayAsync($"https://{registryHost}/v2/{repository}/blobs/{newLayerDigest}"); + UnixFileMode mode = ReadEntryMode(gzipped, "app/run.sh"); + _output.WriteLine($"app/run.sh mode in ECR: {mode}"); + + Assert.True(mode.HasFlag(UnixFileMode.UserExecute), $"expected the owner-execute bit to survive, got {mode}"); + } + finally + { + try + { + await ecr.BatchDeleteImageAsync(new BatchDeleteImageRequest + { + RepositoryName = repository, + ImageIds = new List { new() { ImageTag = tag } }, + }); + } + catch (Exception ex) { _output.WriteLine($"Cleanup failed (non-fatal): {ex.Message}"); } + } + } + + /// Decompresses a gzipped tar blob and returns the Unix mode of the entry named . + private static UnixFileMode ReadEntryMode(byte[] gzippedTar, string entryName) + { + using var input = new MemoryStream(gzippedTar); + using var gzip = new System.IO.Compression.GZipStream(input, System.IO.Compression.CompressionMode.Decompress); + using var reader = new System.Formats.Tar.TarReader(gzip); + while (reader.GetNextEntry() is { } entry) + { + if (entry.Name.TrimStart('\0') == entryName) + { + return entry.Mode; + } + } + throw new Xunit.Sdk.XunitException($"Entry '{entryName}' not found in the pulled layer."); + } + + private static async Task GetAccountIdAsync(IAmazonECR ecr) + { + // GetAuthorizationToken's response identifies the registry (account) the caller is authenticated against. + GetAuthorizationTokenResponse token = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); + AuthorizationData data = Assert.Single(token.AuthorizationData); + + // proxyEndpoint is https://.dkr.ecr..amazonaws.com — extract the account id from the host. + var proxy = new Uri(data.ProxyEndpoint); + return proxy.Host.Split('.')[0]; + } + + /// Fetches the ECR authorization token string (base64 "AWS:password") to pass to PushToEcrAsync. + private static async Task GetAuthorizationTokenStringAsync(IAmazonECR ecr) + { + GetAuthorizationTokenResponse token = await ecr.GetAuthorizationTokenAsync(new GetAuthorizationTokenRequest()); + AuthorizationData data = Assert.Single(token.AuthorizationData); + return data.AuthorizationToken; + } + + private async Task EnsureRepositoryAsync(IAmazonECR ecr, string repository) + { + try + { + await ecr.CreateRepositoryAsync(new CreateRepositoryRequest { RepositoryName = repository }); + _output.WriteLine($"Created repository {repository}."); + } + catch (RepositoryAlreadyExistsException) + { + _output.WriteLine($"Repository {repository} already exists."); + } + } + + private static string EnvOr(string name, string fallback) + { + string? value = Environment.GetEnvironmentVariable(name); + return string.IsNullOrWhiteSpace(value) ? fallback : value; + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj b/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj new file mode 100644 index 000000000..2578b3bb6 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj @@ -0,0 +1,27 @@ + + + + + + $(DefaultPackageTargets) + false + + false + false + enable + enable + + + + + + + + + + + + + + + diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs new file mode 100644 index 000000000..fb68ba1f5 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs @@ -0,0 +1,108 @@ +using System.Net; +using System.Net.Http.Headers; +using Amazon.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class AuthHandshakeHandlerTests +{ + /// A programmable inner handler that responds based on the request URI. + private sealed class StubHandler : HttpMessageHandler + { + private readonly Func _responder; + public List Requests { get; } = new(); + + public StubHandler(Func responder) => _responder = responder; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(_responder(request)); + } + } + + private sealed class StaticCredentials : IRegistryCredentialSource + { + private readonly BasicRegistryCredentials? _creds; + public StaticCredentials(BasicRegistryCredentials? creds) => _creds = creds; + public Task GetCredentialsAsync(string registryHost, CancellationToken ct = default) + => Task.FromResult(_creds); + } + + [Fact] + public async Task Performs_bearer_handshake_and_retries_with_token() + { + var stub = new StubHandler(request => + { + string url = request.RequestUri!.ToString(); + + // The token endpoint: return a token when Basic auth was presented. + if (url.StartsWith("https://auth.example.com/token")) + { + Assert.Equal("Basic", request.Headers.Authorization?.Scheme); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + // The resource endpoint: 401 until a Bearer token is presented. + if (request.Headers.Authorization?.Scheme == "Bearer") + { + Assert.Equal("abc123", request.Headers.Authorization.Parameter); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("ok") }; + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", + "realm=\"https://auth.example.com/token\",service=\"registry.example.com\",scope=\"repository:app:pull\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/app/manifests/latest"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("ok", await response.Content.ReadAsStringAsync()); + + // Expect: initial 401, token GET, authenticated retry. + Assert.Equal(3, stub.Requests.Count); + } + + [Fact] + public async Task Rejects_bearer_realm_pointing_at_loopback() + { + var stub = new StubHandler(_ => + { + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://127.0.0.1/token\",service=\"evil\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + await Assert.ThrowsAsync( + () => client.GetAsync("https://registry.example.com/v2/app/manifests/latest")); + } + + [Fact] + public async Task Passes_through_non_401_responses() + { + var stub = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("direct") }); + var handler = new AuthHandshakeHandler(new AnonymousRegistryCredentialSource(), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Single(stub.Requests); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs new file mode 100644 index 000000000..f94892f86 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs @@ -0,0 +1,133 @@ +using System.Text; +using Amazon.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class DigestVerificationTests +{ + [Fact] + public void ComputeDigest_of_known_bytes() + { + // SHA-256 of the empty input is a well-known constant. + string digest = RegistryClient.ComputeDigest(ReadOnlySpan.Empty); + Assert.Equal("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", digest); + } + + [Fact] + public void ComputeDigest_byte_and_stream_overloads_agree() + { + byte[] data = Encoding.UTF8.GetBytes("the quick brown fox"); + string fromBytes = RegistryClient.ComputeDigest(data); + string fromStream = RegistryClient.ComputeDigest(new MemoryStream(data)); + Assert.Equal(fromStream, fromBytes); + } + + [Fact] + public void VerifyDigest_passes_for_matching_content() + { + byte[] data = Encoding.UTF8.GetBytes("layer-bytes"); + string digest = RegistryClient.ComputeDigest(data); + + // Should not throw. + RegistryClient.VerifyDigest("test blob", digest, data); + } + + [Fact] + public void VerifyDigest_is_case_insensitive_on_hex() + { + byte[] data = Encoding.UTF8.GetBytes("layer-bytes"); + string digest = RegistryClient.ComputeDigest(data).ToUpperInvariant(); // SHA256: + RegistryClient.VerifyDigest("test blob", digest, data); + } + + [Fact] + public void VerifyDigest_throws_on_mismatch() + { + byte[] data = Encoding.UTF8.GetBytes("real-bytes"); + byte[] other = Encoding.UTF8.GetBytes("tampered-bytes"); + string wrongDigest = RegistryClient.ComputeDigest(other); + + DigestMismatchException ex = Assert.Throws( + () => RegistryClient.VerifyDigest("test blob", wrongDigest, data)); + + Assert.Equal(wrongDigest, ex.ExpectedDigest); + Assert.Equal(RegistryClient.ComputeDigest(data), ex.ActualDigest); + } + + [Fact] + public void VerifyDigest_rejects_unsupported_algorithm() + { + byte[] data = Encoding.UTF8.GetBytes("x"); + RegistryException ex = Assert.Throws( + () => RegistryClient.VerifyDigest("test blob", "sha512:deadbeef", data)); + Assert.Contains("sha256", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void DigestMismatchException_is_a_RegistryException() + => Assert.IsAssignableFrom(new DigestMismatchException("x", "sha256:a", "sha256:b")); + + /// A stub registry handler that serves a single blob's bytes under a given digest on GET. + private sealed class SingleBlobHandler : HttpMessageHandler + { + private readonly string _digest; + private readonly byte[] _bytes; + + public SingleBlobHandler(string digest, byte[] bytes) + { + _digest = digest; + _bytes = bytes; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.Method == HttpMethod.Get && request.RequestUri!.AbsolutePath.EndsWith($"/blobs/{_digest}", StringComparison.Ordinal)) + { + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new ByteArrayContent(_bytes), + }); + } + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.NotFound)); + } + } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_returns_verified_stream_and_deletes_file_on_dispose() + { + byte[] data = Encoding.UTF8.GetBytes("a large-ish base layer spilled to disk"); + string digest = RegistryClient.ComputeDigest(data); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(digest, data))); + + string path; + await using (VerifiedBlobFile handle = await client.GetVerifiedBlobToFileAsync("repo", digest, "base layer")) + { + path = handle.Path; + Assert.True(File.Exists(path)); + Assert.Equal(0, handle.Stream.Position); + + // The stream serves exactly the verified bytes. + using var buffer = new MemoryStream(); + await handle.Stream.CopyToAsync(buffer); + Assert.Equal(data, buffer.ToArray()); + } + + // Disposing the handle deletes the temp file. + Assert.False(File.Exists(path)); + } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_throws_when_bytes_do_not_match_digest() + { + byte[] served = Encoding.UTF8.GetBytes("corrupted-on-the-wire"); + // Declare a digest for DIFFERENT content, so the download hash will not match. + string declaredDigest = RegistryClient.ComputeDigest(Encoding.UTF8.GetBytes("the-real-content")); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(declaredDigest, served))); + + await Assert.ThrowsAsync( + () => client.GetVerifiedBlobToFileAsync("repo", declaredDigest, "base layer")); + // Cleanup-on-failure is covered deterministically by the success test's dispose assertion; a global + // temp-dir file count here would race with other parallel tests sharing the temp directory. + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs new file mode 100644 index 000000000..535213698 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs @@ -0,0 +1,21 @@ +using Amazon.OCI.Builder; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class FileManifestTests +{ + [Fact] + public void Collection_initializer_adds_entries_in_order() + { + var files = new FileManifest + { + { "/local/app.dll", "/app/app.dll" }, + { "/local/config", "/app/config" }, + }; + + Assert.Equal(2, files.Entries.Count); + Assert.Equal("/local/app.dll", files.Entries[0].LocalPath); + Assert.Equal("/app/config", files.Entries[1].ContainerPath); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs new file mode 100644 index 000000000..6782a7994 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs @@ -0,0 +1,263 @@ +using System.Security.Cryptography; +using System.Text.Json.Nodes; +using Amazon.OCI.Builder; +using Amazon.OCI.Builder.Images; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class ImageConfigTests +{ + private const string MinimalConfig = """ + {"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:aaa"]},"history":[{"created":"2020-01-01T00:00:00Z"}],"config":{"Env":["PATH=/usr/bin"]}} + """; + + [Fact] + public void Parse_exposes_os_and_architecture() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + Assert.Equal("linux", config.Os); + Assert.Equal("amd64", config.Architecture); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("[1,2,3]")] // not an object + [InlineData("{ not json")] + public void Parse_rejects_invalid_input(string json) + => Assert.Throws(() => ImageConfig.Parse(json)); + + [Fact] + public void AddLayer_appends_diffId_and_history_entry() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + config.AddLayer("sha256:bbb", "ADD app layer"); + + JsonObject root = (JsonObject)JsonNode.Parse(config.ToJsonString())!; + JsonArray diffIds = (JsonArray)root["rootfs"]!["diff_ids"]!; + JsonArray history = (JsonArray)root["history"]!; + + Assert.Equal(2, diffIds.Count); + Assert.Equal("sha256:bbb", diffIds[1]!.GetValue()); + Assert.Equal(2, history.Count); + Assert.Equal("ADD app layer", history[1]!["created_by"]!.GetValue()); + } + + [Fact] + public void SetEnvironmentVariable_replaces_existing_key_in_place() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + config.SetEnvironmentVariable("PATH", "/custom/bin"); // overwrite existing + config.SetEnvironmentVariable("ASPNETCORE_URLS", "http://+:8080"); // add new + + JsonObject root = (JsonObject)JsonNode.Parse(config.ToJsonString())!; + JsonArray env = (JsonArray)root["config"]!["Env"]!; + + Assert.Equal(2, env.Count); + Assert.Contains("PATH=/custom/bin", env.Select(e => e!.GetValue())); + Assert.DoesNotContain("PATH=/usr/bin", env.Select(e => e!.GetValue())); + Assert.Contains("ASPNETCORE_URLS=http://+:8080", env.Select(e => e!.GetValue())); + } + + [Fact] + public void Sets_entrypoint_cmd_workdir_user_labels_and_ports() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + config.SetEntrypoint("dotnet", "/app/app.dll"); + config.SetCmd("--urls", "http://+:80"); + config.SetWorkingDirectory("/app"); + config.SetUser("1000"); + config.SetLabel("org.opencontainers.image.source", "https://example.com/repo"); + config.ExposePort("8080"); + + JsonObject c = (JsonObject)JsonNode.Parse(config.ToJsonString())!["config"]!; + + Assert.Equal(new[] { "dotnet", "/app/app.dll" }, ((JsonArray)c["Entrypoint"]!).Select(e => e!.GetValue())); + Assert.Equal(new[] { "--urls", "http://+:80" }, ((JsonArray)c["Cmd"]!).Select(e => e!.GetValue())); + Assert.Equal("/app", c["WorkingDir"]!.GetValue()); + Assert.Equal("1000", c["User"]!.GetValue()); + Assert.Equal("https://example.com/repo", c["Labels"]!["org.opencontainers.image.source"]!.GetValue()); + Assert.NotNull(c["ExposedPorts"]!["8080/tcp"]); // bare port got /tcp suffix + } + + [Fact] + public void Serialization_is_compact() + { + ImageConfig config = ImageConfig.Parse(MinimalConfig); + string json = config.ToJsonString(); + Assert.DoesNotContain("\n", json); + Assert.DoesNotContain(" ", json); + } +} + +public class ManifestTests +{ + [Fact] + public void Round_trips_config_and_layers() + { + var manifest = new Manifest + { + MediaType = MediaTypes.OciManifestV1, + Config = new Descriptor { MediaType = MediaTypes.OciImageConfigV1, Size = 10, Digest = "sha256:cfg" }, + Layers = + { + new Descriptor { MediaType = MediaTypes.OciLayerGzipV1, Size = 100, Digest = "sha256:l1" }, + }, + }; + + Manifest parsed = Manifest.Parse(manifest.ToJsonString()); + + Assert.Equal(2, parsed.SchemaVersion); + Assert.Equal(MediaTypes.OciManifestV1, parsed.MediaType); + Assert.Equal("sha256:cfg", parsed.Config.Digest); + Assert.Single(parsed.Layers); + Assert.Equal("sha256:l1", parsed.Layers[0].Digest); + } + + [Fact] + public void Property_order_is_schemaVersion_mediaType_config_layers() + { + var manifest = new Manifest + { + MediaType = MediaTypes.OciManifestV1, + Config = new Descriptor { MediaType = MediaTypes.OciImageConfigV1, Size = 1, Digest = "sha256:cfg" }, + }; + string json = manifest.ToJsonString(); + + int iSchema = json.IndexOf("schemaVersion", StringComparison.Ordinal); + int iMedia = json.IndexOf("mediaType", StringComparison.Ordinal); + int iConfig = json.IndexOf("\"config\"", StringComparison.Ordinal); + int iLayers = json.IndexOf("\"layers\"", StringComparison.Ordinal); + + Assert.True(iSchema < iMedia && iMedia < iConfig && iConfig < iLayers); + } +} + +public class ImageIndexTests +{ + private const string IndexJson = """ + {"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[ + {"mediaType":"application/vnd.oci.image.manifest.v1+json","size":1,"digest":"sha256:amd","platform":{"os":"linux","architecture":"amd64"}}, + {"mediaType":"application/vnd.oci.image.manifest.v1+json","size":1,"digest":"sha256:arm","platform":{"os":"linux","architecture":"arm64"}}, + {"mediaType":"application/vnd.oci.image.manifest.v1+json","size":1,"digest":"sha256:att","platform":{"os":"unknown","architecture":"unknown"}} + ]} + """; + + [Fact] + public void FindManifest_selects_matching_platform() + { + ImageIndex index = ImageIndex.Parse(IndexJson); + PlatformManifestDescriptor? amd = index.FindManifest("linux", "amd64"); + PlatformManifestDescriptor? arm = index.FindManifest("linux", "arm64"); + + Assert.Equal("sha256:amd", amd!.Digest); + Assert.Equal("sha256:arm", arm!.Digest); + } + + [Fact] + public void FindManifest_returns_null_for_absent_platform() + { + ImageIndex index = ImageIndex.Parse(IndexJson); + Assert.Null(index.FindManifest("windows", "amd64")); + } +} + +public class ImageBuilderTests +{ + private const string BaseConfig = """ + {"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:base1"]},"history":[{"created":"2020-01-01T00:00:00Z"}]} + """; + + private static string BaseManifest(string layerMediaType, string configMediaType) => $$""" + {"schemaVersion":2,"mediaType":"{{MediaTypes.OciManifestV1}}","config":{"mediaType":"{{configMediaType}}","size":10,"digest":"sha256:basecfg"},"layers":[{"mediaType":"{{layerMediaType}}","size":100,"digest":"sha256:baselayer"}]} + """; + + private static Layer FakeLayer(string mediaType) => + // Build a real tiny layer so Digest/DiffId are genuine sha256 values. + Layer.CreateAsync(MakeManifest(), mediaType).GetAwaiter().GetResult(); + + private static FileManifest MakeManifest() + { + string tmp = Path.Combine(Path.GetTempPath(), "aws-oci-ib-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tmp); + string file = Path.Combine(tmp, "app.dll"); + File.WriteAllText(file, "payload"); + return new FileManifest { { file, "/app/app.dll" } }; + } + + [Fact] + public void AddLayer_keeps_manifest_layers_and_config_diffids_in_lockstep() + { + ImageBuilder builder = ImageBuilder.FromBaseImage( + BaseConfig, + BaseManifest(MediaTypes.OciLayerGzipV1, MediaTypes.OciImageConfigV1), + MediaTypes.OciManifestV1); + + Layer layer = FakeLayer(MediaTypes.OciLayerGzipV1); + builder.AddLayer(layer, "ADD app"); + + BuiltImage built = builder.Build(); + Manifest manifest = Manifest.Parse(System.Text.Encoding.UTF8.GetString(built.ManifestBytes)); + JsonObject config = (JsonObject)System.Text.Json.Nodes.JsonNode.Parse(System.Text.Encoding.UTF8.GetString(built.ConfigBytes))!; + + // Base had 1 layer / 1 diff_id; after overlay both are 2 and the new entries match the layer. + Assert.Equal(2, manifest.Layers.Count); + Assert.Equal(layer.Digest, manifest.Layers[1].Digest); + + JsonArray diffIds = (JsonArray)config["rootfs"]!["diff_ids"]!; + Assert.Equal(2, diffIds.Count); + Assert.Equal(layer.DiffId, diffIds[1]!.GetValue()); + } + + [Fact] + public void Build_points_manifest_config_descriptor_at_actual_config_bytes() + { + ImageBuilder builder = ImageBuilder.FromBaseImage( + BaseConfig, + BaseManifest(MediaTypes.OciLayerGzipV1, MediaTypes.OciImageConfigV1), + MediaTypes.OciManifestV1); + builder.AddLayer(FakeLayer(MediaTypes.OciLayerGzipV1)); + + BuiltImage built = builder.Build(); + Manifest manifest = Manifest.Parse(System.Text.Encoding.UTF8.GetString(built.ManifestBytes)); + + // The manifest's config descriptor must match the real config bytes (size + digest), or registries reject it. + string expectedDigest = "sha256:" + Convert.ToHexString(SHA256.HashData(built.ConfigBytes)).ToLowerInvariant(); + Assert.Equal(expectedDigest, manifest.Config.Digest); + Assert.Equal(built.ConfigBytes.LongLength, manifest.Config.Size); + Assert.Equal(built.ConfigDigest, manifest.Config.Digest); + } + + [Fact] + public void Detects_oci_schema_and_uses_oci_media_types() + { + ImageBuilder builder = ImageBuilder.FromBaseImage( + BaseConfig, + BaseManifest(MediaTypes.OciLayerGzipV1, MediaTypes.OciImageConfigV1), + MediaTypes.OciManifestV1); + + Assert.Equal(MediaTypes.OciLayerGzipV1, builder.LayerMediaType); + Assert.Equal(MediaTypes.OciImageConfigV1, builder.ConfigMediaType); + Assert.Equal(MediaTypes.OciManifestV1, builder.ManifestMediaType); + } + + [Fact] + public void Detects_docker_schema_from_reported_media_type() + { + ImageBuilder builder = ImageBuilder.FromBaseImage( + BaseConfig, + BaseManifest(MediaTypes.DockerLayerGzip, MediaTypes.DockerImageConfigV1), + MediaTypes.DockerManifestV2); + + Assert.Equal(MediaTypes.DockerLayerGzip, builder.LayerMediaType); + Assert.Equal(MediaTypes.DockerImageConfigV1, builder.ConfigMediaType); + Assert.Equal(MediaTypes.DockerManifestV2, builder.ManifestMediaType); + + // A layer built as OCI is coerced to the Docker layer media type in the manifest. + builder.AddLayer(FakeLayer(MediaTypes.OciLayerGzipV1)); + BuiltImage built = builder.Build(); + Manifest manifest = Manifest.Parse(System.Text.Encoding.UTF8.GetString(built.ManifestBytes)); + Assert.Equal(MediaTypes.DockerLayerGzip, manifest.Layers[^1].MediaType); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs new file mode 100644 index 000000000..cbda95bff --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs @@ -0,0 +1,83 @@ +using Amazon.OCI.Builder; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class ImageReferenceTests +{ + [Theory] + [InlineData("mcr.microsoft.com/dotnet/aspnet:8.0", "mcr.microsoft.com", "dotnet/aspnet", "8.0", false)] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest", + "123456789012.dkr.ecr.us-west-2.amazonaws.com", "myapp", "latest", false)] + [InlineData("library/ubuntu:22.04", "registry-1.docker.io", "library/ubuntu", "22.04", false)] + public void Parses_explicit_and_hub_references( + string input, string registry, string repository, string reference, bool isDigest) + { + ImageReference parsed = ImageReference.Parse(input); + + Assert.Equal(registry, parsed.Registry); + Assert.Equal(repository, parsed.Repository); + Assert.Equal(reference, parsed.Reference); + Assert.Equal(isDigest, parsed.ReferenceIsDigest); + } + + [Fact] + public void Single_segment_hub_repo_gets_library_prefix_and_latest_tag() + { + ImageReference parsed = ImageReference.Parse("ubuntu"); + + Assert.Equal("registry-1.docker.io", parsed.Registry); + Assert.Equal("library/ubuntu", parsed.Repository); + Assert.Equal("latest", parsed.Reference); + } + + [Fact] + public void Parses_digest_reference() + { + const string digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + ImageReference parsed = ImageReference.Parse($"mcr.microsoft.com/dotnet/runtime@{digest}"); + + Assert.Equal("dotnet/runtime", parsed.Repository); + Assert.Equal(digest, parsed.Reference); + Assert.True(parsed.ReferenceIsDigest); + } + + [Theory] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/app:latest", "us-west-2")] + [InlineData("123456789012.dkr.ecr.eu-central-1.amazonaws.com/app:latest", "eu-central-1")] + [InlineData("210987654321.dkr.ecr-fips.us-east-1.amazonaws.com/app:latest", "us-east-1")] + public void Detects_ecr_and_extracts_region(string input, string expectedRegion) + { + ImageReference parsed = ImageReference.Parse(input); + + Assert.True(parsed.IsEcr); + Assert.Equal(expectedRegion, parsed.EcrRegion); + } + + [Theory] + [InlineData("mcr.microsoft.com/dotnet/aspnet:8.0")] + [InlineData("registry-1.docker.io/library/ubuntu:22.04")] + public void Non_ecr_references_are_not_ecr(string input) + { + ImageReference parsed = ImageReference.Parse(input); + + Assert.False(parsed.IsEcr); + Assert.Null(parsed.EcrRegion); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Rejects_null_or_blank(string input) + => Assert.Throws(() => ImageReference.Parse(input)); + + [Fact] + public void Preserves_non_default_registry_port() + { + ImageReference parsed = ImageReference.Parse("localhost:5000/myapp:dev"); + + Assert.Equal("localhost:5000", parsed.Registry); + Assert.Equal("myapp", parsed.Repository); + Assert.Equal("dev", parsed.Reference); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs new file mode 100644 index 000000000..373ea5d40 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs @@ -0,0 +1,340 @@ +using System.Formats.Tar; +using System.IO.Compression; +using Amazon.OCI.Builder; +using Amazon.OCI.Builder.Images; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class LayerTests : IDisposable +{ + private readonly string _tempRoot; + + public LayerTests() + { + // Each test gets an isolated temp directory for its source files. + _tempRoot = Path.Combine(Path.GetTempPath(), "aws-oci-builder-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempRoot); + } + + public void Dispose() + { + try { Directory.Delete(_tempRoot, recursive: true); } catch { /* best effort cleanup */ } + } + + private string WriteFile(string relativePath, string content) + { + string full = Path.Combine(_tempRoot, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(full)!); + File.WriteAllText(full, content); + return full; + } + + [Fact] + public async Task Builds_layer_with_both_digests_populated() + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + + Layer layer = await Layer.CreateAsync(manifest); + + Assert.StartsWith("sha256:", layer.Digest); + Assert.StartsWith("sha256:", layer.DiffId); + Assert.Equal(64 + "sha256:".Length, layer.Digest.Length); + Assert.True(layer.Size > 0); + Assert.Equal(layer.GzippedTarBytes.LongLength, layer.Size); + Assert.Equal(MediaTypes.OciLayerGzipV1, layer.MediaType); + } + + [Fact] + public async Task Digest_and_diffId_differ_because_one_is_compressed() + { + string file = WriteFile("app.dll", new string('x', 4096)); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + + Layer layer = await Layer.CreateAsync(manifest); + + // Digest is over the gzipped bytes; DiffId is over the raw tar. They must not be equal. + Assert.NotEqual(layer.Digest, layer.DiffId); + } + + [Fact] + public async Task Descriptor_reflects_layer_fields() + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + + Layer layer = await Layer.CreateAsync(manifest, MediaTypes.DockerLayerGzip); + Descriptor descriptor = layer.ToDescriptor(); + + Assert.Equal(MediaTypes.DockerLayerGzip, descriptor.MediaType); + Assert.Equal(layer.Size, descriptor.Size); + Assert.Equal(layer.Digest, descriptor.Digest); + } + + [Fact] + public async Task Is_reproducible_identical_inputs_yield_identical_digests() + { + string file = WriteFile("app.dll", "deterministic"); + var manifest1 = new FileManifest { { file, "/app/app.dll" } }; + var manifest2 = new FileManifest { { file, "/app/app.dll" } }; + + Layer a = await Layer.CreateAsync(manifest1); + Layer b = await Layer.CreateAsync(manifest2); + + Assert.Equal(a.Digest, b.Digest); + Assert.Equal(a.DiffId, b.DiffId); + } + + [Fact] + public async Task Entry_order_does_not_affect_digest() + { + string f1 = WriteFile("a.txt", "one"); + string f2 = WriteFile("b.txt", "two"); + + var ordered = new FileManifest { { f1, "/app/a.txt" }, { f2, "/app/b.txt" } }; + var reversed = new FileManifest { { f2, "/app/b.txt" }, { f1, "/app/a.txt" } }; + + Layer a = await Layer.CreateAsync(ordered); + Layer b = await Layer.CreateAsync(reversed); + + Assert.Equal(a.Digest, b.Digest); + } + + [Fact] + public async Task Tar_contains_files_at_relative_container_paths_and_parent_dirs() + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, "/app/bin/app.dll" } }; + + Layer layer = await Layer.CreateAsync(manifest); + List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); + + // Parent directories are synthesized (no leading slash, trailing slash for dirs), file has no leading slash. + Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/" }); + Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/bin/" }); + Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/bin/app.dll" }); + } + + [Fact] + public async Task Directory_mapping_recurses_preserving_subpaths() + { + WriteFile("content/root.txt", "r"); + WriteFile("content/sub/leaf.txt", "l"); + string dir = Path.Combine(_tempRoot, "content"); + + var manifest = new FileManifest { { dir, "/app/data" } }; + + Layer layer = await Layer.CreateAsync(manifest); + List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); + + Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/data/root.txt" }); + Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/data/sub/leaf.txt" }); + Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/data/sub/" }); + } + + [Fact] + public async Task Empty_manifest_throws() + { + var manifest = new FileManifest(); + await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + } + + [Fact] + public async Task Missing_source_throws_FileNotFound() + { + var manifest = new FileManifest { { Path.Combine(_tempRoot, "does-not-exist"), "/app/x" } }; + await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + } + + [Theory] + [InlineData("relative/path")] // not absolute + [InlineData("/app/../etc")] // escaping segment + [InlineData("/")] // names nothing + public async Task Invalid_container_path_throws(string containerPath) + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, containerPath } }; + await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + } + + [Fact] + public async Task Duplicate_container_path_throws() + { + string f1 = WriteFile("a.txt", "one"); + string f2 = WriteFile("b.txt", "two"); + var manifest = new FileManifest { { f1, "/app/same.txt" }, { f2, "/app/same.txt" } }; + await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + } + + /// Decompresses the gzipped tar and returns its real entries (type + name) in order. + /// + /// PAX archives carry extended-attribute header blocks. Depending on the runtime, + /// surfaces these differently: .NET 9+ folds them into the following entry, while .NET 8's reader can emit a + /// separate pseudo-entry (named PaxHeaders.NNN/...) AND then read the real entry with a stray leading + /// NUL in its name and a non-standard entry-type byte. Those are quirks of the .NET 8 reader, not of the + /// archive bytes (which .NET 9+ and real registries parse correctly). This helper filters the metadata + /// pseudo-entries and normalizes the NUL-prefixed names so the assertions test the layer's real structure. + /// + private static List<(TarEntryType Type, string Name)> ReadTarEntries(byte[] gzippedTar) + => ReadTarEntriesWithMode(gzippedTar).Select(e => (e.Type, e.Name)).ToList(); + + /// Like but also returns each entry's Unix . + private static List<(TarEntryType Type, string Name, UnixFileMode Mode)> ReadTarEntriesWithMode(byte[] gzippedTar) + { + var result = new List<(TarEntryType, string, UnixFileMode)>(); + using var input = new MemoryStream(gzippedTar); + using var gzip = new GZipStream(input, CompressionMode.Decompress); + using var reader = new TarReader(gzip); + while (reader.GetNextEntry() is { } entry) + { + string name = entry.Name.TrimStart('\0'); + + bool isMetadata = + entry.EntryType is TarEntryType.ExtendedAttributes or TarEntryType.GlobalExtendedAttributes + || name.Contains("PaxHeaders", StringComparison.Ordinal); + if (isMetadata) + { + continue; + } + + // Normalize the .NET 8 reader artifact: a name not ending in '/' is a regular file even when the + // reader reports a stray type byte for it. + TarEntryType type = entry.EntryType; + if (type is not TarEntryType.Directory && !name.EndsWith('/')) + { + type = TarEntryType.RegularFile; + } + + result.Add((type, name, entry.Mode)); + } + return result; + } + + // --- File permission coverage (matters especially when building on Windows) --------------- + + [Fact] + public async Task Default_file_mode_is_0644_and_directory_mode_is_0755() + { + string file = WriteFile("app.dll", "hello"); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + + Layer layer = await Layer.CreateAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + // On Windows the source has no mode bits, so the file defaults to 0644. On Unix the source file written by + // the test is created 0644 by default too (no execute bit), so this holds on every platform. + (TarEntryType Type, string Name, UnixFileMode Mode) fileEntry = + Assert.Single(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/app.dll" }); + Assert.Equal(Mode0644, fileEntry.Mode); + + (TarEntryType Type, string Name, UnixFileMode Mode) dirEntry = + Assert.Single(entries, e => e is { Type: TarEntryType.Directory, Name: "app/" }); + Assert.Equal(Mode0755, dirEntry.Mode); + } + + [Fact] + public async Task Explicit_mode_overrides_the_default() + { + string file = WriteFile("run.sh", "#!/bin/sh\necho hi\n"); + var manifest = new FileManifest { { file, "/app/run.sh", Mode0750 } }; + + Layer layer = await Layer.CreateAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + var fileEntry = Assert.Single(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/run.sh" }); + Assert.Equal(Mode0750, fileEntry.Mode); + } + + [Fact] + public async Task AddExecutable_marks_the_file_0755() + { + string file = WriteFile("entrypoint.sh", "#!/bin/sh\n"); + var manifest = new FileManifest(); + manifest.AddExecutable(file, "/app/entrypoint.sh"); + + Layer layer = await Layer.CreateAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + var fileEntry = Assert.Single(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/entrypoint.sh" }); + Assert.Equal(Mode0755, fileEntry.Mode); + } + + [Fact] + public async Task Explicit_mode_applies_to_every_file_in_a_mapped_directory() + { + WriteFile("scripts/a.sh", "#!/bin/sh\n"); + WriteFile("scripts/nested/b.sh", "#!/bin/sh\n"); + string dir = Path.Combine(_tempRoot, "scripts"); + + var manifest = new FileManifest { { dir, "/app/scripts", FileManifestEntry.ExecutableMode } }; + + Layer layer = await Layer.CreateAsync(manifest); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + var a = Assert.Single(entries, e => e.Name == "app/scripts/a.sh"); + var b = Assert.Single(entries, e => e.Name == "app/scripts/nested/b.sh"); + Assert.Equal(Mode0755, a.Mode); + Assert.Equal(Mode0755, b.Mode); + } + + [Fact] + public void Executable_bit_from_source_is_preserved_on_unix() + { + // On Windows there is no source executable bit to read, so this behavior only applies to Unix hosts. + if (OperatingSystem.IsWindows()) + { + return; + } + + string file = WriteFile("tool", "#!/bin/sh\necho tool\n"); + File.SetUnixFileMode(file, Mode0755); + + var manifest = new FileManifest { { file, "/app/tool" } }; // no explicit mode -> derive from source + Layer layer = Layer.CreateAsync(manifest).GetAwaiter().GetResult(); + var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); + + var fileEntry = Assert.Single(entries, e => e.Name == "app/tool"); + // The owner-execute bit from the source must survive into the layer. + Assert.True(fileEntry.Mode.HasFlag(UnixFileMode.UserExecute)); + } + + [Fact] + public async Task Explicit_mode_is_deterministic_in_the_digest() + { + string file = WriteFile("run.sh", "#!/bin/sh\n"); + var m1 = new FileManifest(); + m1.AddExecutable(file, "/app/run.sh"); + var m2 = new FileManifest(); + m2.AddExecutable(file, "/app/run.sh"); + + Layer a = await Layer.CreateAsync(m1); + Layer b = await Layer.CreateAsync(m2); + + Assert.Equal(a.Digest, b.Digest); + } + + [Fact] + public async Task Changing_the_mode_changes_the_digest() + { + string file = WriteFile("run.sh", "#!/bin/sh\n"); + var plain = new FileManifest { { file, "/app/run.sh" } }; + var executable = new FileManifest(); + executable.AddExecutable(file, "/app/run.sh"); + + Layer a = await Layer.CreateAsync(plain); + Layer b = await Layer.CreateAsync(executable); + + // Same content, different permissions -> different layer bytes -> different digest. + Assert.NotEqual(a.Digest, b.Digest); + } + + private const UnixFileMode Mode0644 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead; + private const UnixFileMode Mode0755 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + private const UnixFileMode Mode0750 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute; +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs new file mode 100644 index 000000000..1924ca142 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs @@ -0,0 +1,729 @@ +using System.Net; +using System.Text; +using Amazon.OCI.Builder; +using Amazon.OCI.Builder.Images; +using Amazon.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class OciImageBuilderTests +{ + /// + /// A minimal in-memory OCI Distribution registry. It tracks blob existence per repository, accepts the + /// POST/PATCH/PUT upload flow, serves base-layer GETs for the source host, and records the PUT manifest. + /// One instance serves all hosts; it keys state by (host, repository). + /// + private sealed class FakeRegistry : HttpMessageHandler + { + // (host, repository) -> set of blob digests present. + private readonly Dictionary<(string Host, string Repo), HashSet> _blobs = new(); + // Pre-seeded source blobs available for download: (host, repo, digest) -> bytes. + private readonly Dictionary<(string Host, string Repo, string Digest), byte[]> _sourceBlobs = new(); + private int _uploadCounter; + + public List<(string Host, string Repo, string Digest)> UploadedBlobs { get; } = new(); + + // Content-Length advertised on each PATCH body (null when the request did not set one). A non-null value + // means HttpClient was able to stream the body instead of buffering it to measure the length. + public List PatchContentLengths { get; } = new(); + + public (string Host, string Repo, string Reference, string Body, string ContentType)? PutManifest { get; private set; } + + public void SeedSourceBlob(string host, string repo, string digest, byte[] bytes) + => _sourceBlobs[(host, repo, digest)] = bytes; + + private HashSet BlobsFor(string host, string repo) + { + if (!_blobs.TryGetValue((host, repo), out HashSet? set)) + { + set = new HashSet(StringComparer.Ordinal); + _blobs[(host, repo)] = set; + } + return set; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Uri uri = request.RequestUri!; + string host = uri.Host; + string path = uri.AbsolutePath; // /v2//(blobs|manifests|blobs/uploads)/... + HttpMethod method = request.Method; + + // ---- Blob existence: HEAD /v2//blobs/ ---- + if (method == HttpMethod.Head && TryMatch(path, "/blobs/", out string repoHeadBlob, out string headDigest)) + { + bool exists = BlobsFor(host, repoHeadBlob).Contains(headDigest); + return new HttpResponseMessage(exists ? HttpStatusCode.OK : HttpStatusCode.NotFound); + } + + // ---- Blob download: GET /v2//blobs/ ---- + if (method == HttpMethod.Get && TryMatch(path, "/blobs/", out string repoGetBlob, out string getDigest)) + { + if (_sourceBlobs.TryGetValue((host, repoGetBlob, getDigest), out byte[]? bytes)) + { + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(bytes) }; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + // ---- Start upload: POST /v2//blobs/uploads/ ---- + if (method == HttpMethod.Post && path.EndsWith("/blobs/uploads/", StringComparison.Ordinal)) + { + string repo = ExtractRepo(path, "/blobs/uploads/"); + string uploadId = $"upload{Interlocked.Increment(ref _uploadCounter)}"; + var accepted = new HttpResponseMessage(HttpStatusCode.Accepted); + accepted.Headers.Location = new Uri($"https://{host}/v2/{repo}/blobs/uploads/{uploadId}"); + return accepted; + } + + // ---- Chunk: PATCH /v2//blobs/uploads/ ---- + if (method == HttpMethod.Patch && path.Contains("/blobs/uploads/", StringComparison.Ordinal)) + { + // Record the advertised body length before draining (it is cleared once the content is consumed). + PatchContentLengths.Add(request.Content!.Headers.ContentLength); + // Drain the chunk; return the same location for the next chunk/commit. + _ = await request.Content!.ReadAsByteArrayAsync(cancellationToken); + var accepted = new HttpResponseMessage(HttpStatusCode.Accepted); + accepted.Headers.Location = uri; + return accepted; + } + + // ---- Commit: PUT /v2//blobs/uploads/?digest= ---- + if (method == HttpMethod.Put && path.Contains("/blobs/uploads/", StringComparison.Ordinal)) + { + string repo = ExtractRepo(path, "/blobs/uploads/"); + string digest = System.Web.HttpUtility.ParseQueryString(uri.Query).Get("digest")!; + BlobsFor(host, repo).Add(digest); + UploadedBlobs.Add((host, repo, digest)); + return new HttpResponseMessage(HttpStatusCode.Created); + } + + // ---- Manifest PUT: PUT /v2//manifests/ ---- + if (method == HttpMethod.Put && TryMatch(path, "/manifests/", out string repoManifest, out string reference)) + { + string body = await request.Content!.ReadAsStringAsync(cancellationToken); + string contentType = request.Content.Headers.ContentType?.MediaType ?? string.Empty; + PutManifest = (host, repoManifest, reference, body, contentType); + return new HttpResponseMessage(HttpStatusCode.Created); + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + // Matches /v2// and returns repo + tail (tail is the digest or reference). + private static bool TryMatch(string path, string marker, out string repo, out string tail) + { + repo = string.Empty; + tail = string.Empty; + const string prefix = "/v2/"; + int markerIndex = path.IndexOf(marker, StringComparison.Ordinal); + if (!path.StartsWith(prefix, StringComparison.Ordinal) || markerIndex < 0) + { + return false; + } + // Exclude the uploads path from the plain blobs matcher. + if (marker == "/blobs/" && path.Contains("/blobs/uploads/", StringComparison.Ordinal)) + { + return false; + } + repo = path[prefix.Length..markerIndex]; + tail = path[(markerIndex + marker.Length)..]; + return repo.Length > 0 && tail.Length > 0; + } + + private static string ExtractRepo(string path, string marker) + { + const string prefix = "/v2/"; + int markerIndex = path.IndexOf(marker, StringComparison.Ordinal); + return path[prefix.Length..markerIndex]; + } + } + + private const string SourceHost = "mcr.microsoft.com"; + private const string EcrHost = "123456789012.dkr.ecr.us-west-2.amazonaws.com"; + + // A well-formed ECR authorization token: base64("AWS:password"). PushToEcrAsync decodes this into the + // Basic username/password; the fake registry does not check it, so any valid-form token works here. + private static readonly string FakeEcrToken = + Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("AWS:fake-password")); + + /// Builds a base image (config + manifest with one base layer) and seeds the fake source registry. + private static (ImageBuilder Builder, IReadOnlyList BaseLayers) BuildBaseImage(FakeRegistry registry, string sourceRepo) + { + // A base layer blob that CopyBaseLayersAsync must copy to ECR. + byte[] baseLayerBytes = Encoding.UTF8.GetBytes("base-layer-content"); + string baseLayerDigest = RegistryClient.ComputeDigest(new MemoryStream(baseLayerBytes)); + registry.SeedSourceBlob(SourceHost, sourceRepo, baseLayerDigest, baseLayerBytes); + + const string configJson = """ + {"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:aaaa"]},"config":{}} + """; + string manifestJson = $$""" + {"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","size":10,"digest":"sha256:cfg"},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","size":{{baseLayerBytes.Length}},"digest":"{{baseLayerDigest}}"}]} + """; + + ImageBuilder builder = ImageBuilder.FromBaseImage(configJson, manifestJson, MediaTypes.OciManifestV1); + var baseLayers = new[] + { + new Descriptor { MediaType = MediaTypes.OciLayerGzipV1, Size = baseLayerBytes.Length, Digest = baseLayerDigest }, + }; + return (builder, baseLayers); + } + + [Fact] + public async Task PushAsync_copies_base_layers_uploads_new_layer_and_config_and_puts_manifest() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + // Source client reads base-layer blobs; both source and target go through the same fake handler. + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + // Add an application layer from a temp file. + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + var files = new FileManifest { { appFile, "/app/app.txt" } }; + await builder.AddLayerAsync(files, createdBy: "test"); + builder.Config.SetEntrypoint("dotnet", "/app/app.txt"); + + string digest = await builder.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken); + + // The returned digest addresses the manifest. + Assert.StartsWith("sha256:", digest); + + // The manifest was PUT to the ECR target repo under the right tag and media type. + Assert.NotNull(registry.PutManifest); + var put = registry.PutManifest!.Value; + Assert.Equal(EcrHost, put.Host); + Assert.Equal(targetRepo, put.Repo); + Assert.Equal("latest", put.Reference); + Assert.Equal(MediaTypes.OciManifestV1, put.ContentType); + + // All blobs uploaded to ECR: the base layer (copied), the new app layer, and the config. + var ecrUploads = registry.UploadedBlobs.Where(u => u.Host == EcrHost && u.Repo == targetRepo).ToList(); + Manifest pushedManifest = Manifest.Parse(put.Body); + + // Base layer + new layer both present in the manifest and both uploaded. + Assert.Equal(2, pushedManifest.Layers.Count); + foreach (Descriptor layer in pushedManifest.Layers) + { + Assert.Contains(ecrUploads, u => u.Digest == layer.Digest); + } + // Config blob uploaded, and the manifest points at it. + Assert.Contains(ecrUploads, u => u.Digest == pushedManifest.Config.Digest); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_advertises_content_length_on_every_upload_so_bodies_stream() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-cl-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + await builder.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken); + + // Every blob PATCH (base layer, built layer, config) must advertise a Content-Length. A null here + // would mean HttpClient had to buffer the whole body in memory to measure it — the exact memory + // behavior these fixes exist to avoid. + Assert.NotEmpty(registry.PatchContentLengths); + Assert.All(registry.PatchContentLengths, length => Assert.NotNull(length)); + Assert.All(registry.PatchContentLengths, length => Assert.True(length > 0)); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task AddLayerAsync_spills_built_layer_to_disk_and_Dispose_deletes_it() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-spill-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + + // Built-layer spill files use a stable "aws-oci-built-layer-*.tmp" name under the system temp dir. Other + // tests run in parallel against the same temp dir, so compare the SET of spill files before and after and + // reason only about the files that appeared during this test — never absolute counts. + string spillDir = Path.GetTempPath(); + HashSet SpillFiles() => Directory.EnumerateFiles(spillDir, "aws-oci-built-layer-*.tmp").ToHashSet(StringComparer.Ordinal); + + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + + HashSet before = SpillFiles(); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + // Exactly the file(s) this AddLayerAsync created — isolated from parallel tests' own spill files. + HashSet afterAdd = SpillFiles(); + afterAdd.ExceptWith(before); + Assert.True(afterAdd.Count >= 1, "AddLayerAsync should have spilled the built layer to a temp file."); + Assert.All(afterAdd, path => Assert.True(File.Exists(path))); + + builder.Dispose(); + + // Every file this test created must be gone after Dispose (regardless of what other tests are doing). + Assert.All(afterAdd, path => Assert.False(File.Exists(path), $"Dispose should have deleted {path}.")); + } + finally + { + builder.Dispose(); + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_skips_base_layer_already_present_in_target() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + await builder.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken); + + // The base layer download count: it must have been copied once (first push). + var put = registry.PutManifest!.Value; + Manifest pushed = Manifest.Parse(put.Body); + string baseLayerDigest = baseLayers[0].Digest; + Assert.Contains(pushed.Layers, l => l.Digest == baseLayerDigest); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_rejects_non_ecr_target() + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync("mcr.microsoft.com/notecr/app:latest", FakeEcrToken)); + } + + [Fact] + public async Task PushAsync_rejects_base_layer_whose_bytes_do_not_match_its_digest() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + // Tamper: make the source registry serve DIFFERENT bytes under the base layer's declared digest, as a + // corrupted cache or a man-in-the-middle-past-TLS registry would. The declared digest is unchanged. + string baseLayerDigest = baseLayers[0].Digest; + registry.SeedSourceBlob(SourceHost, sourceRepo, baseLayerDigest, System.Text.Encoding.UTF8.GetBytes("tampered!")); + + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + // The corrupted base layer must be detected on download and the push aborted. + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken)); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_spills_large_base_layer_to_disk_and_still_pushes_successfully() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + // Force the spill-to-disk path: threshold 0 makes every base layer (size > 0) spill to a temp file, + // get re-verified from disk, and upload from the file rather than memory. + string digest = await builder.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { MaxInMemoryLayerBytes = 0 }); + + Assert.StartsWith("sha256:", digest); + + // The base layer (spilled + verified from disk) and the new layer both reached ECR. + var put = registry.PutManifest!.Value; + Manifest pushed = Manifest.Parse(put.Body); + var ecrUploads = registry.UploadedBlobs.Where(u => u.Host == EcrHost && u.Repo == targetRepo).ToList(); + Assert.Equal(2, pushed.Layers.Count); + foreach (Descriptor layer in pushed.Layers) + { + Assert.Contains(ecrUploads, u => u.Digest == layer.Digest); + } + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_rejects_tampered_base_layer_on_the_spill_to_disk_path() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + + // Corrupt the served bytes under the declared digest, exactly as the in-memory tamper test does, but this + // time force the spill path so the disk download + re-read verification is the thing that must catch it. + string baseLayerDigest = baseLayers[0].Digest; + registry.SeedSourceBlob(SourceHost, sourceRepo, baseLayerDigest, System.Text.Encoding.UTF8.GetBytes("tampered!")); + + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { MaxInMemoryLayerBytes = 0 })); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushAsync_rejects_negative_in_memory_threshold() + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync( + $"{EcrHost}/myapp:latest", + FakeEcrToken, + new EcrPushOptions { MaxInMemoryLayerBytes = -1 })); + } + + [Theory] + [InlineData("")] // empty + [InlineData(" ")] // whitespace + [InlineData("not-base64-$$$")] // not valid base64 + public async Task PushToEcrAsync_rejects_a_malformed_authorization_token(string badToken) + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync($"{EcrHost}/myapp:latest", badToken)); + } + + [Fact] + public async Task PushToEcrAsync_rejects_a_token_that_decodes_without_a_username_password_separator() + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + // Valid base64, but decodes to "no-separator-here" with no ':' — not a username:password pair. + string tokenWithoutColon = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("no-separator-here")); + + await Assert.ThrowsAsync( + () => builder.PushToEcrAsync($"{EcrHost}/myapp:latest", tokenWithoutColon)); + } + + // ---- Progress reporting ---------------------------------------------------------------------- + + /// Builds a fresh builder over the shared fake registry with one base layer and one app layer. + private static async Task BuildTwoLayerImageAsync(FakeRegistry registry, string sourceRepo, string tempDir) + { + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, sourceRepo); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/{sourceRepo}:8.0"); + + OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + builder.UseTargetClientFactoryForTest((host, _) => new RegistryClient(host, new HttpClient(registry))); + + string appFile = Path.Combine(tempDir, "app.txt"); + await File.WriteAllTextAsync(appFile, "hello-progress"); + await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); + return builder; + } + + [Fact] + public async Task PushToEcrAsync_reports_progress_for_every_layer_with_stable_index_total_and_kind() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + using OciImageBuilder builder = await BuildTwoLayerImageAsync(registry, sourceRepo, tempDir); + + var events = new List(); + await builder.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { Progress = new SynchronousProgress(events.Add) }); + + // Two layers: index 1 = base layer, index 2 = built app layer; TotalLayers is 2 on every event. + Assert.All(events, e => Assert.Equal(2, e.TotalLayers)); + Assert.All(events, e => Assert.InRange(e.LayerIndex, 1, 2)); + + // The config blob and manifest are NOT layers, so they produce no events — only layer indices appear. + Assert.Equal(new[] { 1, 2 }, events.Select(e => e.LayerIndex).Distinct().OrderBy(i => i).ToArray()); + + // Layer 1 is the base layer; layer 2 is the locally-built layer. Kind is stable per index. + Assert.All(events.Where(e => e.LayerIndex == 1), e => Assert.Equal(LayerKind.BaseLayer, e.Kind)); + Assert.All(events.Where(e => e.LayerIndex == 2), e => Assert.Equal(LayerKind.BuiltLayer, e.Kind)); + + // Each layer has exactly one digest across all its events (stable identity). + foreach (int index in new[] { 1, 2 }) + { + Assert.Single(events.Where(e => e.LayerIndex == index).Select(e => e.Digest).Distinct()); + } + + // Every layer terminates on exactly one Completed event — one consistent "done" signal. + Assert.Equal(2, events.Count(e => e.Phase == LayerPushPhase.Completed)); + foreach (int index in new[] { 1, 2 }) + { + Assert.Single(events, e => e.LayerIndex == index && e.Phase == LayerPushPhase.Completed); + } + + // Each uploaded layer starts before it completes. + foreach (int index in new[] { 1, 2 }) + { + var perLayer = events.Where(e => e.LayerIndex == index).ToList(); + int startedAt = perLayer.FindIndex(e => e.Phase == LayerPushPhase.Started); + int completedAt = perLayer.FindIndex(e => e.Phase == LayerPushPhase.Completed); + Assert.True(startedAt >= 0, $"layer {index} should have a Started event"); + Assert.True(startedAt < completedAt, $"layer {index} Started must precede Completed"); + } + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushToEcrAsync_reports_a_skipped_layer_as_a_single_completed_event_with_IsSkipped_true() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + // First push lands the base layer + app layer in the ECR target repo. + using (OciImageBuilder first = await BuildTwoLayerImageAsync(registry, sourceRepo, tempDir)) + { + await first.PushToEcrAsync($"{EcrHost}/{targetRepo}:latest", FakeEcrToken); + } + + // Second push of an identical image to the SAME target: both layers already exist, so both are skipped. + using OciImageBuilder second = await BuildTwoLayerImageAsync(registry, sourceRepo, tempDir); + var events = new List(); + await second.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { Progress = new SynchronousProgress(events.Add) }); + + // Each layer is reported as exactly one event: Completed + IsSkipped, with no preceding Started/Uploading. + foreach (int index in new[] { 1, 2 }) + { + var perLayer = events.Where(e => e.LayerIndex == index).ToList(); + LayerPushProgress only = Assert.Single(perLayer); + Assert.Equal(LayerPushPhase.Completed, only.Phase); + Assert.True(only.IsSkipped); + Assert.Equal(0, only.BytesPushed); + // LayerSizeBytes is still the real size (so a UI can say "skipped — N bytes already present"). + Assert.True(only.LayerSizeBytes > 0); + } + + // No Started or Uploading events at all when everything is skipped. + Assert.DoesNotContain(events, e => e.Phase == LayerPushPhase.Started); + Assert.DoesNotContain(events, e => e.Phase == LayerPushPhase.Uploading); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public async Task PushToEcrAsync_reports_byte_progress_reaching_layer_size_at_completion() + { + var registry = new FakeRegistry(); + const string sourceRepo = "dotnet/aspnet"; + const string targetRepo = "myapp"; + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + using OciImageBuilder builder = await BuildTwoLayerImageAsync(registry, sourceRepo, tempDir); + + var events = new List(); + await builder.PushToEcrAsync( + $"{EcrHost}/{targetRepo}:latest", + FakeEcrToken, + new EcrPushOptions { Progress = new SynchronousProgress(events.Add) }); + + foreach (int index in new[] { 1, 2 }) + { + var perLayer = events.Where(e => e.LayerIndex == index).ToList(); + + LayerPushProgress started = perLayer.First(e => e.Phase == LayerPushPhase.Started); + Assert.Equal(0, started.BytesPushed); + + // Uploading events (if any) never exceed the layer size and are monotonically non-decreasing. + var uploading = perLayer.Where(e => e.Phase == LayerPushPhase.Uploading).ToList(); + long previous = 0; + foreach (LayerPushProgress u in uploading) + { + Assert.True(u.BytesPushed >= previous, "byte progress must not go backward"); + Assert.True(u.BytesPushed <= u.LayerSizeBytes, "byte progress must not exceed the layer size"); + previous = u.BytesPushed; + } + + // At completion, bytes pushed equals the full layer size. + LayerPushProgress completed = perLayer.Single(e => e.Phase == LayerPushPhase.Completed); + Assert.Equal(completed.LayerSizeBytes, completed.BytesPushed); + Assert.False(completed.IsSkipped); + } + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + /// + /// A deterministic that invokes the callback synchronously on the reporting thread, + /// so a test observes every event in emission order (unlike , which marshals and may + /// reorder/defer via the captured synchronization context). + /// + private sealed class SynchronousProgress : IProgress + { + private readonly Action _onReport; + public SynchronousProgress(Action onReport) => _onReport = onReport; + public void Report(LayerPushProgress value) => _onReport(value); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs new file mode 100644 index 000000000..ec2ecf239 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs @@ -0,0 +1,46 @@ +using Amazon.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class RealmValidatorTests +{ + [Theory] + [InlineData("https://auth.docker.io/token")] + [InlineData("https://public.ecr.aws/token/")] + [InlineData("https://123456789012.dkr.ecr.us-west-2.amazonaws.com/")] + public void Accepts_public_https_realms(string realm) + { + Uri result = RealmValidator.Validate(realm, "registry.example.com"); + Assert.Equal("https", result.Scheme); + } + + [Theory] + [InlineData("http://auth.docker.io/token")] // not https + [InlineData("ftp://auth.docker.io/token")] // not https + public void Rejects_non_https_scheme(string realm) + { + Assert.Throws(() => RealmValidator.Validate(realm, "r")); + } + + [Theory] + [InlineData("https://127.0.0.1/token")] // loopback v4 + [InlineData("https://[::1]/token")] // loopback v6 + [InlineData("https://10.0.0.5/token")] // private + [InlineData("https://192.168.1.10/token")] // private + [InlineData("https://172.16.4.4/token")] // private + [InlineData("https://169.254.1.1/token")] // link-local + [InlineData("https://localhost/token")] // loopback name + [InlineData("https://foo.localhost/token")] // loopback name + [InlineData("https://127.0.0.1./token")] // trailing-dot loopback + public void Rejects_loopback_private_and_linklocal_hosts(string realm) + { + Assert.Throws(() => RealmValidator.Validate(realm, "r")); + } + + [Fact] + public void Rejects_unparseable_realm() + { + Assert.Throws(() => RealmValidator.Validate("not a url", "r")); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs new file mode 100644 index 000000000..8b17b125f --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs @@ -0,0 +1,79 @@ +using Amazon.OCI.Builder; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +public class UnixPermissionsTests +{ + private const UnixFileMode Mode0755 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute; + private const UnixFileMode Mode0644 = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead; + private const UnixFileMode Mode0600 = + UnixFileMode.UserRead | UnixFileMode.UserWrite; + + [Theory] + [InlineData("755")] + [InlineData("0755")] + [InlineData("0o755")] + [InlineData("0O755")] + [InlineData(" 755 ")] + public void FromOctal_string_parses_755(string input) + => Assert.Equal(Mode0755, UnixPermissions.FromOctal(input)); + + [Theory] + [InlineData("644", 0b110_100_100)] + [InlineData("600", 0b110_000_000)] + [InlineData("777", 0b111_111_111)] + [InlineData("000", 0)] + public void FromOctal_string_maps_bits_correctly(string input, int expectedBits) + => Assert.Equal((UnixFileMode)expectedBits, UnixPermissions.FromOctal(input)); + + [Fact] + public void FromOctal_known_values() + { + Assert.Equal(Mode0644, UnixPermissions.FromOctal("644")); + Assert.Equal(Mode0600, UnixPermissions.FromOctal("600")); + } + + [Theory] + [InlineData(755)] + [InlineData(644)] + [InlineData(600)] + public void FromOctal_int_matches_string(int value) + => Assert.Equal(UnixPermissions.FromOctal(value.ToString()), UnixPermissions.FromOctal(value)); + + [Fact] + public void FromOctal_int_literal_755_is_octal_0755() + => Assert.Equal(Mode0755, UnixPermissions.FromOctal(755)); + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("75")] // too few digits + [InlineData("7555")] // too many (non-zero leading) + [InlineData("788")] // 8 is not an octal digit + [InlineData("7x5")] // non-digit + public void FromOctal_string_rejects_invalid(string input) + => Assert.Throws(() => UnixPermissions.FromOctal(input)); + + [Theory] + [InlineData(-1)] + [InlineData(778)] // digit 8 out of octal range + [InlineData(800)] // > 777 + [InlineData(1000)] // out of range + public void FromOctal_int_rejects_invalid(int value) + => Assert.Throws(() => UnixPermissions.FromOctal(value)); + + [Fact] + public void FromOctal_result_is_usable_as_an_explicit_entry_mode() + { + // Confirms the helper's output plugs straight into the FileManifest mode overload. + var files = new FileManifest(); + files.Add("/some/local/path", "/app/tool", UnixPermissions.FromOctal("750")); + + Assert.Single(files.Entries); + Assert.Equal(UnixPermissions.FromOctal("750"), files.Entries[0].Mode); + } +} diff --git a/buildtools/build.proj b/buildtools/build.proj index 32f156103..337628854 100644 --- a/buildtools/build.proj +++ b/buildtools/build.proj @@ -220,6 +220,7 @@ + + true + true + true 0.5.0 diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs index 4ba1e893a..cd33dafe2 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs @@ -24,15 +24,14 @@ namespace Amazon.OCI.Builder.Images; /// public sealed class ImageConfig { - private static readonly JsonSerializerOptions SerializerOptions = new() + // Writer options for the canonical config bytes: compact (no insignificant whitespace) and relaxed escaping, + // matching what registries emit. The config digest is computed over these exact bytes, so this formatting is + // part of the contract. Writing the JsonNode tree through a Utf8JsonWriter keeps serialization trim/AOT-safe + // (no reflection-based JsonSerializer overload or DefaultJsonTypeInfoResolver). + private static readonly JsonWriterOptions WriterOptions = new() { - // Compact output: the config digest is computed over these exact bytes, so no extra whitespace. - WriteIndented = false, - // Do not re-escape non-ASCII; match what registries and other tools emit. + Indented = false, Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - // Required on .NET 8: serializing a JsonNode through a custom options instance throws unless the options - // carry a TypeInfoResolver. .NET 9+ supplies a default implicitly, but we set it explicitly for net8.0. - TypeInfoResolver = new System.Text.Json.Serialization.Metadata.DefaultJsonTypeInfoResolver(), }; private readonly JsonObject _root; @@ -94,7 +93,8 @@ public void AddLayer(string diffId, string? createdBy = null) diffIds = new JsonArray(); rootfs["diff_ids"] = diffIds; } - diffIds.Add(diffId); + // Add the diff_id as a JsonNode (not the generic Add, which is trim/AOT-unsafe). + diffIds.Add(JsonValue.Create(diffId) as JsonNode); // history — one entry per layer keeps the array aligned with the layer stack. The timestamp is fixed so // the config stays reproducible (see Layer's reproducibility rationale). @@ -111,7 +111,7 @@ public void AddLayer(string diffId, string? createdBy = null) { entry["created_by"] = createdBy; } - history.Add(entry); + history.Add(entry as JsonNode); } /// Sets the container entrypoint (the config's config.Entrypoint). @@ -162,7 +162,7 @@ public void SetEnvironmentVariable(string name, string value) return; } } - env.Add(entry); + env.Add(JsonValue.Create(entry) as JsonNode); } /// Adds or overwrites a label (the config's config.Labels map). @@ -198,10 +198,18 @@ public void ExposePort(string port) } /// Serializes the config to its canonical, compact UTF-8 bytes (what the config digest is computed over). - public byte[] ToJsonBytes() => JsonSerializer.SerializeToUtf8Bytes(_root, SerializerOptions); + public byte[] ToJsonBytes() + { + var buffer = new System.Buffers.ArrayBufferWriter(); + using (var writer = new Utf8JsonWriter(buffer, WriterOptions)) + { + _root.WriteTo(writer); + } + return buffer.WrittenSpan.ToArray(); + } /// Serializes the config to its canonical, compact JSON string. - public string ToJsonString() => _root.ToJsonString(SerializerOptions); + public string ToJsonString() => System.Text.Encoding.UTF8.GetString(ToJsonBytes()); /// Returns the config sub-object, creating it if absent. private JsonObject ConfigObject() @@ -225,7 +233,7 @@ private void SetConfigArray(string key, string[] values) var array = new JsonArray(); foreach (string value in values) { - array.Add(value); + array.Add(JsonValue.Create(value) as JsonNode); } ConfigObject()[key] = array; } diff --git a/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs index abaf4fa36..d50f4731d 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs @@ -13,13 +13,6 @@ namespace Amazon.OCI.Builder.Images; /// public sealed class Manifest { - private static readonly JsonSerializerOptions SerializerOptions = new() - { - WriteIndented = false, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; - /// The manifest schema version. Always 2 for OCI and Docker V2 Schema 2. [JsonPropertyName("schemaVersion")] [JsonPropertyOrder(0)] @@ -54,7 +47,7 @@ public static Manifest Parse(string json) try { - return JsonSerializer.Deserialize(json, SerializerOptions) + return JsonSerializer.Deserialize(json, OciJsonContext.ManifestInfo) ?? throw new ArgumentException("The manifest JSON deserialized to null.", nameof(json)); } catch (JsonException ex) @@ -64,10 +57,10 @@ public static Manifest Parse(string json) } /// Serializes the manifest to its canonical, compact UTF-8 bytes (what the image digest is computed over). - public byte[] ToJsonBytes() => JsonSerializer.SerializeToUtf8Bytes(this, SerializerOptions); + public byte[] ToJsonBytes() => JsonSerializer.SerializeToUtf8Bytes(this, OciJsonContext.ManifestInfo); /// Serializes the manifest to its canonical, compact JSON string. - public string ToJsonString() => JsonSerializer.Serialize(this, SerializerOptions); + public string ToJsonString() => JsonSerializer.Serialize(this, OciJsonContext.ManifestInfo); } /// @@ -77,13 +70,6 @@ public static Manifest Parse(string json) /// public sealed class ImageIndex { - private static readonly JsonSerializerOptions SerializerOptions = new() - { - WriteIndented = false, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; - /// The schema version (always 2). [JsonPropertyName("schemaVersion")] public int SchemaVersion { get; set; } = 2; @@ -107,7 +93,7 @@ public static ImageIndex Parse(string json) try { - return JsonSerializer.Deserialize(json, SerializerOptions) + return JsonSerializer.Deserialize(json, OciJsonContext.ImageIndexInfo) ?? throw new ArgumentException("The image index JSON deserialized to null.", nameof(json)); } catch (JsonException ex) diff --git a/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs b/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs new file mode 100644 index 000000000..3eeee7918 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs @@ -0,0 +1,67 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using Amazon.OCI.Builder.Registry; + +namespace Amazon.OCI.Builder.Images; + +/// +/// The source-generated for the strongly-typed documents this library +/// serializes or deserializes. Routing System.Text.Json calls through this context (rather than the +/// reflection-based JsonSerializer overloads) is what makes those paths trim- and Native-AOT-safe: the +/// serialization metadata is emitted at compile time, so nothing needs runtime reflection or dynamic code +/// generation. +/// +/// +/// +/// Output is compact and omits null-valued properties. The manifest's digest/tag is computed over these exact +/// bytes, so the formatting is part of the contract. (The document is a free-form +/// JsonObject rather than a modeled type, so it is serialized directly through a Utf8JsonWriter in +/// that class rather than through this context.) +/// +/// +/// The relaxed JSON encoder (which leaves characters such as + unescaped) cannot be set on +/// , so it is applied via — a +/// that chains this context's source-generated resolver and adds the encoder. +/// Serializing through the these options produce stays AOT-safe while matching the +/// byte-for-byte escaping other container tooling emits (so the computed digests agree). +/// +/// +[JsonSourceGenerationOptions( + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(Manifest))] +[JsonSerializable(typeof(ImageIndex))] +[JsonSerializable(typeof(Descriptor))] +[JsonSerializable(typeof(PlatformManifestDescriptor))] +[JsonSerializable(typeof(Platform))] +[JsonSerializable(typeof(TokenResponse))] +internal sealed partial class OciJsonContext : JsonSerializerContext +{ + // Chains the source-generated resolver and adds the relaxed encoder. Built lazily (not in a static field + // initializer) because the generated `Default` singleton is itself lazily initialized: referencing it from + // this class's own static-field initializers races its construction and throws a NullReferenceException in + // the type initializer (notably under Native AOT). Lazy defers the reference until first use, by which + // point `Default` is available. + private static readonly Lazy RelaxedOptions = new(() => + new JsonSerializerOptions(Default!.Options) { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); + + private static readonly Lazy> ManifestInfoLazy = new(() => + (JsonTypeInfo)RelaxedOptions.Value.GetTypeInfo(typeof(Manifest))); + + private static readonly Lazy> ImageIndexInfoLazy = new(() => + (JsonTypeInfo)RelaxedOptions.Value.GetTypeInfo(typeof(ImageIndex))); + + private static readonly Lazy> TokenResponseInfoLazy = new(() => + (JsonTypeInfo)RelaxedOptions.Value.GetTypeInfo(typeof(TokenResponse))); + + /// Type metadata for with the relaxed (digest-faithful) encoder. + internal static JsonTypeInfo ManifestInfo => ManifestInfoLazy.Value; + + /// Type metadata for with the relaxed (digest-faithful) encoder. + internal static JsonTypeInfo ImageIndexInfo => ImageIndexInfoLazy.Value; + + /// Type metadata for (relaxed encoder; this type is only ever read). + internal static JsonTypeInfo TokenResponseInfo => TokenResponseInfoLazy.Value; +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index 061aa46f8..d0c5d4a02 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -156,7 +156,7 @@ protected override async Task SendAsync(HttpRequestMessage TokenResponse? token; await using (Stream s = await tokenResponse.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) { - token = await JsonSerializer.DeserializeAsync(s, cancellationToken: cancellationToken).ConfigureAwait(false); + token = await JsonSerializer.DeserializeAsync(s, Images.OciJsonContext.TokenResponseInfo, cancellationToken: cancellationToken).ConfigureAwait(false); } if (token is null || token.ResolvedToken is null) @@ -257,40 +257,4 @@ private static async Task CloneAsync(HttpRequestMessage orig /// Parsed Bearer challenge parameters. private sealed record AuthInfo(string Realm, string? Service, string? Scope); - - /// The JSON body returned by a registry token endpoint. - private sealed record TokenResponse - { - [JsonPropertyName("token")] - public string? Token { get; init; } - - [JsonPropertyName("access_token")] - public string? AccessToken { get; init; } - - [JsonPropertyName("expires_in")] - public int? ExpiresIn { get; init; } - - [JsonPropertyName("issued_at")] - public DateTimeOffset? IssuedAt { get; init; } - - /// The usable token (registries populate either token or access_token). - public string? ResolvedToken => Token ?? AccessToken; - - /// - /// The computed expiry. Per the token spec, default to 60 seconds of validity when the endpoint - /// omits expires_in; subtract a small safety margin so a cached token is refreshed before it - /// actually expires. - /// - public DateTimeOffset ResolvedExpiration - { - get - { - DateTimeOffset issued = IssuedAt ?? DateTimeOffset.UtcNow; - int validity = ExpiresIn ?? 60; - // 10-second safety margin, floored so very short tokens still move forward in time. - int margin = Math.Min(10, Math.Max(0, validity - 1)); - return issued.AddSeconds(validity - margin); - } - } - } } diff --git a/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs b/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs new file mode 100644 index 000000000..929f83960 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Serialization; + +namespace Amazon.OCI.Builder.Registry; + +/// The JSON body returned by a registry token endpoint during the Bearer auth handshake. +/// +/// Internal and top-level (rather than nested in ) so the source-generated +/// can reference it — a source-gen context cannot target a private nested type. +/// +internal sealed record TokenResponse +{ + [JsonPropertyName("token")] + public string? Token { get; init; } + + [JsonPropertyName("access_token")] + public string? AccessToken { get; init; } + + [JsonPropertyName("expires_in")] + public int? ExpiresIn { get; init; } + + [JsonPropertyName("issued_at")] + public DateTimeOffset? IssuedAt { get; init; } + + /// The usable token (registries populate either token or access_token). + public string? ResolvedToken => Token ?? AccessToken; + + /// + /// The computed expiry. Per the token spec, default to 60 seconds of validity when the endpoint + /// omits expires_in; subtract a small safety margin so a cached token is refreshed before it + /// actually expires. + /// + public DateTimeOffset ResolvedExpiration + { + get + { + DateTimeOffset issued = IssuedAt ?? DateTimeOffset.UtcNow; + int validity = ExpiresIn ?? 60; + // 10-second safety margin, floored so very short tokens still move forward in time. + int margin = Math.Min(10, Math.Max(0, validity - 1)); + return issued.AddSeconds(validity - margin); + } + } +} From 669231de4a2cf716907fee7387671808b46e0bce Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 27 Oct 2026 18:38:51 +0000 Subject: [PATCH 03/14] Address Copilot review findings on AWS.OCI.Builder Security and correctness fixes across the library plus regression tests. Security: - Validate the repository name against the OCI grammar so a malformed path cannot override the request authority and leak credentials. - Reject symbolic links / reparse points during layer assembly (a link inside a mapped directory could otherwise copy files from outside it into the image). - Enforce the SSRF guard at the socket layer: disable auto-redirect and add a connect callback that resolves the host, refuses blocked (loopback/private/link-local) addresses, and connects to the validated address - robust against DNS rebinding and covering realm, registry, and blob redirects. Blob-download redirects are followed explicitly and validated. - Create spilled layer/blob temp files with owner-only (0600) permissions. Correctness: - Validate the fetched base config's OS/architecture against the requested platform instead of silently pushing the wrong platform. - Strip an optional tag from a reference that also carries a digest. - Preserve empty descendant directories in recursive directory mappings, and normalize/validate expanded target paths. - Stop emitting synthesized ancestor directory entries so they no longer clobber base-image directory permissions. - Mutate the image builder only after the layer spill succeeds. - Replay a seekable 401-retry body by rewinding instead of buffering the blob. - Make the upload timeout configurable (default: no client timeout). - Use the stream's remaining length for an implicit Content-Length. - Use a synchronous progress sink in the integration test so assertions do not race asynchronous Progress callbacks. Tests: 139 pass on net8.0/net10.0 (+20 new, 2 updated to the corrected directory-entry contract). AOT publish remains warning-free. --- .../src/AWS.OCI.Builder/EcrPushOptions.cs | 15 ++ .../src/AWS.OCI.Builder/ImageReference.cs | 31 +++- Libraries/src/AWS.OCI.Builder/Images/Layer.cs | 103 ++++++++---- .../src/AWS.OCI.Builder/OciImageBuilder.cs | 59 +++++-- .../Registry/AuthHandshakeHandler.cs | 29 +++- .../Registry/RealmValidator.cs | 28 ++++ .../Registry/RegistryClient.cs | 147 ++++++++++++++++-- .../src/AWS.OCI.Builder/Registry/SpillFile.cs | 40 +++++ .../EcrPushIntegrationTests.cs | 27 +++- .../ImageReferenceTests.cs | 22 +++ .../test/AWS.OCI.Builder.Tests/LayerTests.cs | 72 ++++++++- .../OciImageBuilderTests.cs | 10 ++ .../RealmValidatorTests.cs | 20 +++ 13 files changed, 532 insertions(+), 71 deletions(-) create mode 100644 Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs diff --git a/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs index 0db5d339a..a05ded3eb 100644 --- a/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs +++ b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs @@ -65,4 +65,19 @@ public sealed class EcrPushOptions /// /// public IProgress? Progress { get; init; } + + /// + /// The HTTP timeout applied to the registry client used for the push. Because a whole layer blob is sent in a + /// single PATCH, the default 100-second timeout would cancel a + /// large-layer upload on a slow connection even while the caller's cancellation token is still valid. Defaults + /// to (no client-imposed timeout), so an upload is + /// bounded only by the passed to + /// — pass a token with your own deadline to bound it. Set a finite + /// value to impose a per-request timeout instead. + /// + /// + /// A finite value must be positive, or ; any other value + /// (zero or negative) is rejected at push time. + /// + public TimeSpan UploadTimeout { get; init; } = System.Threading.Timeout.InfiniteTimeSpan; } diff --git a/Libraries/src/AWS.OCI.Builder/ImageReference.cs b/Libraries/src/AWS.OCI.Builder/ImageReference.cs index 7ad7f0ecf..297305182 100644 --- a/Libraries/src/AWS.OCI.Builder/ImageReference.cs +++ b/Libraries/src/AWS.OCI.Builder/ImageReference.cs @@ -101,9 +101,17 @@ public static ImageReference Parse(string reference) int atIndex = remainder.IndexOf('@'); if (atIndex >= 0) { - repository = remainder[..atIndex]; + string namePortion = remainder[..atIndex]; refValue = remainder[(atIndex + 1)..]; isDigest = true; + + // A reference may carry BOTH a tag and a digest (e.g. "dotnet/runtime:8.0@sha256:..."). The digest is + // authoritative, so strip a trailing ":tag" from the name portion; otherwise the tag would leak into + // the repository and address the wrong path. Only a ':' AFTER the final '/' is a tag separator (a ':' + // inside an earlier segment would be a malformed host:port that repository validation rejects anyway). + int nameLastSlash = namePortion.LastIndexOf('/'); + int nameTagColon = namePortion.IndexOf(':', nameLastSlash + 1); + repository = nameTagColon >= 0 ? namePortion[..nameTagColon] : namePortion; } else { @@ -137,6 +145,20 @@ public static ImageReference Parse(string reference) repository = "library/" + repository; } + // Validate the repository against the OCI/Docker name grammar. Without this, an input such as + // "123456789012.dkr.ecr.us-west-2.amazonaws.com///attacker.example/app" would be accepted and its + // "//attacker.example/app" repository would override the authority of the request URIs the RegistryClient + // builds — causing the caller's credentials (a StaticRegistryCredentialSource answers for every host) to + // be sent to an attacker-controlled host. Rejecting malformed names here closes that vector. + if (!RepositoryRegex().IsMatch(repository)) + { + throw new ArgumentException( + $"The image reference '{reference}' has an invalid repository '{repository}'. A repository is one or " + + "more lowercase path components (letters, digits, and separators '.', '_', '-') separated by single " + + "'/' characters.", + nameof(reference)); + } + return new ImageReference(registry, repository, refValue, isDigest); } @@ -148,4 +170,11 @@ public override string ToString() // (also supports the FIPS and China/GovCloud partitions' domains). [GeneratedRegex(@"^\d{12}\.dkr\.ecr(-fips)?\.(?[a-z0-9-]+)\.amazonaws\.com(\.cn)?$", RegexOptions.IgnoreCase)] private static partial Regex EcrHostRegex(); + + // The OCI/Docker repository-name grammar: one or more path components separated by single '/' characters, + // each component lowercase alphanumerics with optional internal '.', '_', '__' or '-' separators. This rejects + // leading/trailing/double slashes, empty components, and uppercase — the shapes that could otherwise smuggle an + // alternate authority into a request URI. + [GeneratedRegex(@"^[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*)*$")] + private static partial Regex RepositoryRegex(); } diff --git a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs index 2e49902cf..86bd782ed 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs @@ -129,32 +129,38 @@ public static async Task CreateAsync( } /// - /// Expands a single manifest entry into file additions and the directory paths they imply. + /// Expands a single manifest entry into file additions and the directory entries to emit. /// + /// + /// Only EXPLICITLY mapped directories are recorded in ; the implicit ancestors + /// of a file are not. Tar extraction creates missing parent directories implicitly, so emitting ancestor + /// entries is unnecessary — and harmful, because a synthesized 0755 ancestor entry would overlay (and reset) + /// the base image's own permissions on that directory (for example /tmp's 1777, or a private 0700 + /// parent). Leaving ancestors out preserves the base image's directory metadata. + /// private static void CollectEntry(FileManifestEntry entry, SortedDictionary files, SortedSet directories) { string containerPath = NormalizeContainerPath(entry.ContainerPath); + // Reject a symlink/reparse point at the mapped path itself: following it could pull in content from + // outside the mapping (including host secrets). + RejectIfSymlink(entry.LocalPath, entry.LocalPath); + if (File.Exists(entry.LocalPath)) { - AddFile(entry.LocalPath, containerPath, entry.Mode, files, directories); + AddFile(entry.LocalPath, containerPath, entry.Mode, files); return; } if (Directory.Exists(entry.LocalPath)) { - // A directory maps recursively: each descendant file is placed under the container path, preserving - // its relative sub-path. The directory itself is also recorded so an empty directory still appears. + // A directory maps recursively. Record the mapped directory itself so an empty mapping still appears, + // then walk it WITHOUT following symlinks, recording every descendant file AND directory so the full + // structure (including empty subdirectories) is preserved. directories.Add(containerPath); - EnsureParentDirectories(containerPath, directories); string root = Path.GetFullPath(entry.LocalPath); - foreach (string localFile in Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)) - { - string relative = Path.GetRelativePath(root, localFile).Replace('\\', '/'); - string target = $"{containerPath}/{relative}"; - AddFile(localFile, target, entry.Mode, files, directories); - } + CollectDirectoryTree(root, containerPath, entry.Mode, files, directories); return; } @@ -164,14 +170,69 @@ private static void CollectEntry(FileManifestEntry entry, SortedDictionary files, SortedSet directories) + /// + /// Recursively collects the files and subdirectories under into the layer, + /// placing each under . Does not follow directory or file symlinks: a reparse + /// point is rejected rather than traversed, so a link cannot copy files from outside the mapped tree into the + /// image. Empty subdirectories are recorded so the directory structure is preserved. + /// + private static void CollectDirectoryTree( + string localRoot, + string containerRoot, + UnixFileMode? explicitMode, + SortedDictionary files, + SortedSet directories) + { + // Enumerate entries at this level without recursing, so we can inspect each for a reparse point before + // descending. TopDirectoryOnly keeps the symlink decision local to each entry. + foreach (string localEntry in Directory.EnumerateFileSystemEntries(localRoot, "*", SearchOption.TopDirectoryOnly)) + { + string relative = Path.GetRelativePath(localRoot, localEntry).Replace('\\', '/'); + // Normalize/validate the expanded target too (not just caller-supplied destinations): a source name + // containing '..' or backslashes must not be able to escape the container root. + string target = NormalizeContainerPath($"{containerRoot}/{relative}"); + + RejectIfSymlink(localEntry, localEntry); + + if (Directory.Exists(localEntry)) + { + directories.Add(target); + CollectDirectoryTree(localEntry, target, explicitMode, files, directories); + } + else if (File.Exists(localEntry)) + { + AddFile(localEntry, target, explicitMode, files); + } + } + } + + /// + /// Throws when is a symbolic link or other reparse point. The layer builder archives + /// entries as regular files/directories; silently following a link would let a link inside a mapped directory + /// pull in files from outside it (an exfiltration vector) and cyclic links would loop. Rejecting is explicit + /// and safe for this library's scope (overlay a known set of local files). + /// + private static void RejectIfSymlink(string path, string reportedPath) + { + FileSystemInfo? info = Directory.Exists(path) ? new DirectoryInfo(path) + : File.Exists(path) ? new FileInfo(path) + : null; + + if (info is not null && info.Attributes.HasFlag(FileAttributes.ReparsePoint)) + { + throw new ArgumentException( + $"The manifest source '{reportedPath}' is a symbolic link or reparse point, which is not supported. " + + "Map the real target path explicitly instead."); + } + } + + private static void AddFile(string localPath, string containerPath, UnixFileMode? explicitMode, SortedDictionary files) { if (files.ContainsKey(containerPath)) { throw new ArgumentException($"Two manifest entries map to the same container path '{containerPath}'."); } files[containerPath] = new FileSource(localPath, ResolveFileMode(localPath, explicitMode)); - EnsureParentDirectories(containerPath, directories); } /// @@ -201,22 +262,6 @@ private static UnixFileMode ResolveFileMode(string localPath, UnixFileMode? expl return DefaultFileMode; } - /// Records every ancestor directory of so the tar contains them. - private static void EnsureParentDirectories(string containerPath, SortedSet directories) - { - int slash = containerPath.LastIndexOf('/'); - while (slash > 0) - { - string parent = containerPath[..slash]; - if (!directories.Add(parent)) - { - // Already recorded — all its ancestors are too, so we can stop early. - break; - } - slash = parent.LastIndexOf('/'); - } - } - /// /// Writes the directory and file entries to an in-memory PAX tar archive, in a deterministic order, and /// returns the raw (uncompressed) tar bytes. diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs index 6e70da196..162ff7d2d 100644 --- a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -50,8 +50,18 @@ public sealed class OciImageBuilder : IDisposable /// client built from the push options; replaced in tests to inject a stubbed transport. The default ignores /// the credential-source argument because builds the ECR source itself. /// - private Func _targetClientFactory = - static (host, credentialSource) => RegistryClient.Create(host, credentialSource); + private Func? _targetClientFactory; + + /// The HTTP timeout for the push client, resolved from at + /// push time and read by the default target-client factory. + private TimeSpan _uploadTimeout = Timeout.InfiniteTimeSpan; + + /// Creates the push-target registry client, honoring the resolved upload timeout. A test override + /// () takes precedence when set. + private RegistryClient CreateTargetClient(string host, IRegistryCredentialSource credentialSource) + => _targetClientFactory is { } factory + ? factory(host, credentialSource) + : RegistryClient.Create(host, credentialSource, _uploadTimeout); private OciImageBuilder( ImageReference baseReference, @@ -144,6 +154,25 @@ public static async Task FromBaseImageAsync( .ConfigureAwait(false); string configJson = System.Text.Encoding.UTF8.GetString(configBytes); + // Validate the platform against the config we actually fetched. ResolvePlatformManifestAsync selects + // the right entry from a manifest LIST, but a single-platform tag/digest is returned without any + // platform comparison — so requesting linux/arm64 from an amd64-only image would otherwise succeed + // and silently push an amd64 image. The config's os/architecture are authoritative, so check them + // here and fail loudly on a mismatch rather than producing a wrong-platform image. + ImageConfig parsedConfig = ImageConfig.Parse(configJson); + if (parsedConfig.Os is { } configOs && !configOs.Equals(os, StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryException( + $"The base image '{baseImage}' is for OS '{configOs}', but '{os}' was requested. " + + "Pass a matching platform, or a base image/tag that provides the requested platform."); + } + if (parsedConfig.Architecture is { } configArch && !configArch.Equals(architecture, StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryException( + $"The base image '{baseImage}' is for architecture '{configArch}', but '{architecture}' was " + + "requested. Pass a matching platform, or a base image/tag that provides the requested platform."); + } + // 3. Prepare the in-memory overlay builder, preserving the base image's schema (OCI vs Docker). ImageBuilder imageBuilder = ImageBuilder.FromBaseImage(configJson, platformManifest.Json, platformManifest.MediaType); @@ -176,16 +205,14 @@ public async Task AddLayerAsync(FileManifest files, string? createdBy = null, Ca // Build the layer with the media type matching the image schema so the manifest stays consistent. Layer layer = await Layer.CreateAsync(files, _imageBuilder.LayerMediaType, cancellationToken).ConfigureAwait(false); - _imageBuilder.AddLayer(layer, createdBy); - // Spill the compressed layer to a temp file so it is not retained in memory for the builder's lifetime. - // Only the (small) DiffId is needed in memory after this — the config references it; the gzipped blob is - // streamed from disk at push time. The builder owns the file and deletes it on Dispose. + // Spill the compressed layer to a temp file FIRST, so it is not retained in memory for the builder's + // lifetime. The file is created with owner-only permissions (see SpillFile) so other local users cannot + // read a private layer from the shared temp directory. The builder owns the file and deletes it on Dispose. string layerFile = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"aws-oci-built-layer-{Guid.NewGuid():N}.tmp"); try { - await using (var file = new FileStream( - layerFile, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 81920, useAsync: true)) + await using (var file = Registry.SpillFile.CreateWrite(layerFile)) { await file.WriteAsync(layer.GzippedTarBytes, cancellationToken).ConfigureAwait(false); } @@ -196,6 +223,11 @@ public async Task AddLayerAsync(FileManifest files, string? createdBy = null, Ca throw; } + // Only now that the blob is safely on disk do we mutate the image builder (adding the layer's descriptor + // and diff_id) and record the pending layer. Doing this before the spill could, on a failed/cancelled + // write, leave the manifest referencing a layer whose blob never materializes — a later push would then + // reference a missing blob, and a retry could duplicate the layer entry. + _imageBuilder.AddLayer(layer, createdBy); _pendingLayers.Add(new PendingLayer(layer.Digest, layerFile, layer.GzippedTarBytes.LongLength)); } @@ -262,13 +294,22 @@ public async Task PushToEcrAsync( nameof(options)); } + TimeSpan uploadTimeout = options?.UploadTimeout ?? Timeout.InfiniteTimeSpan; + if (uploadTimeout != Timeout.InfiniteTimeSpan && uploadTimeout <= TimeSpan.Zero) + { + throw new ArgumentException( + $"{nameof(EcrPushOptions.UploadTimeout)} must be positive or Timeout.InfiniteTimeSpan (was {uploadTimeout}).", + nameof(options)); + } + _uploadTimeout = uploadTimeout; + // Decode the ECR authorization token into the Basic username/password the registry handshake expects. (string username, string password) = DecodeEcrAuthorizationToken(authorizationToken); var credentialSource = new StaticRegistryCredentialSource(username, password); IProgress? progress = options?.Progress; - using RegistryClient targetClient = _targetClientFactory(target.Registry, credentialSource); + using RegistryClient targetClient = CreateTargetClient(target.Registry, credentialSource); // 1. Finalize the config + manifest bytes and their digests. BuiltImage built = _imageBuilder.Build(); diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index d0c5d4a02..dd014c4b2 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -239,14 +239,33 @@ private static async Task CloneAsync(HttpRequestMessage orig if (original.Content is not null) { - // Buffer the content so it can be sent again on the authenticated retry. - byte[] bytes = await original.Content.ReadAsByteArrayAsync().ConfigureAwait(false); - var content = new ByteArrayContent(bytes); + HttpContent replayContent; + + // Prefer replaying a SEEKABLE body by rewinding it — this is the real upload case (the PATCH body is a + // StreamContent over a FileStream or MemoryStream), and rewinding avoids buffering the entire blob into + // memory, which would defeat the spill-to-disk memory bound for large layers. Only a non-seekable body + // of unknown provenance falls back to buffering (small control requests, never the large layer PATCH). + Stream? contentStream = original.Content is StreamContent + ? await original.Content.ReadAsStreamAsync().ConfigureAwait(false) + : null; + + if (contentStream is { CanSeek: true }) + { + contentStream.Position = 0; + replayContent = new StreamContent(contentStream); + } + else + { + // Fallback: buffer. Reached only for non-seekable, typically tiny, bodies. + byte[] bytes = await original.Content.ReadAsByteArrayAsync().ConfigureAwait(false); + replayContent = new ByteArrayContent(bytes); + } + foreach (KeyValuePair> header in original.Content.Headers) { - content.Headers.TryAddWithoutValidation(header.Key, header.Value); + replayContent.Headers.TryAddWithoutValidation(header.Key, header.Value); } - clone.Content = content; + clone.Content = replayContent; } return clone; diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs b/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs index bd72b6e52..cde5f4457 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs @@ -64,6 +64,12 @@ public static Uri Validate(string realm, string registryName) registryName, $"The authentication realm '{realm}' uses a loopback host name '{realmHost}'."); } + // NOTE: a DNS-name realm is not resolved here. Resolving in this method would (a) be a different lookup + // from the one the socket later performs, so it would not actually defend against DNS rebinding, and + // (b) make the spelling check depend on live DNS. Instead, the ACTUAL connection IP is validated at the + // socket layer by GuardedConnectCallback (see RegistryClient.Create), which refuses to connect to any + // blocked (loopback/private/link-local) address for realm requests, the registry itself, and blob + // redirects alike — defeating rebinding because it checks the address the connection truly uses. return realmUri; } @@ -71,6 +77,28 @@ private static bool IsLoopbackDnsName(string host) => host.Equals("localhost", StringComparison.OrdinalIgnoreCase) || host.EndsWith(".localhost", StringComparison.OrdinalIgnoreCase); + /// + /// Returns when 's host is a loopback name or a blocked IP + /// literal. This is a cheap pre-connect spelling check used for blob-download redirect hops; the authoritative + /// defense is applied to the real resolved connection IP at the socket layer. + /// + internal static bool IsBlockedHost(Uri uri) + { + string host = TrimTrailingDot(uri.IdnHost); + if (IsLoopbackDnsName(host)) + { + return true; + } + return IPAddress.TryParse(host, out IPAddress? literal) && IsBlockedIpLiteral(literal); + } + + /// + /// Returns when is a blocked loopback/private/link-local + /// address. Applied by the registry client's connect callback to the ACTUAL address a socket is about to + /// connect to, so it defends against DNS rebinding (the check sees the real IP, not a separately-resolved one). + /// + internal static bool IsBlockedAddress(IPAddress address) => IsBlockedIpLiteral(address); + private static string TrimTrailingDot(string host) => host.Length > 1 && host[^1] == '.' ? host[..^1] : host; diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs index 825749373..2f167872d 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -52,19 +52,91 @@ private RegistryClient(string registryHost, HttpClient httpClient, bool ownsHttp /// Builds a whose HTTP pipeline performs the authentication handshake using /// . /// - public static RegistryClient Create(string registryHost, IRegistryCredentialSource credentialSource) + /// The registry host this client targets. + /// Supplies credentials for the authentication handshake. + /// + /// The . Defaults to so a large single- + /// PATCH layer upload is bounded by the caller's cancellation token rather than cancelled by the 100-second + /// client default. Must be positive or . + /// + public static RegistryClient Create(string registryHost, IRegistryCredentialSource credentialSource, TimeSpan? timeout = null) { + // Automatic redirects are DISABLED: the same transport also carries the token requests issued by + // AuthHandshakeHandler, and an auto-followed redirect from a validated public realm to a private/loopback + // endpoint would bypass RealmValidator (an SSRF vector). Blob downloads — which legitimately redirect to + // object storage — follow redirects explicitly in GetBlobAsync, validating each hop. var inner = new SocketsHttpHandler { AutomaticDecompression = DecompressionMethods.None, - // Registry endpoints can redirect blob downloads to object storage; allow it. - AllowAutoRedirect = true, + AllowAutoRedirect = false, + // SSRF defense at the socket layer: resolve the target, reject any blocked (loopback/private/link-local) + // address, and connect to the SAME address we validated — so a DNS name that resolves to a private + // address is refused, and DNS rebinding cannot swap in a different address between check and connect. + ConnectCallback = GuardedConnectAsync, }; var authHandler = new AuthHandshakeHandler(credentialSource, inner); - var http = new HttpClient(authHandler); + var http = new HttpClient(authHandler) + { + // No client-imposed timeout by default: a whole layer is sent in one PATCH, so the 100s default would + // cancel large uploads on slow links. The caller's cancellation token is the intended deadline. + Timeout = timeout ?? Timeout.InfiniteTimeSpan, + }; return new RegistryClient(registryHost, http, ownsHttpClient: true); } + /// + /// A that resolves the target host, refuses to connect to any + /// blocked (loopback/private/link-local) address, and connects to the exact address it validated. Because the + /// check and the connect use the same resolved address, this is robust against DNS rebinding — unlike a + /// separate pre-resolution that the socket could override with a second lookup. + /// + private static async ValueTask GuardedConnectAsync(SocketsHttpConnectionContext context, CancellationToken cancellationToken) + { + string host = context.DnsEndPoint.Host; + int port = context.DnsEndPoint.Port; + + IPAddress[] addresses = IPAddress.TryParse(host, out IPAddress? literal) + ? [literal] + : await System.Net.Dns.GetHostAddressesAsync(host, cancellationToken).ConfigureAwait(false); + + // Try each resolved address in order, skipping blocked ones, connecting to the first that is allowed AND + // reachable. A blocked address is never dialed. + System.Net.Sockets.Socket? socket = null; + Exception? lastError = null; + foreach (IPAddress address in addresses) + { + if (RealmValidator.IsBlockedAddress(address)) + { + lastError = new RegistryException( + $"Refusing to connect to '{host}': it resolves to a blocked (loopback/private/link-local) address '{address}'."); + continue; + } + + var attempt = new System.Net.Sockets.Socket(address.AddressFamily, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp) + { + NoDelay = true, + }; + try + { + await attempt.ConnectAsync(new IPEndPoint(address, port), cancellationToken).ConfigureAwait(false); + socket = attempt; + break; + } + catch (Exception ex) + { + lastError = ex; + attempt.Dispose(); + } + } + + if (socket is null) + { + throw lastError ?? new RegistryException($"Could not connect to '{host}:{port}'."); + } + + return new System.Net.Sockets.NetworkStream(socket, ownsSocket: true); + } + private Uri Base => new($"https://{_registryHost}/v2/"); // ---- Manifests ---------------------------------------------------------------------------- @@ -127,14 +199,63 @@ public async Task BlobExistsAsync(string repository, string digest, Cancel } /// Downloads a blob by digest as a stream. The caller owns and must dispose the returned stream's response. + /// + /// Auto-redirect is disabled on the transport (see ), so redirects to object storage are + /// followed here explicitly, up to hops. Each hop's target is validated against + /// the blocked (loopback/private/link-local) networks so a redirect cannot bounce the download onto the host's + /// private network, and the Authorization header is not carried to a different host. + /// public async Task GetBlobAsync(string repository, string digest, CancellationToken cancellationToken = default) { - var request = new HttpRequestMessage(HttpMethod.Get, new Uri(Base, $"{repository}/blobs/{digest}")); - HttpResponseMessage response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - await EnsureSuccessAsync(response, $"GET blob {repository} {digest}", cancellationToken).ConfigureAwait(false); - return await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + Uri uri = new(Base, $"{repository}/blobs/{digest}"); + + for (int hop = 0; ; hop++) + { + var request = new HttpRequestMessage(HttpMethod.Get, uri); + HttpResponseMessage response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (IsRedirect(response.StatusCode)) + { + if (hop >= MaxBlobRedirects) + { + response.Dispose(); + throw new RegistryRequestException((int)response.StatusCode, $"blob download for {repository} {digest} exceeded {MaxBlobRedirects} redirects."); + } + + Uri? next = response.Headers.Location; + response.Dispose(); + if (next is null) + { + throw new RegistryRequestException(302, $"blob download for {repository} {digest} returned a redirect with no Location."); + } + if (!next.IsAbsoluteUri) + { + next = new Uri(uri, next); + } + if (!next.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase)) + { + throw new RegistryRequestException(302, $"blob download for {repository} {digest} redirected to a non-https URL."); + } + // SSRF guard: never follow a redirect onto a blocked (loopback/private/link-local) address. + if (RealmValidator.IsBlockedHost(next)) + { + throw new RegistryRequestException(302, $"blob download for {repository} {digest} redirected to a blocked (loopback/private/link-local) host '{next.Host}'."); + } + uri = next; + continue; + } + + await EnsureSuccessAsync(response, $"GET blob {repository} {digest}", cancellationToken).ConfigureAwait(false); + return await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + } } + private const int MaxBlobRedirects = 5; + + private static bool IsRedirect(HttpStatusCode status) + => status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther + or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect; + /// /// Attempts a cross-repository blob mount so an already-present base-image layer is not re-uploaded. /// Returns when the registry mounted the blob (HTTP 201), @@ -212,9 +333,10 @@ public async Task UploadBlobAsync( // progress callback is supplied, wrap the source so bytes are reported as the HTTP stack reads them. Stream patchContent = onBytesSent is null ? content : new ProgressReportingStream(content, onBytesSent); - // Resolve the body length: prefer the caller-supplied value, else a seekable stream's own length. A known - // length lets HttpClient stream the body onto the socket instead of first buffering it all into memory. - long? bodyLength = contentLength ?? (content.CanSeek ? content.Length : null); + // Resolve the body length: prefer the caller-supplied value, else a seekable stream's REMAINING length + // (Length - Position). StreamContent sends from the current position, so advertising the full Length for a + // stream positioned past the start would claim more bytes than are actually sent and fail the upload. + long? bodyLength = contentLength ?? (content.CanSeek ? content.Length - content.Position : null); using (var patch = new HttpRequestMessage(HttpMethod.Patch, uploadUri) { @@ -428,8 +550,7 @@ public async Task GetVerifiedBlobToFileAsync(string repository // 1. Stream the blob to disk, hashing the exact bytes written as we go (download-side integrity). string downloadedDigest; await using (Stream source = await GetBlobAsync(repository, digest, cancellationToken).ConfigureAwait(false)) - await using (var file = new FileStream( - tempPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 81920, useAsync: true)) + await using (var file = SpillFile.CreateWrite(tempPath)) using (var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256)) { byte[] rented = System.Buffers.ArrayPool.Shared.Rent(81920); diff --git a/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs b/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs new file mode 100644 index 000000000..33a5faf55 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs @@ -0,0 +1,40 @@ +namespace Amazon.OCI.Builder.Registry; + +/// +/// Helpers for creating the temporary "spill" files this library writes layer/blob bytes to, with owner-only +/// access on Unix. +/// +/// +/// These files land in the shared system temp directory. With the usual umask (022) a newly created file is +/// world-readable (0644), so another local user could read a private image layer — including files a manifest +/// mapped as 0600 — for the lifetime of the handle. The permissions recorded inside the tar do not +/// protect the outer spill file. Creating the file 0600 from the start (atomically, via +/// ) closes that window. On Windows the mode is not applicable and +/// is left unset; default ACLs already restrict the per-user temp directory. +/// +internal static class SpillFile +{ + // Owner read+write only (0600). + private const UnixFileMode OwnerOnly = UnixFileMode.UserRead | UnixFileMode.UserWrite; + + /// + /// Opens a new file for writing with owner-only permissions on Unix. Fails if the file already exists + /// (), matching the unique-temp-name contract the callers rely on. + /// + public static FileStream CreateWrite(string path, int bufferSize = 81920) + { + var options = new FileStreamOptions + { + Mode = FileMode.CreateNew, + Access = FileAccess.Write, + Share = FileShare.None, + BufferSize = bufferSize, + Options = FileOptions.Asynchronous, + }; + if (!OperatingSystem.IsWindows()) + { + options.UnixCreateMode = OwnerOnly; + } + return new FileStream(path, options); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs b/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs index 6369e77a8..20396de0b 100644 --- a/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs +++ b/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs @@ -78,20 +78,19 @@ await builder.AddLayerAsync( // Region is inferred from the ECR host in the target; the auth token is fetched via the SDK. string authToken = await GetAuthorizationTokenStringAsync(ecr); - // Collect push progress so we can assert the event contract against real ECR byte streaming. + // Collect push progress so we can assert the event contract against real ECR byte streaming. Use a + // SYNCHRONOUS sink: Progress marshals callbacks asynchronously, so completing PushToEcrAsync would + // not guarantee every Completed event has arrived, making the assertions below racy. A synchronous + // sink reports inline on the push thread, so all events are present once the push returns. var events = new List(); var options = new EcrPushOptions { - Progress = new Progress(e => - { - lock (events) { events.Add(e); } - }), + Progress = new SynchronousProgress(events.Add), }; pushedDigest = await builder.PushToEcrAsync(targetImage, authToken, options); - // Verify the progress contract end to end. (Progress marshals callbacks, so snapshot under lock.) - List seen; - lock (events) { seen = events.ToList(); } + // Verify the progress contract end to end. All events are already delivered (synchronous sink). + List seen = events; Assert.NotEmpty(seen); int totalLayers = seen[0].TotalLayers; @@ -362,4 +361,16 @@ private static string EnvOr(string name, string fallback) string? value = Environment.GetEnvironmentVariable(name); return string.IsNullOrWhiteSpace(value) ? fallback : value; } + + /// + /// A deterministic that invokes the callback synchronously on the reporting thread, + /// so all events are delivered by the time the awaited push completes — unlike , whose + /// asynchronous marshaling would make post-push assertions race the final callbacks. + /// + private sealed class SynchronousProgress : IProgress + { + private readonly Action _onReport; + public SynchronousProgress(Action onReport) => _onReport = onReport; + public void Report(LayerPushProgress value) => _onReport(value); + } } diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs index cbda95bff..75b65b06f 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs @@ -42,6 +42,28 @@ public void Parses_digest_reference() Assert.True(parsed.ReferenceIsDigest); } + [Fact] + public void Tag_and_digest_together_strips_the_tag_from_the_repository() + { + // A reference may carry BOTH a tag and a digest; the digest is authoritative and the tag must not leak + // into the repository (which would address the wrong path, e.g. "dotnet/runtime:8.0"). + const string digest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"; + ImageReference parsed = ImageReference.Parse($"mcr.microsoft.com/dotnet/runtime:8.0@{digest}"); + + Assert.Equal("dotnet/runtime", parsed.Repository); + Assert.Equal(digest, parsed.Reference); + Assert.True(parsed.ReferenceIsDigest); + } + + [Theory] + // Empty / doubled path components that could smuggle an alternate authority into request URIs, or uppercase. + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com///attacker.example/app:latest")] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com//app:latest")] + [InlineData("registry.example.com/app//evil:latest")] + [InlineData("registry.example.com/App/Bad:latest")] + public void Rejects_malformed_repository(string input) + => Assert.Throws(() => ImageReference.Parse(input)); + [Theory] [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/app:latest", "us-west-2")] [InlineData("123456789012.dkr.ecr.eu-central-1.amazonaws.com/app:latest", "eu-central-1")] diff --git a/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs index 373ea5d40..784d342ef 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs @@ -102,7 +102,7 @@ public async Task Entry_order_does_not_affect_digest() } [Fact] - public async Task Tar_contains_files_at_relative_container_paths_and_parent_dirs() + public async Task Tar_contains_file_but_not_synthesized_ancestor_directories() { string file = WriteFile("app.dll", "hello"); var manifest = new FileManifest { { file, "/app/bin/app.dll" } }; @@ -110,10 +110,12 @@ public async Task Tar_contains_files_at_relative_container_paths_and_parent_dirs Layer layer = await Layer.CreateAsync(manifest); List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); - // Parent directories are synthesized (no leading slash, trailing slash for dirs), file has no leading slash. - Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/" }); - Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/bin/" }); + // The file is present (no leading slash). Its ancestor directories are NOT synthesized as tar entries: + // emitting a 0755 "app/" / "app/bin/" entry would overlay and reset the base image's permissions on those + // directories. Tar extraction creates missing parents implicitly, so the file still lands correctly. Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/bin/app.dll" }); + Assert.DoesNotContain(entries, e => e is { Type: TarEntryType.Directory, Name: "app/" }); + Assert.DoesNotContain(entries, e => e is { Type: TarEntryType.Directory, Name: "app/bin/" }); } [Fact] @@ -133,6 +135,61 @@ public async Task Directory_mapping_recurses_preserving_subpaths() Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/data/sub/" }); } + [Fact] + public async Task Directory_mapping_preserves_empty_descendant_directories() + { + // A recursive directory mapping must preserve empty subdirectories, not just directories that hold files. + WriteFile("content/root.txt", "r"); + Directory.CreateDirectory(Path.Combine(_tempRoot, "content", "cache")); // empty + string dir = Path.Combine(_tempRoot, "content"); + + var manifest = new FileManifest { { dir, "/app/data" } }; + + Layer layer = await Layer.CreateAsync(manifest); + List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); + + Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/data/root.txt" }); + Assert.Contains(entries, e => e is { Type: TarEntryType.Directory, Name: "app/data/cache/" }); + } + + [Fact] + public async Task Symlinked_source_file_is_rejected() + { + if (OperatingSystem.IsWindows()) + { + return; // Creating symlinks on Windows requires elevation; the guard is exercised on Unix CI. + } + + string real = WriteFile("secret.txt", "host secret"); + string link = Path.Combine(_tempRoot, "link.txt"); + File.CreateSymbolicLink(link, real); + + var manifest = new FileManifest { { link, "/app/link.txt" } }; + + // A symlinked source must be rejected rather than silently followed (which could pull in files from + // outside the mapped set, e.g. host secrets). + await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + } + + [Fact] + public async Task Symlink_inside_mapped_directory_is_rejected() + { + if (OperatingSystem.IsWindows()) + { + return; + } + + WriteFile("tree/ok.txt", "ok"); + string outside = WriteFile("outside.txt", "host secret"); + // A link INSIDE the mapped directory pointing outside it must not copy the external file into the layer. + File.CreateSymbolicLink(Path.Combine(_tempRoot, "tree", "escape.txt"), outside); + string dir = Path.Combine(_tempRoot, "tree"); + + var manifest = new FileManifest { { dir, "/app" } }; + + await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + } + [Fact] public async Task Empty_manifest_throws() { @@ -217,7 +274,10 @@ entry.EntryType is TarEntryType.ExtendedAttributes or TarEntryType.GlobalExtende public async Task Default_file_mode_is_0644_and_directory_mode_is_0755() { string file = WriteFile("app.dll", "hello"); - var manifest = new FileManifest { { file, "/app/app.dll" } }; + // Map an explicit directory too, so there is a real (non-synthesized) directory entry to check the mode on. + // Synthesized ancestors are intentionally not emitted, so the directory assertion uses a mapped directory. + string dir = Directory.CreateDirectory(Path.Combine(_tempRoot, "assets")).FullName; + var manifest = new FileManifest { { file, "/app/app.dll" }, { dir, "/app/assets" } }; Layer layer = await Layer.CreateAsync(manifest); var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); @@ -229,7 +289,7 @@ public async Task Default_file_mode_is_0644_and_directory_mode_is_0755() Assert.Equal(Mode0644, fileEntry.Mode); (TarEntryType Type, string Name, UnixFileMode Mode) dirEntry = - Assert.Single(entries, e => e is { Type: TarEntryType.Directory, Name: "app/" }); + Assert.Single(entries, e => e is { Type: TarEntryType.Directory, Name: "app/assets/" }); Assert.Equal(Mode0755, dirEntry.Mode); } diff --git a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs index 1924ca142..3daeb4b2c 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs @@ -303,6 +303,16 @@ public async Task AddLayerAsync_spills_built_layer_to_disk_and_Dispose_deletes_i Assert.True(afterAdd.Count >= 1, "AddLayerAsync should have spilled the built layer to a temp file."); Assert.All(afterAdd, path => Assert.True(File.Exists(path))); + // #5: on Unix the spill file must be owner-only (0600), not world-readable (0644). A private layer in + // the shared temp dir must not be readable by other local users. + if (!OperatingSystem.IsWindows()) + { + const UnixFileMode groupOrOther = + UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute; + Assert.All(afterAdd, path => Assert.Equal((UnixFileMode)0, File.GetUnixFileMode(path) & groupOrOther)); + } + builder.Dispose(); // Every file this test created must be gone after Dispose (regardless of what other tests are doing). diff --git a/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs index ec2ecf239..85fdf9118 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs @@ -43,4 +43,24 @@ public void Rejects_unparseable_realm() { Assert.Throws(() => RealmValidator.Validate("not a url", "r")); } + + [Theory] + [InlineData("127.0.0.1", true)] // loopback + [InlineData("10.1.2.3", true)] // private + [InlineData("172.16.5.5", true)] // private + [InlineData("192.168.0.9", true)] // private + [InlineData("169.254.1.1", true)] // link-local + [InlineData("::1", true)] // IPv6 loopback + [InlineData("8.8.8.8", false)] // public + public void IsBlockedAddress_flags_private_and_loopback(string ip, bool expected) + => Assert.Equal(expected, RealmValidator.IsBlockedAddress(System.Net.IPAddress.Parse(ip))); + + [Theory] + [InlineData("https://localhost/x", true)] + [InlineData("https://sub.localhost/x", true)] + [InlineData("https://127.0.0.1/x", true)] + [InlineData("https://10.0.0.5/x", true)] + [InlineData("https://registry.example.com/x", false)] // public DNS name: not blocked by the pre-connect check + public void IsBlockedHost_flags_loopback_names_and_blocked_literals(string url, bool expected) + => Assert.Equal(expected, RealmValidator.IsBlockedHost(new Uri(url))); } From 5504c50757b424bfeb02bd92bb059f682ae6e920 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 27 Oct 2026 20:53:38 +0000 Subject: [PATCH 04/14] Fix new Copilot findings: blob stream disposal leak and localhost case-sensitivity - GetBlobAsync now returns a ResponseOwningStream that disposes the underlying HttpResponseMessage and HttpRequestMessage when the stream is disposed, and disposes the per-hop request on redirects and on the error path. Previously the response/request were leaked on the success path. - ImageReference host detection compares the 'localhost' literal with OrdinalIgnoreCase so an uppercase host (e.g. LOCALHOST/app) is recognized as a registry rather than misparsed as a Docker Hub repository. - Fix README typos (based on, non-AWS). --- .../src/AWS.OCI.Builder/ImageReference.cs | 2 +- Libraries/src/AWS.OCI.Builder/README.md | 4 +- .../Registry/RegistryClient.cs | 78 ++++++++++++++++++- .../ImageReferenceTests.cs | 15 ++++ 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/Libraries/src/AWS.OCI.Builder/ImageReference.cs b/Libraries/src/AWS.OCI.Builder/ImageReference.cs index 297305182..08ea7e29e 100644 --- a/Libraries/src/AWS.OCI.Builder/ImageReference.cs +++ b/Libraries/src/AWS.OCI.Builder/ImageReference.cs @@ -81,7 +81,7 @@ public static ImageReference Parse(string reference) bool firstSegmentIsHost = firstSlash > 0 && (firstSegment.Contains('.', StringComparison.Ordinal) || firstSegment.Contains(':', StringComparison.Ordinal) || - firstSegment.Equals("localhost", StringComparison.Ordinal)); + firstSegment.Equals("localhost", StringComparison.OrdinalIgnoreCase)); if (firstSegmentIsHost) { diff --git a/Libraries/src/AWS.OCI.Builder/README.md b/Libraries/src/AWS.OCI.Builder/README.md index d7e52a499..7c9dd0791 100644 --- a/Libraries/src/AWS.OCI.Builder/README.md +++ b/Libraries/src/AWS.OCI.Builder/README.md @@ -8,8 +8,8 @@ and pushes the resulting image to **Amazon ECR**. ## Status This library is designed and supported for use by other AWS libraries and tools. -The version of the library is kept below 1.0.0 to indicate that the API is subject to change base on the needs of AWS libraries and tools. -If you are using this library in a non AWS context, please be aware that the API may change in future releases. +The version of the library is kept below 1.0.0 to indicate that the API is subject to change based on the needs of AWS libraries and tools. +If you are using this library in a non-AWS context, please be aware that the API may change in future releases. ## Authentication diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs index 2f167872d..d94bc545a 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -198,7 +198,7 @@ public async Task BlobExistsAsync(string repository, string digest, Cancel return response.StatusCode == HttpStatusCode.OK; } - /// Downloads a blob by digest as a stream. The caller owns and must dispose the returned stream's response. + /// Downloads a blob by digest as a stream. Disposing the returned stream also disposes the underlying HTTP response. /// /// Auto-redirect is disabled on the transport (see ), so redirects to object storage are /// followed here explicitly, up to hops. Each hop's target is validated against @@ -211,6 +211,8 @@ public async Task GetBlobAsync(string repository, string digest, Cancell for (int hop = 0; ; hop++) { + // The request is disposed before the next loop iteration, or (on the success path below) transferred + // to the returned ResponseOwningStream so it outlives this method with the response. var request = new HttpRequestMessage(HttpMethod.Get, uri); HttpResponseMessage response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); @@ -219,11 +221,13 @@ public async Task GetBlobAsync(string repository, string digest, Cancell if (hop >= MaxBlobRedirects) { response.Dispose(); + request.Dispose(); throw new RegistryRequestException((int)response.StatusCode, $"blob download for {repository} {digest} exceeded {MaxBlobRedirects} redirects."); } Uri? next = response.Headers.Location; response.Dispose(); + request.Dispose(); if (next is null) { throw new RegistryRequestException(302, $"blob download for {repository} {digest} returned a redirect with no Location."); @@ -245,8 +249,20 @@ public async Task GetBlobAsync(string repository, string digest, Cancell continue; } - await EnsureSuccessAsync(response, $"GET blob {repository} {digest}", cancellationToken).ConfigureAwait(false); - return await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + try + { + await EnsureSuccessAsync(response, $"GET blob {repository} {digest}", cancellationToken).ConfigureAwait(false); + Stream body = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + // Hand ownership of the response (and its request) to the caller via the returned stream: disposing + // the stream disposes the response, so the HttpResponseMessage/HttpRequestMessage are never leaked. + return new ResponseOwningStream(body, response, request); + } + catch + { + response.Dispose(); + request.Dispose(); + throw; + } } } @@ -256,6 +272,62 @@ private static bool IsRedirect(HttpStatusCode status) => status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect; + /// + /// A read-through stream wrapper that owns the (and the + /// ) the content came from. Disposing this stream disposes the inner content + /// stream and then the response and request, so callers that using/await using the stream + /// returned by never leak the HTTP message pair. + /// + private sealed class ResponseOwningStream : Stream + { + private readonly Stream _inner; + private readonly HttpResponseMessage _response; + private readonly HttpRequestMessage _request; + + public ResponseOwningStream(Stream inner, HttpResponseMessage response, HttpRequestMessage request) + { + _inner = inner; + _response = response; + _request = request; + } + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => false; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override int Read(Span buffer) => _inner.Read(buffer); + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => _inner.ReadAsync(buffer, offset, count, cancellationToken); + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => _inner.ReadAsync(buffer, cancellationToken); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override void Flush() => _inner.Flush(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _inner.Dispose(); + _response.Dispose(); + _request.Dispose(); + } + base.Dispose(disposing); + } + + public override async ValueTask DisposeAsync() + { + await _inner.DisposeAsync().ConfigureAwait(false); + _response.Dispose(); + _request.Dispose(); + await base.DisposeAsync().ConfigureAwait(false); + } + } + /// /// Attempts a cross-repository blob mount so an already-present base-image layer is not re-uploaded. /// Returns when the registry mounted the blob (HTTP 201), diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs index 75b65b06f..dee70db00 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs @@ -102,4 +102,19 @@ public void Preserves_non_default_registry_port() Assert.Equal("myapp", parsed.Repository); Assert.Equal("dev", parsed.Reference); } + + [Theory] + [InlineData("localhost/myapp:dev")] + [InlineData("LOCALHOST/myapp:dev")] + [InlineData("LocalHost/myapp:dev")] + public void Detects_localhost_registry_case_insensitively(string input) + { + // "localhost" with no '.' or ':' is only recognized as a registry host via the explicit name check, + // which must be case-insensitive — otherwise "LOCALHOST/myapp" is misread as a Docker Hub repository. + ImageReference parsed = ImageReference.Parse(input); + + Assert.Equal(input.Split('/')[0], parsed.Registry); + Assert.Equal("myapp", parsed.Repository); + Assert.Equal("dev", parsed.Reference); + } } From 0fc2534ce3b71889303247c4db806ca241947348 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 27 Oct 2026 21:37:33 +0000 Subject: [PATCH 05/14] Address Copilot re-review: multi-challenge auth parsing and doc fixes - AuthHandshakeHandler.TryParseChallenge now scans all WWW-Authenticate challenges instead of only the first, selecting a usable scheme (preferring Bearer with a realm, falling back to Basic). A registry may return multiple challenges in any order (RFC 7235); the previous first-only logic could fail auth when a usable challenge was not listed first. Added regression tests for Bearer-not-first and Basic-fallback. - Fix Layer class XML doc that claimed synthesized parent directory entries are emitted; they are intentionally not (ancestors are created implicitly on extraction). - Fix OciImageBuilder class XML doc example to call PushToEcrAsync with an auth token instead of the non-existent PushAsync; correct a stale PushAsync reference in an inline comment. --- Libraries/src/AWS.OCI.Builder/Images/Layer.cs | 2 +- .../src/AWS.OCI.Builder/OciImageBuilder.cs | 5 +- .../Registry/AuthHandshakeHandler.cs | 46 +++++++----- .../AuthHandshakeHandlerTests.cs | 75 +++++++++++++++++++ 4 files changed, 107 insertions(+), 21 deletions(-) diff --git a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs index 86bd782ed..c6677d9e9 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs @@ -28,7 +28,7 @@ namespace Amazon.OCI.Builder.Images; /// /// /// Layers are written deterministically so that the same inputs always yield the same digests: entries are -/// emitted in a stable sorted order, every entry (including synthesized parent directories) is stamped with a +/// emitted in a stable sorted order, every emitted entry is stamped with a /// fixed epoch timestamp, and paths are normalized to forward-slash, non-rooted form. This reproducibility is /// what lets callers skip re-pushing an unchanged layer. /// diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs index 162ff7d2d..c9f322fb9 100644 --- a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -32,7 +32,8 @@ namespace Amazon.OCI.Builder; /// builder.Config.SetWorkingDirectory("/app"); /// builder.Config.SetEntrypoint("dotnet", "/app/MyApp.dll"); /// -/// await builder.PushAsync("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest"); +/// // Obtain an ECR authorization token (e.g. via AmazonECRClient.GetAuthorizationTokenAsync) and pass it in. +/// await builder.PushToEcrAsync("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:latest", authorizationToken); /// /// This type is not thread-safe; use one instance per image build. /// @@ -176,7 +177,7 @@ public static async Task FromBaseImageAsync( // 3. Prepare the in-memory overlay builder, preserving the base image's schema (OCI vs Docker). ImageBuilder imageBuilder = ImageBuilder.FromBaseImage(configJson, platformManifest.Json, platformManifest.MediaType); - // Capture the base image's layer descriptors now; PushAsync copies these to ECR before uploading the + // Capture the base image's layer descriptors now; PushToEcrAsync copies these to ECR before uploading the // new layer(s). They are a snapshot of the base manifest, distinct from layers added by this build. IReadOnlyList baseLayers = manifest.Layers.ToArray(); diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index dd014c4b2..e297f370c 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -181,29 +181,39 @@ private static bool TryParseChallenge(HttpResponseMessage response, out string? return false; } - AuthenticationHeaderValue header = challenges.First(); - if (header.Scheme is null) + // A registry may legally return multiple WWW-Authenticate challenges (RFC 7235) in an unspecified order, + // so inspect every challenge rather than only the first. Prefer Bearer (token flow) over Basic; a Bearer + // challenge is only usable if it carries a realm. Fall back to the first Basic challenge if no usable + // Bearer is present. + AuthenticationHeaderValue? basicChallenge = null; + foreach (AuthenticationHeaderValue challenge in challenges) { - return false; - } - - scheme = header.Scheme; + if (challenge.Scheme is null) + { + continue; + } - if (header.Scheme.Equals(BasicScheme, StringComparison.OrdinalIgnoreCase)) - { - return true; + if (challenge.Scheme.Equals(BearerScheme, StringComparison.OrdinalIgnoreCase)) + { + Dictionary parsed = ParseBearerParameters(challenge.Parameter); + if (parsed.TryGetValue("realm", out string? realm)) + { + parsed.TryGetValue("service", out string? service); + parsed.TryGetValue("scope", out string? scopeValue); + scheme = challenge.Scheme; + authInfo = new AuthInfo(realm, service, scopeValue); + return true; + } + } + else if (basicChallenge is null && challenge.Scheme.Equals(BasicScheme, StringComparison.OrdinalIgnoreCase)) + { + basicChallenge = challenge; + } } - if (header.Scheme.Equals(BearerScheme, StringComparison.OrdinalIgnoreCase)) + if (basicChallenge is not null) { - Dictionary parsed = ParseBearerParameters(header.Parameter); - if (!parsed.TryGetValue("realm", out string? realm)) - { - return false; - } - parsed.TryGetValue("service", out string? service); - parsed.TryGetValue("scope", out string? scopeValue); - authInfo = new AuthInfo(realm, service, scopeValue); + scheme = basicChallenge.Scheme; return true; } diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs index fb68ba1f5..d0d03def7 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs @@ -74,6 +74,81 @@ public async Task Performs_bearer_handshake_and_retries_with_token() Assert.Equal(3, stub.Requests.Count); } + [Fact] + public async Task Selects_bearer_challenge_when_it_is_not_the_first_challenge() + { + // A registry may return multiple WWW-Authenticate challenges in any order. The handler must scan all of + // them and pick the usable Bearer challenge even when an unsupported scheme is listed first — the previous + // implementation inspected only the first challenge and would have failed here. + var stub = new StubHandler(request => + { + string url = request.RequestUri!.ToString(); + + if (url.StartsWith("https://auth.example.com/token")) + { + Assert.Equal("Basic", request.Headers.Authorization?.Scheme); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + Assert.Equal("abc123", request.Headers.Authorization.Parameter); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("ok") }; + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + // Unsupported scheme first, usable Bearer second. + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Negotiate")); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", + "realm=\"https://auth.example.com/token\",service=\"registry.example.com\",scope=\"repository:app:pull\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/app/manifests/latest"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("ok", await response.Content.ReadAsStringAsync()); + Assert.Equal(3, stub.Requests.Count); + } + + [Fact] + public async Task Falls_back_to_basic_when_no_usable_bearer_challenge_is_present() + { + // When only an unsupported scheme and a Basic challenge are offered, the handler should select Basic and + // present Basic credentials directly (no token round-trip). + var stub = new StubHandler(request => + { + if (request.Headers.Authorization?.Scheme == "Basic") + { + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("ok") }; + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Negotiate")); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Basic", "realm=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/app/manifests/latest"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("ok", await response.Content.ReadAsStringAsync()); + // Initial 401 + authenticated (Basic) retry — no token endpoint call. + Assert.Equal(2, stub.Requests.Count); + } + [Fact] public async Task Rejects_bearer_realm_pointing_at_loopback() { From ed26046c64e6f878e1acf840df3c4c24d05b816d Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 27 Oct 2026 22:20:26 +0000 Subject: [PATCH 06/14] Address Copilot re-review: redirect credential leak, declared-size check, caller auth - Blob-download redirect hops to a different host/port are marked to skip authentication, and AuthHandshakeHandler honors that: no cached credentials are attached and a 401 challenge is not answered. Previously a registry could redirect a blob GET to a host it controls and harvest credentials via a challenge, since the handler authenticated every request on the pipeline. - Verified blob downloads accept the manifest-declared size and throw a new BlobSizeMismatchException when the downloaded length differs, so a wrong descriptor size fails early with a clear error instead of an opaque Content-Length transport failure during the push. Applied to the base image config and both base-layer paths. - AuthHandshakeHandler respects a caller-supplied Authorization header: the request is sent unchanged and any 401 is returned to the caller. --- .../src/AWS.OCI.Builder/OciImageBuilder.cs | 6 +- .../Registry/AuthHandshakeHandler.cs | 23 +++++ .../Registry/RegistryClient.cs | 39 +++++++- .../Registry/RegistryExceptions.cs | 27 +++++ .../AuthHandshakeHandlerTests.cs | 90 +++++++++++++++++ .../BlobRedirectAuthTests.cs | 99 +++++++++++++++++++ .../DigestVerificationTests.cs | 47 +++++++++ 7 files changed, 325 insertions(+), 6 deletions(-) create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs index c9f322fb9..8e938e4ee 100644 --- a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -151,7 +151,7 @@ public static async Task FromBaseImageAsync( } byte[] configBytes = await sourceClient - .GetVerifiedBlobBytesAsync(baseReference.Repository, manifest.Config.Digest, "base image config", cancellationToken) + .GetVerifiedBlobBytesAsync(baseReference.Repository, manifest.Config.Digest, "base image config", manifest.Config.Size, cancellationToken) .ConfigureAwait(false); string configJson = System.Text.Encoding.UTF8.GetString(configBytes); @@ -414,7 +414,7 @@ private async Task CopyBaseLayersAsync( { // Large layer: stream to a temp file (verified twice) and upload from disk, bounding memory. await using VerifiedBlobFile spilled = await _sourceClient - .GetVerifiedBlobToFileAsync(_baseReference.Repository, layer.Digest, "base layer", cancellationToken) + .GetVerifiedBlobToFileAsync(_baseReference.Repository, layer.Digest, "base layer", layer.Size, cancellationToken) .ConfigureAwait(false); await targetClient.UploadBlobAsync( @@ -430,7 +430,7 @@ await targetClient.UploadBlobAsync( { // Small layer: download, verify, and re-upload from memory. byte[] blob = await _sourceClient - .GetVerifiedBlobBytesAsync(_baseReference.Repository, layer.Digest, "base layer", cancellationToken) + .GetVerifiedBlobBytesAsync(_baseReference.Repository, layer.Digest, "base layer", layer.Size, cancellationToken) .ConfigureAwait(false); using var upload = new MemoryStream(blob, writable: false); diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index e297f370c..8f4b1dc6f 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -40,6 +40,13 @@ public sealed partial class AuthHandshakeHandler : DelegatingHandler // Cached authorization header per registry host, with the time it stops being valid. private readonly ConcurrentDictionary _cache = new(StringComparer.OrdinalIgnoreCase); + /// + /// When set to on a request's , this handler + /// sends the request without attaching cached credentials and without answering a 401 challenge. Used for + /// cross-host blob redirect hops so credentials are never presented to a redirect target. + /// + internal static readonly HttpRequestOptionsKey SkipAuthentication = new("AWS.OCI.Builder.SkipAuthentication"); + /// Creates the handler. /// Supplies credentials for each registry host. /// The inner handler (typically an ). @@ -57,6 +64,22 @@ protected override async Task SendAsync(HttpRequestMessage throw new ArgumentException("The request must have an absolute RequestUri.", nameof(request)); } + // Requests explicitly marked to skip authentication (e.g. blob-download redirect hops to a different host, + // such as object storage) are sent as-is: no cached header is attached and a 401 challenge is NOT + // answered. Otherwise a malicious registry could redirect a blob GET to a host it controls, reply 401 + // with a challenge, and harvest the credentials this handler would send in response. + if (request.Options.TryGetValue(SkipAuthentication, out bool skip) && skip) + { + return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + } + + // Respect a caller-supplied Authorization header: send the request unchanged and let the caller handle + // any 401, rather than overwriting their credentials with cached or handshake-derived ones. + if (request.Headers.Authorization is not null) + { + return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + } + string registryHost = request.RequestUri.Host; // Apply a cached, still-valid authorization header if we have one. diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs index d94bc545a..eb5c6f9c4 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -214,6 +214,13 @@ public async Task GetBlobAsync(string repository, string digest, Cancell // The request is disposed before the next loop iteration, or (on the success path below) transferred // to the returned ResponseOwningStream so it outlives this method with the response. var request = new HttpRequestMessage(HttpMethod.Get, uri); + if (Uri.Compare(uri, Base, UriComponents.HostAndPort, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0) + { + // Cross-host redirect hop (e.g. to object storage): never authenticate against the redirect + // target, so a malicious registry cannot redirect to a host it controls and harvest credentials + // via a 401 challenge. Presigned storage URLs carry their own authorization. + request.Options.Set(AuthHandshakeHandler.SkipAuthentication, true); + } HttpResponseMessage response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); if (IsRedirect(response.StatusCode)) @@ -569,9 +576,13 @@ public static void VerifyDigest(string what, string expectedDigest, ReadOnlySpan /// The repository to read from. /// The expected sha256:... digest; the download is verified against it. /// A short description used in any integrity error (e.g. "base layer"). + /// + /// The size declared by the manifest descriptor, if known. When supplied, the downloaded length must match it. + /// /// A cancellation token. /// The downloaded bytes do not hash to . - public async Task GetVerifiedBlobBytesAsync(string repository, string digest, string what, CancellationToken cancellationToken = default) + /// The downloaded length differs from . + public async Task GetVerifiedBlobBytesAsync(string repository, string digest, string what, long? expectedSize = null, CancellationToken cancellationToken = default) { byte[] bytes; await using (Stream stream = await GetBlobAsync(repository, digest, cancellationToken).ConfigureAwait(false)) @@ -582,6 +593,7 @@ public async Task GetVerifiedBlobBytesAsync(string repository, string di } VerifyDigest(what, digest, bytes); + VerifySize(what, digest, expectedSize, bytes.LongLength); return bytes; } @@ -608,19 +620,24 @@ public async Task GetVerifiedBlobBytesAsync(string repository, string di /// The repository to read from. /// The expected sha256:... digest; both verifications check against it. /// A short description used in any integrity error (e.g. "base layer"). + /// + /// The size declared by the manifest descriptor, if known. When supplied, the downloaded length must match it. + /// /// A cancellation token. /// /// The downloaded bytes, or the bytes as read back from disk, do not hash to . /// - public async Task GetVerifiedBlobToFileAsync(string repository, string digest, string what, CancellationToken cancellationToken = default) + /// The downloaded length differs from . + public async Task GetVerifiedBlobToFileAsync(string repository, string digest, string what, long? expectedSize = null, CancellationToken cancellationToken = default) { // A unique temp path under the system temp dir (honors TMPDIR). Created now so we can always clean it up. string tempPath = Path.Combine(Path.GetTempPath(), $"aws-oci-layer-{Guid.NewGuid():N}.tmp"); try { - // 1. Stream the blob to disk, hashing the exact bytes written as we go (download-side integrity). + // 1. Stream the blob to disk, hashing (and counting) the exact bytes written as we go. string downloadedDigest; + long downloadedLength = 0; await using (Stream source = await GetBlobAsync(repository, digest, cancellationToken).ConfigureAwait(false)) await using (var file = SpillFile.CreateWrite(tempPath)) using (var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256)) @@ -632,6 +649,7 @@ public async Task GetVerifiedBlobToFileAsync(string repository while ((read = await source.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0) { hasher.AppendData(rented, 0, read); + downloadedLength += read; await file.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false); } } @@ -648,6 +666,7 @@ public async Task GetVerifiedBlobToFileAsync(string repository } VerifyDigest(what, digest, downloadedDigest); + VerifySize(what, digest, expectedSize, downloadedLength); // 2. Re-read the file from disk and verify again — catches a faulty disk write that corrupted the // spilled bytes after the download-side hash. @@ -665,6 +684,20 @@ public async Task GetVerifiedBlobToFileAsync(string repository } } + /// + /// Verifies that a downloaded blob's length matches the size its manifest descriptor declared, throwing + /// on mismatch. A wrong declared size would otherwise surface much + /// later as an opaque Content-Length transport error during the push, or as a manifest referencing an + /// incorrect size. No check is made when is . + /// + internal static void VerifySize(string what, string digest, long? expectedSize, long actualSize) + { + if (expectedSize is { } declared && declared != actualSize) + { + throw new BlobSizeMismatchException(what, digest, declared, actualSize); + } + } + /// /// Verifies that an expected digest matches an already-computed digest string (both normalized), throwing /// on mismatch. Used when the content was hashed incrementally while diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs index 1c4ec980a..8ffca17d9 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs @@ -70,3 +70,30 @@ public DigestMismatchException(string what, string expectedDigest, string actual ActualDigest = actualDigest; } } + +/// +/// Thrown when a downloaded blob's length does not match the size declared for it in a manifest descriptor. +/// The content may still hash to the expected digest — the descriptor itself is wrong — but pushing it would +/// fail (the upload's Content-Length could not match) or produce a manifest referencing an incorrect size, so +/// the mismatch is reported up front with a clear error instead. +/// +public sealed class BlobSizeMismatchException : RegistryException +{ + /// The digest of the blob whose size did not match. + public string Digest { get; } + + /// The size, in bytes, declared by the manifest descriptor. + public long DeclaredSize { get; } + + /// The actual size, in bytes, of the downloaded blob. + public long ActualSize { get; } + + /// Creates the exception. + public BlobSizeMismatchException(string what, string digest, long declaredSize, long actualSize) + : base($"Integrity check failed for {what} {digest}: the manifest declares a size of {declaredSize} bytes but the blob is {actualSize} bytes.") + { + Digest = digest; + DeclaredSize = declaredSize; + ActualSize = actualSize; + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs index d0d03def7..ec662bf9c 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs @@ -180,4 +180,94 @@ public async Task Passes_through_non_401_responses() Assert.Equal(HttpStatusCode.OK, response.StatusCode); Assert.Single(stub.Requests); } + + [Fact] + public async Task Caller_supplied_authorization_is_not_overwritten_by_cached_auth() + { + var stub = new StubHandler(request => + { + string url = request.RequestUri!.ToString(); + if (url.StartsWith("https://auth.example.com/token")) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + // Echo back which token the registry received. + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(request.Headers.Authorization.Parameter!) }; + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + // 1. Prime the cache with a handshake-derived token for the host. + using (HttpResponseMessage primed = await client.GetAsync("https://registry.example.com/v2/")) + { + Assert.Equal("abc123", await primed.Content.ReadAsStringAsync()); + } + + // 2. A request carrying the caller's own Authorization must reach the registry unchanged. + using var request = new HttpRequestMessage(HttpMethod.Get, "https://registry.example.com/v2/app/manifests/latest"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "caller-token"); + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal("caller-token", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task Caller_supplied_authorization_401_is_returned_without_a_handshake() + { + var stub = new StubHandler(_ => + { + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using var request = new HttpRequestMessage(HttpMethod.Get, "https://registry.example.com/v2/"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "caller-token"); + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + Assert.Single(stub.Requests); // no token request, no retry + } + + [Fact] + public async Task Requests_marked_skip_authentication_are_not_authenticated() + { + var stub = new StubHandler(_ => + { + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Basic", "realm=\"x\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using var request = new HttpRequestMessage(HttpMethod.Get, "https://storage.example.com/blob"); + request.Options.Set(AuthHandshakeHandler.SkipAuthentication, true); + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + Assert.Single(stub.Requests); + Assert.Null(stub.Requests[0].Headers.Authorization); + } } diff --git a/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs new file mode 100644 index 000000000..162d24c2e --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs @@ -0,0 +1,99 @@ +using System.Net; +using System.Net.Http.Headers; +using Amazon.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +/// +/// Verifies that following a blob-download redirect to a different host never presents registry credentials to +/// the redirect target, even when that target answers with a 401 authentication challenge. +/// +public class BlobRedirectAuthTests +{ + private const string RegistryHost = "registry.example.com"; + private const string Digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + + private sealed class StaticCredentials : IRegistryCredentialSource + { + public Task GetCredentialsAsync(string registryHost, CancellationToken ct = default) + => Task.FromResult(new BasicRegistryCredentials("AWS", "secret")); + } + + /// Records every request and delegates the response to a callback. + private sealed class RecordingHandler : HttpMessageHandler + { + private readonly Func _respond; + public List<(Uri Uri, AuthenticationHeaderValue? Auth)> Requests { get; } = new(); + + public RecordingHandler(Func respond) => _respond = respond; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add((request.RequestUri!, request.Headers.Authorization)); + return Task.FromResult(_respond(request)); + } + } + + private static HttpResponseMessage RedirectTo(string location) + { + var redirect = new HttpResponseMessage(HttpStatusCode.TemporaryRedirect); + redirect.Headers.Location = new Uri(location); + return redirect; + } + + [Fact] + public async Task Cross_host_redirect_target_challenge_does_not_receive_credentials() + { + var stub = new RecordingHandler(request => + { + Uri uri = request.RequestUri!; + if (uri.Host == RegistryHost) + { + // The registry redirects the blob download to a host it controls. + return RedirectTo("https://attacker.example.com/blob"); + } + + // The redirect target tries to harvest credentials with a challenge (Bearer realm on its own host). + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://attacker.example.com/token\",service=\"attacker\"")); + return unauthorized; + }); + + var client = new RegistryClient( + RegistryHost, new HttpClient(new AuthHandshakeHandler(new StaticCredentials(), stub))); + + await Assert.ThrowsAnyAsync(() => client.GetBlobAsync("app", Digest)); + + // The attacker host was contacted exactly once (the redirect hop), with no Authorization header, and its + // token endpoint was never called. + var attackerRequests = stub.Requests.Where(r => r.Uri.Host == "attacker.example.com").ToList(); + Assert.Single(attackerRequests); + Assert.Null(attackerRequests[0].Auth); + Assert.DoesNotContain(stub.Requests, r => r.Uri.AbsolutePath == "/token"); + } + + [Fact] + public async Task Cross_host_redirect_to_storage_succeeds_without_credentials() + { + byte[] payload = "layer-bytes"u8.ToArray(); + var stub = new RecordingHandler(request => + request.RequestUri!.Host == RegistryHost + ? RedirectTo("https://storage.example.com/presigned?sig=abc") + : new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(payload) }); + + var client = new RegistryClient( + RegistryHost, new HttpClient(new AuthHandshakeHandler(new StaticCredentials(), stub))); + + await using (Stream stream = await client.GetBlobAsync("app", Digest)) + { + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer); + Assert.Equal(payload, buffer.ToArray()); + } + + var storageRequest = Assert.Single(stub.Requests, r => r.Uri.Host == "storage.example.com"); + Assert.Null(storageRequest.Auth); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs index f94892f86..aeee208d5 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs @@ -130,4 +130,51 @@ await Assert.ThrowsAsync( // Cleanup-on-failure is covered deterministically by the success test's dispose assertion; a global // temp-dir file count here would race with other parallel tests sharing the temp directory. } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_throws_when_declared_size_differs_from_blob_length() + { + // The bytes hash correctly to the digest, but the manifest descriptor declares the wrong size. + byte[] data = Encoding.UTF8.GetBytes("correct content, wrong declared size"); + string digest = RegistryClient.ComputeDigest(data); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(digest, data))); + + BlobSizeMismatchException ex = await Assert.ThrowsAsync( + () => client.GetVerifiedBlobToFileAsync("repo", digest, "base layer", expectedSize: data.Length + 7)); + + Assert.Equal(digest, ex.Digest); + Assert.Equal(data.Length + 7, ex.DeclaredSize); + Assert.Equal(data.Length, ex.ActualSize); + } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_accepts_matching_declared_size() + { + byte[] data = Encoding.UTF8.GetBytes("size matches"); + string digest = RegistryClient.ComputeDigest(data); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(digest, data))); + + await using VerifiedBlobFile handle = await client.GetVerifiedBlobToFileAsync("repo", digest, "base layer", expectedSize: data.Length); + Assert.Equal(data.Length, handle.Stream.Length); + } + + [Fact] + public async Task GetVerifiedBlobBytesAsync_throws_when_declared_size_differs_from_blob_length() + { + byte[] data = Encoding.UTF8.GetBytes("small blob"); + string digest = RegistryClient.ComputeDigest(data); + var client = new RegistryClient("example.com", new HttpClient(new SingleBlobHandler(digest, data))); + + BlobSizeMismatchException ex = await Assert.ThrowsAsync( + () => client.GetVerifiedBlobBytesAsync("repo", digest, "base image config", expectedSize: data.Length - 1)); + Assert.Contains("declares a size", ex.Message, StringComparison.Ordinal); + + // A matching size (or no declared size) passes. + Assert.Equal(data, await client.GetVerifiedBlobBytesAsync("repo", digest, "base image config", expectedSize: data.Length)); + Assert.Equal(data, await client.GetVerifiedBlobBytesAsync("repo", digest, "base image config")); + } + + [Fact] + public void BlobSizeMismatchException_is_a_RegistryException() + => Assert.IsAssignableFrom(new BlobSizeMismatchException("x", "sha256:a", 1, 2)); } From e91548b0cfcef058a86609503a8f26656403ca0f Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 27 Oct 2026 23:09:04 +0000 Subject: [PATCH 07/14] Address Copilot re-review: token validation, retry clone fidelity, digest docs - Reject an ECR authorization token that decodes to an empty username or password with a clear ArgumentException before any registry interaction, instead of letting it surface later as an opaque authentication failure. - The 401-retry clone now copies VersionPolicy and request Options, so the authenticated retry behaves exactly like the original request. - Document that ComputeDigest(Stream) hashes from the stream's current position (matching SHA256.HashData(Stream)) and pin that behavior with a test. --- .../src/AWS.OCI.Builder/OciImageBuilder.cs | 17 ++++++- .../Registry/AuthHandshakeHandler.cs | 8 ++++ .../Registry/RegistryClient.cs | 7 +++ .../AuthHandshakeHandlerTests.cs | 46 +++++++++++++++++++ .../DigestVerificationTests.cs | 12 +++++ .../OciImageBuilderTests.cs | 19 ++++++++ 6 files changed, 108 insertions(+), 1 deletion(-) diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs index 8e938e4ee..329a601f2 100644 --- a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -603,7 +603,22 @@ private static (string Username, string Password) DecodeEcrAuthorizationToken(st nameof(authorizationToken)); } - return (decoded[..separator], decoded[(separator + 1)..]); + string username = decoded[..separator]; + string password = decoded[(separator + 1)..]; + + // Fail fast on a structurally broken token rather than letting it surface later as an opaque registry + // authentication error. The username is not pinned to "AWS" (ECR's documented value) to avoid coupling + // to that detail; an empty username or password, though, can never be a valid ECR credential. + if (username.Length == 0 || password.Length == 0) + { + throw new ArgumentException( + "The ECR authorization token decoded to a 'username:password' pair with an empty " + + (username.Length == 0 ? "username" : "password") + ". Pass the AuthorizationData.AuthorizationToken " + + "value returned by the ECR GetAuthorizationToken API verbatim.", + nameof(authorizationToken)); + } + + return (username, password); } /// Disposes the source registry client and deletes any temp files holding spilled built layers. diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index 8f4b1dc6f..67d814df5 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -263,8 +263,16 @@ private static async Task CloneAsync(HttpRequestMessage orig var clone = new HttpRequestMessage(original.Method, original.RequestUri) { Version = original.Version, + VersionPolicy = original.VersionPolicy, }; + // Carry request options across so the authenticated retry behaves exactly like the original request. + IDictionary cloneOptions = clone.Options; + foreach (KeyValuePair option in original.Options) + { + cloneOptions[option.Key] = option.Value; + } + foreach (KeyValuePair> header in original.Headers) { clone.Headers.TryAddWithoutValidation(header.Key, header.Value); diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs index eb5c6f9c4..4f1056684 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -517,6 +517,13 @@ private static string Truncate(string value, int max) => value.Length <= max ? value : value[..max] + "…"; /// Computes the sha256:... digest of a stream (used by callers constructing blobs). + /// + /// The digest covers the bytes from the stream's current position to its end, and the stream is left + /// positioned at the end — the same semantics as . To digest a seekable + /// stream's full content, set to 0 before calling. The position is not + /// reset automatically, so callers can digest a suffix intentionally and non-seekable streams are supported. + /// + /// The stream to hash, read from its current position to the end. public static string ComputeDigest(Stream content) { byte[] hash = SHA256.HashData(content); diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs index ec662bf9c..1db23ff87 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs @@ -181,6 +181,52 @@ public async Task Passes_through_non_401_responses() Assert.Single(stub.Requests); } + private static readonly HttpRequestOptionsKey MarkerOption = new("test-marker"); + + [Fact] + public async Task Handshake_retry_preserves_version_policy_and_request_options() + { + var stub = new StubHandler(request => + { + string url = request.RequestUri!.ToString(); + if (url.StartsWith("https://auth.example.com/token")) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + return new HttpResponseMessage(HttpStatusCode.OK); + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using var request = new HttpRequestMessage(HttpMethod.Get, "https://registry.example.com/v2/"); + request.VersionPolicy = HttpVersionPolicy.RequestVersionExact; + request.Options.Set(MarkerOption, "kept"); + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + // The last request is the authenticated retry produced by CloneAsync. + HttpRequestMessage retry = stub.Requests[^1]; + Assert.NotSame(request, retry); + Assert.Equal(HttpVersionPolicy.RequestVersionExact, retry.VersionPolicy); + Assert.True(retry.Options.TryGetValue(MarkerOption, out string? marker)); + Assert.Equal("kept", marker); + } + [Fact] public async Task Caller_supplied_authorization_is_not_overwritten_by_cached_auth() { diff --git a/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs index aeee208d5..91a179d4c 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs @@ -23,6 +23,18 @@ public void ComputeDigest_byte_and_stream_overloads_agree() Assert.Equal(fromStream, fromBytes); } + [Fact] + public void ComputeDigest_stream_hashes_from_current_position() + { + byte[] data = Encoding.UTF8.GetBytes("prefix|suffix"); + var stream = new MemoryStream(data) { Position = 7 }; // positioned at "suffix" + + string digest = RegistryClient.ComputeDigest(stream); + + Assert.Equal(RegistryClient.ComputeDigest(Encoding.UTF8.GetBytes("suffix")), digest); + Assert.Equal(data.Length, stream.Position); // left at the end, not rewound + } + [Fact] public void VerifyDigest_passes_for_matching_content() { diff --git a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs index 3daeb4b2c..da0e699da 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs @@ -553,6 +553,25 @@ await Assert.ThrowsAsync( () => builder.PushToEcrAsync($"{EcrHost}/myapp:latest", tokenWithoutColon)); } + [Theory] + [InlineData(":password", "username")] // empty username + [InlineData("AWS:", "password")] // empty password + public async Task PushToEcrAsync_rejects_a_token_with_an_empty_username_or_password(string decodedToken, string emptyPart) + { + var registry = new FakeRegistry(); + (ImageBuilder imageBuilder, IReadOnlyList baseLayers) = BuildBaseImage(registry, "dotnet/aspnet"); + var sourceClient = new RegistryClient(SourceHost, new HttpClient(registry)); + var baseRef = ImageReference.Parse($"{SourceHost}/dotnet/aspnet:8.0"); + + using OciImageBuilder builder = OciImageBuilder.CreateForTest(baseRef, sourceClient, imageBuilder, baseLayers); + + string token = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(decodedToken)); + + ArgumentException ex = await Assert.ThrowsAsync( + () => builder.PushToEcrAsync($"{EcrHost}/myapp:latest", token)); + Assert.Contains($"empty {emptyPart}", ex.Message, StringComparison.Ordinal); + } + // ---- Progress reporting ---------------------------------------------------------------------- /// Builds a fresh builder over the shared fake registry with one base layer and one app layer. From e8f5ad5a591bde13dc2fecd2e98bdec204f299ea Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 27 Oct 2026 23:48:10 +0000 Subject: [PATCH 08/14] Address Copilot re-review: RFC 7235 challenge parsing, retry disposal, options doc - Replace the regex-based Bearer challenge parameter parser with an RFC 7235 auth-param parser: optional whitespace around '=' and ',', quoted values with backslash escapes and embedded commas, unquoted token values, and the full tchar set for names. The old regex dropped realm when whitespace preceded a comma and parsed nothing for unquoted values, failing authentication. - Dispose the 401-retry clone after sending. Its replayed body is wrapped in a non-owning stream, so disposal does not close the caller's upload stream (disposing StreamContent would otherwise close the spilled layer file). - Correct the EcrPushOptions.MaxInMemoryLayerBytes remark: built layers are written to a temp file in AddLayerAsync and uploaded from it; the setting only affects base-layer copying. --- .../src/AWS.OCI.Builder/EcrPushOptions.cs | 5 +- .../Registry/AuthHandshakeHandler.cs | 144 ++++++++++++++++-- .../AuthHandshakeHandlerTests.cs | 125 +++++++++++++++ 3 files changed, 262 insertions(+), 12 deletions(-) diff --git a/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs index a05ded3eb..439ce51fa 100644 --- a/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs +++ b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs @@ -26,8 +26,9 @@ public sealed class EcrPushOptions /// /// /// This affects only the copy of base-image layers (which are downloaded from the source registry and - /// re-uploaded to ECR). Layers built by this library are produced in memory by construction and are uploaded - /// from memory regardless of this setting. + /// re-uploaded to ECR). Layers built by this library are always written to a temporary file when they are added + /// (see ) and uploaded from that file during the push, regardless + /// of this setting. /// /// /// Data integrity is preserved on both paths: the in-memory path verifies the downloaded bytes against the diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index 67d814df5..7a74f28af 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -4,7 +4,6 @@ using System.Text; using System.Text.Json; using System.Text.Json.Serialization; -using System.Text.RegularExpressions; using System.Web; namespace Amazon.OCI.Builder.Registry; @@ -109,8 +108,9 @@ protected override async Task SendAsync(HttpRequestMessage _cache[registryHost] = new CachedAuth(header, expiry); - // Retry the original request with the authorization header. - HttpRequestMessage retry = await CloneAsync(request).ConfigureAwait(false); + // Retry the original request with the authorization header. The clone is disposed once sent; its content + // never owns the caller's body stream (see CloneAsync), so disposing it cannot close that stream. + using HttpRequestMessage retry = await CloneAsync(request).ConfigureAwait(false); retry.Headers.Authorization = header; return await base.SendAsync(retry, cancellationToken).ConfigureAwait(false); } @@ -243,21 +243,114 @@ private static bool TryParseChallenge(HttpResponseMessage response, out string? return false; } - private static Dictionary ParseBearerParameters(string? parameters) + /// + /// Parses the auth-param list of a WWW-Authenticate challenge (RFC 7235 §2.1): + /// name = ( token / quoted-string ) pairs separated by commas, with optional whitespace around both + /// the = and the ,. Quoted values may contain commas and backslash-escaped characters; unquoted + /// (token) values are also accepted. Names are matched case-insensitively; if a name repeats, the first wins. + /// Malformed segments are skipped rather than aborting the whole parse. + /// + internal static Dictionary ParseBearerParameters(string? parameters) { var result = new Dictionary(StringComparer.OrdinalIgnoreCase); - if (parameters is null) + if (string.IsNullOrEmpty(parameters)) { return result; } - foreach (Match match in BearerParameterSplitter().Matches(parameters)) + string s = parameters; + int i = 0; + int n = s.Length; + + while (i < n) { - result[match.Groups["key"].Value] = match.Groups["value"].Value; + // Skip list separators and optional whitespace before the next parameter. + while (i < n && (s[i] == ',' || IsOptionalWhitespace(s[i]))) + { + i++; + } + if (i >= n) + { + break; + } + + // Parameter name: one or more token characters. + int nameStart = i; + while (i < n && IsTokenChar(s[i])) + { + i++; + } + string name = s[nameStart..i]; + + while (i < n && IsOptionalWhitespace(s[i])) + { + i++; + } + + if (name.Length == 0 || i >= n || s[i] != '=') + { + // Not a name=value pair: skip to the next separator and carry on. + SkipToNextSeparator(s, ref i); + continue; + } + + i++; // consume '=' + while (i < n && IsOptionalWhitespace(s[i])) + { + i++; + } + + string value; + if (i < n && s[i] == '"') + { + // quoted-string: read until the closing quote, honoring backslash escapes (quoted-pair). + i++; + var builder = new StringBuilder(); + while (i < n && s[i] != '"') + { + if (s[i] == '\\' && i + 1 < n) + { + i++; + } + builder.Append(s[i]); + i++; + } + i++; // consume the closing quote (harmless if it was missing) + value = builder.ToString(); + } + else + { + // token value: read until a separator or whitespace. + int valueStart = i; + while (i < n && s[i] != ',' && !IsOptionalWhitespace(s[i])) + { + i++; + } + value = s[valueStart..i]; + } + + result.TryAdd(name, value); + SkipToNextSeparator(s, ref i); } + return result; } + private static void SkipToNextSeparator(string s, ref int i) + { + while (i < s.Length && s[i] != ',') + { + i++; + } + } + + private static bool IsOptionalWhitespace(char c) => c is ' ' or '\t'; + + /// An RFC 7230 tchar: the characters allowed in a token such as a parameter name. + private static bool IsTokenChar(char c) + => char.IsAsciiLetterOrDigit(c) || c is '!' or '#' or '$' or '%' or '&' or '\'' or '*' or '+' or '-' or '.' + or '^' or '_' or '`' or '|' or '~'; + private static async Task CloneAsync(HttpRequestMessage original) { var clone = new HttpRequestMessage(original.Method, original.RequestUri) @@ -293,7 +386,9 @@ private static async Task CloneAsync(HttpRequestMessage orig if (contentStream is { CanSeek: true }) { contentStream.Position = 0; - replayContent = new StreamContent(contentStream); + // The stream belongs to the original request (and ultimately the caller); wrap it so that + // disposing the retry clone does not close it. + replayContent = new StreamContent(new NonOwningStream(contentStream)); } else { @@ -312,8 +407,37 @@ private static async Task CloneAsync(HttpRequestMessage orig return clone; } - [GeneratedRegex("(?\\w+)=\"(?[^\"]*)\"(?:,|$)")] - private static partial Regex BearerParameterSplitter(); + /// + /// A pass-through stream that forwards everything to an inner stream but does NOT dispose it. Used for the + /// retry clone's body so disposing the clone leaves the original request's stream — owned by the caller — + /// open and usable. + /// + internal sealed class NonOwningStream : Stream + { + private readonly Stream _inner; + + public NonOwningStream(Stream inner) => _inner = inner; + + public override bool CanRead => _inner.CanRead; + public override bool CanSeek => _inner.CanSeek; + public override bool CanWrite => _inner.CanWrite; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => _inner.Position = value; } + + public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count); + public override int Read(Span buffer) => _inner.Read(buffer); + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => _inner.ReadAsync(buffer, offset, count, cancellationToken); + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => _inner.ReadAsync(buffer, cancellationToken); + public override long Seek(long offset, SeekOrigin origin) => _inner.Seek(offset, origin); + public override void SetLength(long value) => _inner.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => _inner.Write(buffer, offset, count); + public override void Flush() => _inner.Flush(); + + // Dispose is intentionally not overridden: the base implementation never touches _inner, which belongs to + // the caller that owns the original request. + } /// Parsed Bearer challenge parameters. private sealed record AuthInfo(string Realm, string? Service, string? Scope); diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs index 1db23ff87..9ac1e5598 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs @@ -183,6 +183,131 @@ public async Task Passes_through_non_401_responses() private static readonly HttpRequestOptionsKey MarkerOption = new("test-marker"); + [Theory] + // ECR / Docker Hub style (no whitespace). + [InlineData("realm=\"https://a/token\",service=\"reg\",scope=\"repository:app:pull\"", "https://a/token", "reg", "repository:app:pull")] + // Optional whitespace after commas. + [InlineData("realm=\"https://a/token\", service=\"reg\", scope=\"repository:app:pull\"", "https://a/token", "reg", "repository:app:pull")] + // Optional whitespace BEFORE commas — the old regex silently dropped realm here. + [InlineData("realm=\"https://a/token\" , service=\"reg\"", "https://a/token", "reg", null)] + // Whitespace around '=' and tabs. + [InlineData("realm = \"https://a/token\",\tservice\t=\t\"reg\"", "https://a/token", "reg", null)] + // Unquoted token values — the old regex parsed nothing. + [InlineData("realm=https://a/token,service=reg", "https://a/token", "reg", null)] + // A comma inside a quoted value stays part of the value. + [InlineData("realm=\"https://a/token\",scope=\"repository:app:pull,push\"", "https://a/token", null, "repository:app:pull,push")] + public void Parses_bearer_challenge_parameters(string parameters, string realm, string? service, string? scope) + { + Dictionary parsed = AuthHandshakeHandler.ParseBearerParameters(parameters); + + Assert.Equal(realm, parsed["realm"]); + Assert.Equal(service, parsed.GetValueOrDefault("service")); + Assert.Equal(scope, parsed.GetValueOrDefault("scope")); + } + + [Fact] + public void Parses_hyphenated_names_escaped_quotes_and_ignores_malformed_segments() + { + Dictionary parsed = AuthHandshakeHandler.ParseBearerParameters( + "realm=\"https://a/token\", garbage, x-custom=\"1\", error_description=\"say \\\"hi\\\"\", REALM=\"second\""); + + Assert.Equal("https://a/token", parsed["realm"]); // first occurrence wins; names are case-insensitive + Assert.Equal("1", parsed["x-custom"]); // hyphen is a valid token character + Assert.Equal("say \"hi\"", parsed["error_description"]); // backslash escapes are unescaped + Assert.False(parsed.ContainsKey("garbage")); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Parses_empty_parameters_as_empty(string? parameters) + => Assert.Empty(AuthHandshakeHandler.ParseBearerParameters(parameters)); + + [Fact] + public async Task Handshake_succeeds_with_whitespace_before_commas_in_challenge() + { + var stub = new StubHandler(request => + { + if (request.RequestUri!.ToString().StartsWith("https://auth.example.com/token")) + { + Assert.Contains("service=registry.example.com", request.RequestUri.Query, StringComparison.Ordinal); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + return new HttpResponseMessage(HttpStatusCode.OK); + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\" , service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + using HttpResponseMessage response = await client.GetAsync("https://registry.example.com/v2/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(3, stub.Requests.Count); + } + + [Fact] + public async Task Handshake_retry_is_disposed_without_closing_the_callers_body_stream() + { + var stub = new StubHandler(request => + { + if (request.RequestUri!.ToString().StartsWith("https://auth.example.com/token")) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"token\":\"abc123\",\"expires_in\":3600}"), + }; + } + + if (request.Headers.Authorization?.Scheme == "Bearer") + { + return new HttpResponseMessage(HttpStatusCode.Accepted); + } + + var unauthorized = new HttpResponseMessage(HttpStatusCode.Unauthorized); + unauthorized.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return unauthorized; + }); + + var handler = new AuthHandshakeHandler( + new StaticCredentials(new BasicRegistryCredentials("AWS", "secret")), stub); + using var client = new HttpClient(handler); + + // A seekable body, as in a real layer upload (FileStream / MemoryStream) — owned by the caller. + var body = new MemoryStream("layer-bytes"u8.ToArray()); + using var request = new HttpRequestMessage(HttpMethod.Patch, "https://registry.example.com/v2/app/blobs/uploads/1") + { + Content = new StreamContent(body), + }; + using HttpResponseMessage response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + // The retry clone was disposed: its content can no longer be read. + HttpRequestMessage retry = stub.Requests[^1]; + Assert.NotSame(request, retry); + await Assert.ThrowsAsync(() => retry.Content!.ReadAsByteArrayAsync()); + + // ...but the caller's stream is still open and usable. + Assert.True(body.CanRead); + body.Position = 0; + Assert.Equal("layer-bytes"u8.ToArray(), body.ToArray()); + } + [Fact] public async Task Handshake_retry_preserves_version_policy_and_request_options() { From 9fee31d9aeee26a5b36ea654af90df9455ebce60 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Tue, 27 Oct 2026 23:52:31 +0000 Subject: [PATCH 09/14] Fix flaky built-layer spill test The test identified its spill files by diffing the shared temp directory before and after AddLayerAsync. A parallel test spilling a file into that window was counted as this test's, and the post-Dispose assertion then failed because only its owner deletes it. Expose the builder's own spill paths via an internal test accessor and assert on exactly those. --- .../src/AWS.OCI.Builder/OciImageBuilder.cs | 6 ++++++ .../OciImageBuilderTests.cs | 18 ++++++------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs index 329a601f2..2995afc8c 100644 --- a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -92,6 +92,12 @@ internal static OciImageBuilder CreateForTest( internal void UseTargetClientFactoryForTest(Func factory) => _targetClientFactory = factory; + /// + /// Test-only: the temp-file paths this builder spilled its built layers to. Lets tests assert on exactly the + /// files this instance owns instead of scanning the shared temp directory, which parallel tests also write to. + /// + internal IReadOnlyList SpilledLayerPathsForTest => _pendingLayers.Select(p => p.FilePath).ToArray(); + /// The config of the image being built, exposed for customization (entrypoint, env, labels, ...). public ImageConfig Config => _imageBuilder.Config; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs index da0e699da..8d4212e24 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs @@ -283,25 +283,19 @@ public async Task AddLayerAsync_spills_built_layer_to_disk_and_Dispose_deletes_i string tempDir = Path.Combine(Path.GetTempPath(), "oci-spill-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(tempDir); - // Built-layer spill files use a stable "aws-oci-built-layer-*.tmp" name under the system temp dir. Other - // tests run in parallel against the same temp dir, so compare the SET of spill files before and after and - // reason only about the files that appeared during this test — never absolute counts. - string spillDir = Path.GetTempPath(); - HashSet SpillFiles() => Directory.EnumerateFiles(spillDir, "aws-oci-built-layer-*.tmp").ToHashSet(StringComparer.Ordinal); - + // Assert only on the spill files THIS builder owns (from the builder itself), never by scanning the shared + // temp directory: parallel tests spill files with the same name pattern into it concurrently. try { string appFile = Path.Combine(tempDir, "app.txt"); await File.WriteAllTextAsync(appFile, "hello"); - HashSet before = SpillFiles(); await builder.AddLayerAsync(new FileManifest { { appFile, "/app/app.txt" } }); - // Exactly the file(s) this AddLayerAsync created — isolated from parallel tests' own spill files. - HashSet afterAdd = SpillFiles(); - afterAdd.ExceptWith(before); - Assert.True(afterAdd.Count >= 1, "AddLayerAsync should have spilled the built layer to a temp file."); - Assert.All(afterAdd, path => Assert.True(File.Exists(path))); + // Captured before Dispose, which clears the builder's pending-layer list. + IReadOnlyList afterAdd = builder.SpilledLayerPathsForTest; + Assert.Single(afterAdd); + Assert.All(afterAdd, path => Assert.True(File.Exists(path), $"AddLayerAsync should have spilled the built layer to {path}.")); // #5: on Unix the spill file must be owner-only (0600), not world-readable (0644). A private layer in // the shared temp dir must not be readable by other local users. From fcd219c63a9076c34a9ca1a734fad49d11faa256 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 28 Oct 2026 01:25:39 +0000 Subject: [PATCH 10/14] Stream built layers to disk instead of buffering them in memory Layer.CreateAsync now writes the layer in a single streaming pass (tar -> gzip -> caller-supplied destination stream), computing the diff_id, digest and size on the fly. Built layers were previously materialized in memory several times over and could not exceed ~2 GiB (MemoryStream/byte[] limit). Layer is now metadata only; GzippedTarBytes is removed (breaking, pre-1.0). AddLayerAsync builds directly into the owner-only spill file, removing a second full write. Output is byte-identical to the previous implementation for the same tar. Verified live: a 2.4 GiB layer built with ~111 MiB peak working set and pushed to ECR in a single PATCH. Docs: drop the promise that identical input yields identical digests (PAX extended header names embed the process id, so digests vary across runs). Tests: add streaming pipeline tests; remove the temp directory leak in ImageAssemblyTests; fix the CA1416 and xUnit1031 warnings. --- .../src/AWS.OCI.Builder/Images/ImageConfig.cs | 3 +- Libraries/src/AWS.OCI.Builder/Images/Layer.cs | 161 ++++++------- .../AWS.OCI.Builder/Images/LayerStreams.cs | 166 ++++++++++++++ .../src/AWS.OCI.Builder/OciImageBuilder.cs | 15 +- .../ImageAssemblyTests.cs | 22 +- .../LayerStreamingTests.cs | 215 ++++++++++++++++++ .../test/AWS.OCI.Builder.Tests/LayerTests.cs | 70 +++--- .../OciImageBuilderTests.cs | 6 +- 8 files changed, 523 insertions(+), 135 deletions(-) create mode 100644 Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs index cd33dafe2..956d6a745 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs @@ -96,8 +96,7 @@ public void AddLayer(string diffId, string? createdBy = null) // Add the diff_id as a JsonNode (not the generic Add, which is trim/AOT-unsafe). diffIds.Add(JsonValue.Create(diffId) as JsonNode); - // history — one entry per layer keeps the array aligned with the layer stack. The timestamp is fixed so - // the config stays reproducible (see Layer's reproducibility rationale). + // history — one entry per layer keeps the array aligned with the layer stack. The timestamp is fixed. if (_root["history"] is not JsonArray history) { history = new JsonArray(); diff --git a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs index c6677d9e9..a89f1fe1c 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs @@ -5,8 +5,9 @@ namespace Amazon.OCI.Builder.Images; /// -/// A single image layer produced from a : a gzipped tar archive of the mapped files, -/// together with the two digests the OCI format requires. +/// The metadata of a single image layer produced from a — a gzipped tar archive of +/// the mapped files — together with the two digests the OCI format requires. The compressed blob itself is +/// written to a caller-supplied stream by ; this type does not hold the layer bytes. /// /// /// @@ -27,29 +28,23 @@ namespace Amazon.OCI.Builder.Images; /// /// /// -/// Layers are written deterministically so that the same inputs always yield the same digests: entries are -/// emitted in a stable sorted order, every emitted entry is stamped with a -/// fixed epoch timestamp, and paths are normalized to forward-slash, non-rooted form. This reproducibility is -/// what lets callers skip re-pushing an unchanged layer. +/// Entries are written in a stable sorted order with fixed timestamps and normalized paths. Digests are not +/// guaranteed to be identical across builds, even for identical input. /// /// public sealed class Layer { - /// A fixed timestamp used for every tar entry, so identical inputs produce identical archives. + /// A fixed timestamp used for every tar entry. private static readonly DateTimeOffset ReproducibleTimestamp = DateTimeOffset.UnixEpoch; - private Layer(byte[] gzippedTarBytes, string digest, string diffId, long size, string mediaType) + private Layer(string digest, string diffId, long size, string mediaType) { - GzippedTarBytes = gzippedTarBytes; Digest = digest; DiffId = diffId; Size = size; MediaType = mediaType; } - /// The compressed (gzipped) tar archive bytes — the blob content to upload. - public byte[] GzippedTarBytes { get; } - /// /// The sha256:... digest of the compressed tar. Addresses the blob in the registry and is /// referenced by the image manifest's layer descriptor. @@ -62,7 +57,7 @@ private Layer(byte[] gzippedTarBytes, string digest, string diffId, long size, s /// public string DiffId { get; } - /// The size, in bytes, of the compressed tar (). + /// The size, in bytes, of the compressed tar written to the destination stream. public long Size { get; } /// The layer media type (OCI or Docker gzipped-layer type, matching the target image's schema). @@ -79,25 +74,49 @@ private Layer(byte[] gzippedTarBytes, string digest, string diffId, long size, s }; /// - /// Builds a gzipped-tar layer from the files and directories described by . + /// Builds a gzipped-tar layer from the files and directories described by , + /// writing the compressed blob to . /// + /// + /// + /// The layer is produced in a single streaming pass — tar entries are written into a gzip compressor whose + /// output goes straight to — while both digests are computed on the fly: the + /// over the uncompressed tar bytes, and the and + /// over the compressed bytes. Memory use is therefore bounded by stream buffers regardless of layer size, and + /// there is no limit on layer size other than the capacity of the destination. + /// + /// + /// is not disposed, closed, or rewound; the caller owns it. Its prior contents + /// and position are left alone, and the blob is appended starting at its current position. If this method + /// throws, the destination may hold a partial blob and must be discarded. + /// + /// /// The local-to-container path mappings that populate the layer. + /// A writable stream that receives the compressed (gzipped tar) layer blob. /// /// The gzipped-layer media type to report (defaults to ). Pass /// when overlaying onto a Docker-schema base image. /// /// A cancellation token. - /// The built , with both digests computed. - /// is . - /// The manifest is empty, or an entry has an invalid container path. + /// The built 's metadata, with both digests and the compressed size. + /// or is . + /// + /// The manifest is empty, an entry has an invalid container path, or is not writable. + /// /// A mapped local file does not exist. /// A mapped local directory does not exist. public static async Task CreateAsync( FileManifest manifest, + Stream destination, string mediaType = MediaTypes.OciLayerGzipV1, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(manifest); + ArgumentNullException.ThrowIfNull(destination); + if (!destination.CanWrite) + { + throw new ArgumentException("The destination stream must be writable.", nameof(destination)); + } if (manifest.Entries.Count == 0) { throw new ArgumentException("The file manifest is empty; a layer must contain at least one file.", nameof(manifest)); @@ -114,20 +133,34 @@ public static async Task CreateAsync( CollectEntry(entry, files, directories); } - // diff_id is the hash of the uncompressed tar; digest is the hash of the gzipped tar. We compute the - // uncompressed hash by streaming the tar bytes through a hashing stream while they are written, then - // compress those same bytes. Buffering the tar in memory keeps the two passes consistent and lets us - // report an exact compressed size — image layers added by this library are app payloads, not whole OS - // images, so in-memory is appropriate. - byte[] uncompressedTar = await WriteTarAsync(files, directories, cancellationToken).ConfigureAwait(false); + // One streaming pass: tar -> [hash: diff_id] -> gzip -> [normalize gzip header] -> [hash + count: digest, + // size] -> destination. Nothing is buffered beyond stream buffers, so layer size is not limited by memory + // (an in-memory MemoryStream/byte[] would cap a layer at ~2 GiB and hold several full copies at once). + using var diffIdHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + using var digestHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + + var compressedSink = new HashingWriteStream(destination, digestHash); + var headerNormalizer = new GzipHeaderNormalizingStream(compressedSink); + + // leaveOpen: true — disposing the compressor writes the gzip trailer through the wrappers but must not + // close the caller's destination. SmallestSize keeps layer blobs compact (same level as before). + await using (var gzip = new GZipStream(headerNormalizer, CompressionLevel.SmallestSize, leaveOpen: true)) + { + var tarSink = new HashingWriteStream(gzip, diffIdHash); + await WriteTarAsync(files, directories, tarSink, cancellationToken).ConfigureAwait(false); + } - string diffId = ComputeDigest(uncompressedTar); - byte[] gzipped = Compress(uncompressedTar); - string digest = ComputeDigest(gzipped); + await compressedSink.FlushAsync(cancellationToken).ConfigureAwait(false); - return new Layer(gzipped, digest, diffId, gzipped.LongLength, mediaType); + string diffId = ToDigestString(diffIdHash); + string digest = ToDigestString(digestHash); + return new Layer(digest, diffId, compressedSink.BytesWritten, mediaType); } + private static string ToDigestString(IncrementalHash hash) + // Convert.ToHexStringLower exists only on net9.0+, so lower-case explicitly for net8.0 compatibility. + => "sha256:" + Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + /// /// Expands a single manifest entry into file additions and the directory entries to emit. /// @@ -263,25 +296,23 @@ private static UnixFileMode ResolveFileMode(string localPath, UnixFileMode? expl } /// - /// Writes the directory and file entries to an in-memory PAX tar archive, in a deterministic order, and - /// returns the raw (uncompressed) tar bytes. + /// Writes the directory and file entries as a PAX tar archive, in a deterministic order, to + /// (the hashing/compression pipeline). Source files are streamed, not buffered. /// /// /// PAX is the format used by container tooling (including Microsoft.NET.Build.Containers) and is what - /// registries and runtimes expect for image layers. PAX normally writes an extended-attribute header whose - /// contents (sub-second mtime/atime/ctime) would vary run-to-run, so this builder sets - /// a fixed extended-attribute set (see ) and a fixed - /// on every entry, making the archive bytes reproducible. + /// registries and runtimes expect for image layers. Every entry gets a fixed extended-attribute set (see + /// ) and a fixed rather than + /// the current time. /// - private static async Task WriteTarAsync( + private static async Task WriteTarAsync( SortedDictionary files, SortedSet directories, + Stream destination, CancellationToken cancellationToken) { - using var tarBuffer = new MemoryStream(); - - // leaveOpen: true so we can read the buffer back after the writer is disposed/flushed. - await using (var tar = new TarWriter(tarBuffer, TarEntryFormat.Pax, leaveOpen: true)) + // leaveOpen: true — the destination is the hashing/compression pipeline, finalized by the caller. + await using (var tar = new TarWriter(destination, TarEntryFormat.Pax, leaveOpen: true)) { // Directories first, shallowest-to-deepest (the sorted set already yields a stable order), so every // file's parent exists before the file entry. Both collections are ordinally sorted for determinism. @@ -311,8 +342,6 @@ private static async Task WriteTarAsync( await tar.WriteEntryAsync(fileEntry, cancellationToken).ConfigureAwait(false); } } - - return tarBuffer.ToArray(); } /// A resolved file to place in the layer: its source path and the Unix mode to stamp on it. @@ -322,11 +351,8 @@ private static async Task WriteTarAsync( /// The PAX extended attributes applied to every entry: fixed mtime, atime and ctime. /// /// - /// On .NET 8 the PAX writer auto-populates atime and ctime in the extended header with the - /// current wall-clock time when they are not supplied, which makes the archive bytes (and thus the layer - /// digest) differ on every run. .NET 9+ stopped doing this. Pinning all three timestamps to a fixed epoch - /// here makes the output reproducible on every supported runtime. (Verified by byte-diffing two archives - /// built from identical input on .NET 8.) + /// On .NET 8 the PAX writer otherwise fills atime and ctime with the current wall-clock time; + /// pinning all three to a fixed epoch keeps them out of the archive. /// private static Dictionary ReproducibleExtendedAttributes() { @@ -339,51 +365,6 @@ private static Dictionary ReproducibleExtendedAttributes() }; } - /// - /// Gzip-compresses the supplied bytes, then normalizes the gzip member header so the output is byte-for-byte - /// reproducible across runs and target frameworks. - /// - /// - /// A gzip member begins with a 10-byte header whose bytes 4–7 hold a modification time (MTIME) and whose - /// byte 9 holds the source OS. populates these with values that vary by run and by - /// runtime (for example net8.0 writes the current time into MTIME while net9.0+ zeroes it). Left as-is, the - /// compressed digest would change run-to-run even for identical input. We therefore zero MTIME and - /// set the OS byte to 0xFF ("unknown"), matching the convention used by reproducible-build tooling. - /// - private static byte[] Compress(byte[] data) - { - using var output = new MemoryStream(); - // SmallestSize keeps layer blobs compact; layer payloads here are modest, so the extra CPU is cheap. - using (var gzip = new GZipStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) - { - gzip.Write(data, 0, data.Length); - } - - byte[] bytes = output.ToArray(); - - // Normalize the gzip header for deterministic output. The header is present whenever there is output; - // a valid gzip stream is always at least the 10-byte header plus the 8-byte trailer. - if (bytes.Length >= 10) - { - // Bytes 4–7: MTIME (little-endian) -> zero. - bytes[4] = 0; - bytes[5] = 0; - bytes[6] = 0; - bytes[7] = 0; - // Byte 9: OS -> 0xFF (unknown), so it does not vary by platform. - bytes[9] = 0xFF; - } - - return bytes; - } - - private static string ComputeDigest(byte[] data) - { - byte[] hash = SHA256.HashData(data); - // Convert.ToHexStringLower exists only on net9.0+, so lower-case explicitly for net8.0 compatibility. - return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); - } - /// /// Normalizes a caller-supplied container path to an absolute, forward-slash path with no trailing slash /// and no redundant segments. diff --git a/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs b/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs new file mode 100644 index 000000000..d57b9c99a --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs @@ -0,0 +1,166 @@ +using System.Security.Cryptography; + +namespace Amazon.OCI.Builder.Images; + +/// +/// A write-only pass-through stream that feeds every byte written to an and counts +/// them before forwarding to an inner stream. Used to compute a layer's digests (and its compressed size) in the +/// same single pass that writes the layer, so the layer never has to be held in memory. +/// +/// The inner stream is never disposed by this wrapper; its owner remains responsible for it. +internal sealed class HashingWriteStream : Stream +{ + private readonly Stream _inner; + private readonly IncrementalHash _hash; + + public HashingWriteStream(Stream inner, IncrementalHash hash) + { + _inner = inner; + _hash = hash; + } + + /// The total number of bytes written through this stream. + public long BytesWritten { get; private set; } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + + public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); + + public override void Write(ReadOnlySpan buffer) + { + _hash.AppendData(buffer); + BytesWritten += buffer.Length; + _inner.Write(buffer); + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + _hash.AppendData(buffer.Span); + BytesWritten += buffer.Length; + return _inner.WriteAsync(buffer, cancellationToken); + } + + public override void Flush() => _inner.Flush(); + public override Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); +} + +/// +/// A write-only pass-through stream that normalizes the 10-byte gzip member header as it is written, so the +/// header carries no build time or OS byte. +/// +/// +/// +/// A gzip member begins with a 10-byte header whose bytes 4–7 hold a modification time (MTIME) and whose byte 9 +/// holds the source OS. fills these with values that vary by run +/// and by runtime (net8.0 writes the current time into MTIME; net9.0+ zeroes it). This stream rewrites MTIME to +/// zero and the OS byte to 0xFF ("unknown") as those bytes pass through. +/// +/// +/// The rewrite is driven by the absolute stream position, so it is correct however the writer chunks the header +/// (one byte at a time, or one write spanning the header and beyond). The caller's buffer is never modified: the +/// header slice is copied before it is patched. The inner stream is never disposed by this wrapper. +/// +/// +internal sealed class GzipHeaderNormalizingStream : Stream +{ + /// Length of the fixed gzip member header (RFC 1952 §2.3). + internal const int HeaderLength = 10; + + private readonly Stream _inner; + private long _position; + + public GzipHeaderNormalizingStream(Stream inner) => _inner = inner; + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + + public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count)); + + public override void Write(ReadOnlySpan buffer) + { + int headerBytes = HeaderBytesIn(buffer.Length); + if (headerBytes > 0) + { + Span patched = stackalloc byte[HeaderLength]; + patched = patched[..headerBytes]; + buffer[..headerBytes].CopyTo(patched); + Normalize(patched, _position); + _inner.Write(patched); + _position += headerBytes; + buffer = buffer[headerBytes..]; + } + + if (!buffer.IsEmpty) + { + _inner.Write(buffer); + _position += buffer.Length; + } + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + int headerBytes = HeaderBytesIn(buffer.Length); + if (headerBytes > 0) + { + byte[] patched = buffer[..headerBytes].ToArray(); + Normalize(patched, _position); + await _inner.WriteAsync(patched, cancellationToken).ConfigureAwait(false); + _position += headerBytes; + buffer = buffer[headerBytes..]; + } + + if (!buffer.IsEmpty) + { + await _inner.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + _position += buffer.Length; + } + } + + public override void Flush() => _inner.Flush(); + public override Task FlushAsync(CancellationToken cancellationToken) => _inner.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + + /// How many of the next bytes still fall inside the gzip header. + private int HeaderBytesIn(int count) + => _position >= HeaderLength ? 0 : (int)Math.Min(count, HeaderLength - _position); + + /// + /// Patches the header bytes in , which begins at absolute header offset + /// : offsets 4–7 (MTIME) become zero and offset 9 (OS) becomes 0xFF. + /// + private static void Normalize(Span segment, long start) + { + for (int i = 0; i < segment.Length; i++) + { + long headerOffset = start + i; + if (headerOffset is >= 4 and <= 7) + { + segment[i] = 0; + } + else if (headerOffset == 9) + { + segment[i] = 0xFF; + } + } + } +} diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs index 2995afc8c..73b807b7e 100644 --- a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -210,18 +210,17 @@ public async Task AddLayerAsync(FileManifest files, string? createdBy = null, Ca ThrowIfDisposed(); ArgumentNullException.ThrowIfNull(files); - // Build the layer with the media type matching the image schema so the manifest stays consistent. - Layer layer = await Layer.CreateAsync(files, _imageBuilder.LayerMediaType, cancellationToken).ConfigureAwait(false); - - // Spill the compressed layer to a temp file FIRST, so it is not retained in memory for the builder's - // lifetime. The file is created with owner-only permissions (see SpillFile) so other local users cannot - // read a private layer from the shared temp directory. The builder owns the file and deletes it on Dispose. + // Build the layer straight into an owner-only temp file (see SpillFile) in a single streaming pass, so it + // is never held in memory — layers can exceed the ~2 GiB a byte[]/MemoryStream could hold. The builder + // owns the file and deletes it on Dispose (or immediately, if building fails or is cancelled). The media + // type matches the image schema so the manifest stays consistent. string layerFile = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"aws-oci-built-layer-{Guid.NewGuid():N}.tmp"); + Layer layer; try { await using (var file = Registry.SpillFile.CreateWrite(layerFile)) { - await file.WriteAsync(layer.GzippedTarBytes, cancellationToken).ConfigureAwait(false); + layer = await Layer.CreateAsync(files, file, _imageBuilder.LayerMediaType, cancellationToken).ConfigureAwait(false); } } catch @@ -235,7 +234,7 @@ public async Task AddLayerAsync(FileManifest files, string? createdBy = null, Ca // write, leave the manifest referencing a layer whose blob never materializes — a later push would then // reference a missing blob, and a retry could duplicate the layer entry. _imageBuilder.AddLayer(layer, createdBy); - _pendingLayers.Add(new PendingLayer(layer.Digest, layerFile, layer.GzippedTarBytes.LongLength)); + _pendingLayers.Add(new PendingLayer(layer.Digest, layerFile, layer.Size)); } /// diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs index 6782a7994..58c6cccd2 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs @@ -173,17 +173,23 @@ private static string BaseManifest(string layerMediaType, string configMediaType {"schemaVersion":2,"mediaType":"{{MediaTypes.OciManifestV1}}","config":{"mediaType":"{{configMediaType}}","size":10,"digest":"sha256:basecfg"},"layers":[{"mediaType":"{{layerMediaType}}","size":100,"digest":"sha256:baselayer"}]} """; - private static Layer FakeLayer(string mediaType) => - // Build a real tiny layer so Digest/DiffId are genuine sha256 values. - Layer.CreateAsync(MakeManifest(), mediaType).GetAwaiter().GetResult(); - - private static FileManifest MakeManifest() + private static Layer FakeLayer(string mediaType) { + // Build a real tiny layer so Digest/DiffId are genuine sha256 values. The layer is fully built (into a + // throwaway stream) before returning, so its source directory can be removed immediately. string tmp = Path.Combine(Path.GetTempPath(), "aws-oci-ib-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(tmp); - string file = Path.Combine(tmp, "app.dll"); - File.WriteAllText(file, "payload"); - return new FileManifest { { file, "/app/app.dll" } }; + try + { + string file = Path.Combine(tmp, "app.dll"); + File.WriteAllText(file, "payload"); + var manifest = new FileManifest { { file, "/app/app.dll" } }; + return Layer.CreateAsync(manifest, new MemoryStream(), mediaType).GetAwaiter().GetResult(); + } + finally + { + Directory.Delete(tmp, recursive: true); + } } [Fact] diff --git a/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs new file mode 100644 index 000000000..d1359d6bb --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs @@ -0,0 +1,215 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using Amazon.OCI.Builder; +using Amazon.OCI.Builder.Images; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +/// +/// Tests for the streaming layer pipeline: the gzip-header normalizer and hashing pass-through streams, and the +/// destination-stream contract. +/// +public class LayerStreamingTests : IDisposable +{ + private readonly string _tempRoot; + + public LayerStreamingTests() + { + _tempRoot = Path.Combine(Path.GetTempPath(), "aws-oci-builder-stream-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempRoot); + } + + public void Dispose() + { + try { Directory.Delete(_tempRoot, recursive: true); } catch { /* best effort cleanup */ } + } + + private FileManifest SampleManifest() + { + string file = Path.Combine(_tempRoot, "app.dll"); + // Incompressible content so deflate emits several blocks and many writes reach the sink. + var data = new byte[512 * 1024]; + new Random(42).NextBytes(data); + File.WriteAllBytes(file, data); + return new FileManifest { { file, "/app/app.dll" } }; + } + + // ---- GzipHeaderNormalizingStream ------------------------------------------------------------- + + /// A fake "gzip" byte sequence whose header has non-normalized MTIME/OS fields, plus a body. + private static byte[] RawGzipLikeBytes() + { + byte[] bytes = Enumerable.Range(0, 32).Select(i => (byte)(0xA0 + i)).ToArray(); + bytes[0] = 0x1F; bytes[1] = 0x8B; // magic, untouched by the normalizer + return bytes; + } + + private static byte[] Expected(byte[] raw) + { + byte[] expected = (byte[])raw.Clone(); + expected[4] = expected[5] = expected[6] = expected[7] = 0; + expected[9] = 0xFF; + return expected; + } + + public static IEnumerable ChunkPatterns() => new[] + { + new object[] { new[] { 32 } }, // whole buffer in one write + new object[] { Enumerable.Repeat(1, 32).ToArray() }, // one byte at a time + new object[] { new[] { 3, 2, 4, 1, 22 } }, // splits inside MTIME and right before OS + new object[] { new[] { 9, 1, 22 } }, // OS byte written alone + new object[] { new[] { 10, 22 } }, // exactly the header, then the body + new object[] { new[] { 11, 21 } }, // a write spanning the header boundary + }; + + [Theory] + [MemberData(nameof(ChunkPatterns))] + public void Normalizer_patches_header_regardless_of_write_chunking(int[] chunks) + { + byte[] raw = RawGzipLikeBytes(); + byte[] original = (byte[])raw.Clone(); + var sink = new MemoryStream(); + var normalizer = new GzipHeaderNormalizingStream(sink); + + int offset = 0; + foreach (int chunk in chunks) + { + normalizer.Write(raw, offset, chunk); + offset += chunk; + } + + Assert.Equal(Expected(raw), sink.ToArray()); + Assert.Equal(original, raw); // the caller's buffer is never modified + } + + [Theory] + [MemberData(nameof(ChunkPatterns))] + public async Task Normalizer_patches_header_regardless_of_write_chunking_async(int[] chunks) + { + byte[] raw = RawGzipLikeBytes(); + byte[] original = (byte[])raw.Clone(); + var sink = new MemoryStream(); + var normalizer = new GzipHeaderNormalizingStream(sink); + + int offset = 0; + foreach (int chunk in chunks) + { + await normalizer.WriteAsync(raw.AsMemory(offset, chunk)); + offset += chunk; + } + + Assert.Equal(Expected(raw), sink.ToArray()); + Assert.Equal(original, raw); + } + + // ---- HashingWriteStream ----------------------------------------------------------------------- + + [Fact] + public async Task HashingWriteStream_hashes_and_counts_bytes_across_sync_and_async_writes() + { + byte[] data = Enumerable.Range(0, 1000).Select(i => (byte)i).ToArray(); + var sink = new MemoryStream(); + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var stream = new HashingWriteStream(sink, hash); + + stream.Write(data, 0, 100); + stream.Write(data.AsSpan(100, 400)); + await stream.WriteAsync(data.AsMemory(500, 500)); + + Assert.Equal(1000, stream.BytesWritten); + Assert.Equal(data, sink.ToArray()); + Assert.Equal(SHA256.HashData(data), hash.GetHashAndReset()); + } + + // ---- Layer.CreateAsync destination contract --------------------------------------------------- + + [Fact] + public async Task CreateAsync_streams_bytes_matching_the_reported_digests_and_size() + { + var destination = new MemoryStream(); + Layer layer = await Layer.CreateAsync(SampleManifest(), destination); + byte[] blob = destination.ToArray(); + + Assert.Equal(blob.LongLength, layer.Size); + Assert.Equal(RegistryDigest(blob), layer.Digest); + Assert.Equal(RegistryDigest(Gunzip(blob)), layer.DiffId); + } + + [Fact] + public async Task CreateAsync_output_matches_one_shot_compression_of_the_same_tar() + { + // Guards that streaming the tar into the compressor in many small writes produces exactly the bytes the + // previous implementation produced by compressing the whole buffered tar in a single write. + var destination = new MemoryStream(); + await Layer.CreateAsync(SampleManifest(), destination); + byte[] streamed = destination.ToArray(); + + byte[] oneShot = OneShotCompress(Gunzip(streamed)); + + Assert.Equal(oneShot, streamed); + } + + [Fact] + public async Task CreateAsync_normalizes_the_gzip_header() + { + var destination = new MemoryStream(); + await Layer.CreateAsync(SampleManifest(), destination); + byte[] blob = destination.ToArray(); + + Assert.Equal(new byte[] { 0, 0, 0, 0 }, blob[4..8]); // MTIME + Assert.Equal(0xFF, blob[9]); // OS = unknown + } + + [Fact] + public async Task CreateAsync_leaves_destination_open_and_appends_at_its_current_position() + { + byte[] prefix = "existing-content"u8.ToArray(); + var destination = new MemoryStream(); + destination.Write(prefix); + + Layer layer = await Layer.CreateAsync(SampleManifest(), destination); + + Assert.True(destination.CanWrite); // not disposed/closed + byte[] all = destination.ToArray(); + Assert.Equal(prefix, all[..prefix.Length]); + Assert.Equal(prefix.Length + layer.Size, all.LongLength); + Assert.Equal(layer.Digest, RegistryDigest(all[prefix.Length..])); + } + + [Fact] + public async Task CreateAsync_validates_the_destination() + { + await Assert.ThrowsAsync(() => Layer.CreateAsync(SampleManifest(), null!)); + + using var readOnly = new MemoryStream(new byte[16], writable: false); + await Assert.ThrowsAsync(() => Layer.CreateAsync(SampleManifest(), readOnly)); + } + + // ---- helpers ---------------------------------------------------------------------------------- + + private static string RegistryDigest(byte[] data) + => "sha256:" + Convert.ToHexString(SHA256.HashData(data)).ToLowerInvariant(); + + private static byte[] Gunzip(byte[] gz) + { + using var gunzip = new GZipStream(new MemoryStream(gz), CompressionMode.Decompress); + using var tar = new MemoryStream(); + gunzip.CopyTo(tar); + return tar.ToArray(); + } + + /// The previous implementation: compress the whole tar in one write, then normalize the header. + private static byte[] OneShotCompress(byte[] tar) + { + using var output = new MemoryStream(); + using (var gzip = new GZipStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + { + gzip.Write(tar, 0, tar.Length); + } + byte[] bytes = output.ToArray(); + bytes[4] = bytes[5] = bytes[6] = bytes[7] = 0; + bytes[9] = 0xFF; + return bytes; + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs index 784d342ef..98925d108 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs @@ -22,6 +22,24 @@ public void Dispose() try { Directory.Delete(_tempRoot, recursive: true); } catch { /* best effort cleanup */ } } + /// A built layer plus the compressed bytes streamed out. + private sealed record BuiltLayer(Layer Layer, byte[] GzippedTarBytes) + { + public string Digest => Layer.Digest; + public string DiffId => Layer.DiffId; + public long Size => Layer.Size; + public string MediaType => Layer.MediaType; + public Descriptor ToDescriptor() => Layer.ToDescriptor(); + } + + /// Builds a layer into an in-memory destination (fine for these small fixtures). + private static async Task BuildAsync(FileManifest manifest, string mediaType = MediaTypes.OciLayerGzipV1) + { + var destination = new MemoryStream(); + Layer layer = await Layer.CreateAsync(manifest, destination, mediaType); + return new BuiltLayer(layer, destination.ToArray()); + } + private string WriteFile(string relativePath, string content) { string full = Path.Combine(_tempRoot, relativePath); @@ -36,7 +54,7 @@ public async Task Builds_layer_with_both_digests_populated() string file = WriteFile("app.dll", "hello"); var manifest = new FileManifest { { file, "/app/app.dll" } }; - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); Assert.StartsWith("sha256:", layer.Digest); Assert.StartsWith("sha256:", layer.DiffId); @@ -52,7 +70,7 @@ public async Task Digest_and_diffId_differ_because_one_is_compressed() string file = WriteFile("app.dll", new string('x', 4096)); var manifest = new FileManifest { { file, "/app/app.dll" } }; - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); // Digest is over the gzipped bytes; DiffId is over the raw tar. They must not be equal. Assert.NotEqual(layer.Digest, layer.DiffId); @@ -64,7 +82,7 @@ public async Task Descriptor_reflects_layer_fields() string file = WriteFile("app.dll", "hello"); var manifest = new FileManifest { { file, "/app/app.dll" } }; - Layer layer = await Layer.CreateAsync(manifest, MediaTypes.DockerLayerGzip); + BuiltLayer layer = await BuildAsync(manifest, MediaTypes.DockerLayerGzip); Descriptor descriptor = layer.ToDescriptor(); Assert.Equal(MediaTypes.DockerLayerGzip, descriptor.MediaType); @@ -79,8 +97,8 @@ public async Task Is_reproducible_identical_inputs_yield_identical_digests() var manifest1 = new FileManifest { { file, "/app/app.dll" } }; var manifest2 = new FileManifest { { file, "/app/app.dll" } }; - Layer a = await Layer.CreateAsync(manifest1); - Layer b = await Layer.CreateAsync(manifest2); + BuiltLayer a = await BuildAsync(manifest1); + BuiltLayer b = await BuildAsync(manifest2); Assert.Equal(a.Digest, b.Digest); Assert.Equal(a.DiffId, b.DiffId); @@ -95,8 +113,8 @@ public async Task Entry_order_does_not_affect_digest() var ordered = new FileManifest { { f1, "/app/a.txt" }, { f2, "/app/b.txt" } }; var reversed = new FileManifest { { f2, "/app/b.txt" }, { f1, "/app/a.txt" } }; - Layer a = await Layer.CreateAsync(ordered); - Layer b = await Layer.CreateAsync(reversed); + BuiltLayer a = await BuildAsync(ordered); + BuiltLayer b = await BuildAsync(reversed); Assert.Equal(a.Digest, b.Digest); } @@ -107,7 +125,7 @@ public async Task Tar_contains_file_but_not_synthesized_ancestor_directories() string file = WriteFile("app.dll", "hello"); var manifest = new FileManifest { { file, "/app/bin/app.dll" } }; - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); // The file is present (no leading slash). Its ancestor directories are NOT synthesized as tar entries: @@ -127,7 +145,7 @@ public async Task Directory_mapping_recurses_preserving_subpaths() var manifest = new FileManifest { { dir, "/app/data" } }; - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/data/root.txt" }); @@ -145,7 +163,7 @@ public async Task Directory_mapping_preserves_empty_descendant_directories() var manifest = new FileManifest { { dir, "/app/data" } }; - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); List<(TarEntryType Type, string Name)> entries = ReadTarEntries(layer.GzippedTarBytes); Assert.Contains(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/data/root.txt" }); @@ -168,7 +186,7 @@ public async Task Symlinked_source_file_is_rejected() // A symlinked source must be rejected rather than silently followed (which could pull in files from // outside the mapped set, e.g. host secrets). - await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + await Assert.ThrowsAsync(() => BuildAsync(manifest)); } [Fact] @@ -187,21 +205,21 @@ public async Task Symlink_inside_mapped_directory_is_rejected() var manifest = new FileManifest { { dir, "/app" } }; - await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + await Assert.ThrowsAsync(() => BuildAsync(manifest)); } [Fact] public async Task Empty_manifest_throws() { var manifest = new FileManifest(); - await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + await Assert.ThrowsAsync(() => BuildAsync(manifest)); } [Fact] public async Task Missing_source_throws_FileNotFound() { var manifest = new FileManifest { { Path.Combine(_tempRoot, "does-not-exist"), "/app/x" } }; - await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + await Assert.ThrowsAsync(() => BuildAsync(manifest)); } [Theory] @@ -212,7 +230,7 @@ public async Task Invalid_container_path_throws(string containerPath) { string file = WriteFile("app.dll", "hello"); var manifest = new FileManifest { { file, containerPath } }; - await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + await Assert.ThrowsAsync(() => BuildAsync(manifest)); } [Fact] @@ -221,7 +239,7 @@ public async Task Duplicate_container_path_throws() string f1 = WriteFile("a.txt", "one"); string f2 = WriteFile("b.txt", "two"); var manifest = new FileManifest { { f1, "/app/same.txt" }, { f2, "/app/same.txt" } }; - await Assert.ThrowsAsync(() => Layer.CreateAsync(manifest)); + await Assert.ThrowsAsync(() => BuildAsync(manifest)); } /// Decompresses the gzipped tar and returns its real entries (type + name) in order. @@ -279,7 +297,7 @@ public async Task Default_file_mode_is_0644_and_directory_mode_is_0755() string dir = Directory.CreateDirectory(Path.Combine(_tempRoot, "assets")).FullName; var manifest = new FileManifest { { file, "/app/app.dll" }, { dir, "/app/assets" } }; - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); // On Windows the source has no mode bits, so the file defaults to 0644. On Unix the source file written by @@ -299,7 +317,7 @@ public async Task Explicit_mode_overrides_the_default() string file = WriteFile("run.sh", "#!/bin/sh\necho hi\n"); var manifest = new FileManifest { { file, "/app/run.sh", Mode0750 } }; - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); var fileEntry = Assert.Single(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/run.sh" }); @@ -313,7 +331,7 @@ public async Task AddExecutable_marks_the_file_0755() var manifest = new FileManifest(); manifest.AddExecutable(file, "/app/entrypoint.sh"); - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); var fileEntry = Assert.Single(entries, e => e is { Type: TarEntryType.RegularFile, Name: "app/entrypoint.sh" }); @@ -329,7 +347,7 @@ public async Task Explicit_mode_applies_to_every_file_in_a_mapped_directory() var manifest = new FileManifest { { dir, "/app/scripts", FileManifestEntry.ExecutableMode } }; - Layer layer = await Layer.CreateAsync(manifest); + BuiltLayer layer = await BuildAsync(manifest); var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); var a = Assert.Single(entries, e => e.Name == "app/scripts/a.sh"); @@ -339,7 +357,7 @@ public async Task Explicit_mode_applies_to_every_file_in_a_mapped_directory() } [Fact] - public void Executable_bit_from_source_is_preserved_on_unix() + public async Task Executable_bit_from_source_is_preserved_on_unix() { // On Windows there is no source executable bit to read, so this behavior only applies to Unix hosts. if (OperatingSystem.IsWindows()) @@ -351,7 +369,7 @@ public void Executable_bit_from_source_is_preserved_on_unix() File.SetUnixFileMode(file, Mode0755); var manifest = new FileManifest { { file, "/app/tool" } }; // no explicit mode -> derive from source - Layer layer = Layer.CreateAsync(manifest).GetAwaiter().GetResult(); + BuiltLayer layer = await BuildAsync(manifest); var entries = ReadTarEntriesWithMode(layer.GzippedTarBytes); var fileEntry = Assert.Single(entries, e => e.Name == "app/tool"); @@ -368,8 +386,8 @@ public async Task Explicit_mode_is_deterministic_in_the_digest() var m2 = new FileManifest(); m2.AddExecutable(file, "/app/run.sh"); - Layer a = await Layer.CreateAsync(m1); - Layer b = await Layer.CreateAsync(m2); + BuiltLayer a = await BuildAsync(m1); + BuiltLayer b = await BuildAsync(m2); Assert.Equal(a.Digest, b.Digest); } @@ -382,8 +400,8 @@ public async Task Changing_the_mode_changes_the_digest() var executable = new FileManifest(); executable.AddExecutable(file, "/app/run.sh"); - Layer a = await Layer.CreateAsync(plain); - Layer b = await Layer.CreateAsync(executable); + BuiltLayer a = await BuildAsync(plain); + BuiltLayer b = await BuildAsync(executable); // Same content, different permissions -> different layer bytes -> different digest. Assert.NotEqual(a.Digest, b.Digest); diff --git a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs index 8d4212e24..ba15584ce 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs @@ -304,7 +304,11 @@ public async Task AddLayerAsync_spills_built_layer_to_disk_and_Dispose_deletes_i const UnixFileMode groupOrOther = UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute; - Assert.All(afterAdd, path => Assert.Equal((UnixFileMode)0, File.GetUnixFileMode(path) & groupOrOther)); + // A plain loop (not an Assert.All lambda) keeps the call inside the platform guard for CA1416. + foreach (string path in afterAdd) + { + Assert.Equal((UnixFileMode)0, File.GetUnixFileMode(path) & groupOrOther); + } } builder.Dispose(); From f831260427eb4ab8cef4b1b0b7aaf162996b305e Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 28 Oct 2026 17:03:49 +0000 Subject: [PATCH 11/14] Rename base namespace from Amazon.OCI.Builder to AWS.OCI.Builder Match the AWS.OCI.Builder project and assembly name. Updates RootNamespace, all namespace/using declarations in the library and tests, and the README sample. --- Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj | 2 +- Libraries/src/AWS.OCI.Builder/Descriptor.cs | 2 +- Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs | 2 +- Libraries/src/AWS.OCI.Builder/FileManifest.cs | 2 +- Libraries/src/AWS.OCI.Builder/ImageReference.cs | 2 +- Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs | 2 +- Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs | 2 +- Libraries/src/AWS.OCI.Builder/Images/Layer.cs | 2 +- Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs | 2 +- Libraries/src/AWS.OCI.Builder/Images/Manifest.cs | 2 +- Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs | 4 ++-- Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs | 2 +- Libraries/src/AWS.OCI.Builder/MediaTypes.cs | 2 +- Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs | 6 +++--- Libraries/src/AWS.OCI.Builder/README.md | 2 +- .../src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs | 2 +- .../AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs | 2 +- Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs | 2 +- Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs | 2 +- .../src/AWS.OCI.Builder/Registry/RegistryExceptions.cs | 2 +- Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs | 2 +- Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs | 2 +- Libraries/src/AWS.OCI.Builder/UnixPermissions.cs | 2 +- .../EcrPushIntegrationTests.cs | 2 +- .../test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs | 2 +- .../test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs | 2 +- .../test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs | 2 +- Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs | 2 +- Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs | 4 ++-- Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs | 2 +- Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs | 4 ++-- Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs | 4 ++-- .../test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs | 6 +++--- Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs | 2 +- .../test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs | 2 +- 35 files changed, 43 insertions(+), 43 deletions(-) diff --git a/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj b/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj index 6ee1ba8e5..73d472b8e 100644 --- a/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj +++ b/Libraries/src/AWS.OCI.Builder/AWS.OCI.Builder.csproj @@ -6,7 +6,7 @@ $(DefaultPackageTargets) AWS.OCI.Builder AWS.OCI.Builder - Amazon.OCI.Builder + AWS.OCI.Builder A .NET library for building OCI container images programmatically — overlay local files as a new layer on a base image and push the result to Amazon ECR, without a Docker daemon. enable enable diff --git a/Libraries/src/AWS.OCI.Builder/Descriptor.cs b/Libraries/src/AWS.OCI.Builder/Descriptor.cs index 4086164d3..9d7da4ca2 100644 --- a/Libraries/src/AWS.OCI.Builder/Descriptor.cs +++ b/Libraries/src/AWS.OCI.Builder/Descriptor.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace Amazon.OCI.Builder; +namespace AWS.OCI.Builder; /// /// An OCI content descriptor: a typed, content-addressable reference to a blob diff --git a/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs index 439ce51fa..ec904cd04 100644 --- a/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs +++ b/Libraries/src/AWS.OCI.Builder/EcrPushOptions.cs @@ -1,4 +1,4 @@ -namespace Amazon.OCI.Builder; +namespace AWS.OCI.Builder; /// /// Options controlling how an image is pushed to Amazon ECR. diff --git a/Libraries/src/AWS.OCI.Builder/FileManifest.cs b/Libraries/src/AWS.OCI.Builder/FileManifest.cs index ef5742d03..62639c5cf 100644 --- a/Libraries/src/AWS.OCI.Builder/FileManifest.cs +++ b/Libraries/src/AWS.OCI.Builder/FileManifest.cs @@ -1,6 +1,6 @@ using System.Collections; -namespace Amazon.OCI.Builder; +namespace AWS.OCI.Builder; /// /// Describes a set of local files and/or directories to add to an image, each mapped to an absolute diff --git a/Libraries/src/AWS.OCI.Builder/ImageReference.cs b/Libraries/src/AWS.OCI.Builder/ImageReference.cs index 08ea7e29e..f17491caf 100644 --- a/Libraries/src/AWS.OCI.Builder/ImageReference.cs +++ b/Libraries/src/AWS.OCI.Builder/ImageReference.cs @@ -1,6 +1,6 @@ using System.Text.RegularExpressions; -namespace Amazon.OCI.Builder; +namespace AWS.OCI.Builder; /// /// A parsed container image reference: the registry host, repository, and a tag or digest reference. diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs index c074be358..d58153680 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs @@ -1,6 +1,6 @@ using System.Security.Cryptography; -namespace Amazon.OCI.Builder.Images; +namespace AWS.OCI.Builder.Images; /// /// Assembles a derived image from a base image's config and manifest plus one or more overlaid diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs index 956d6a745..e399041b5 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Nodes; -namespace Amazon.OCI.Builder.Images; +namespace AWS.OCI.Builder.Images; /// /// A mutable view over an OCI / Docker image configuration JSON document — the object referenced by an image diff --git a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs index a89f1fe1c..47c77420c 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs @@ -2,7 +2,7 @@ using System.IO.Compression; using System.Security.Cryptography; -namespace Amazon.OCI.Builder.Images; +namespace AWS.OCI.Builder.Images; /// /// The metadata of a single image layer produced from a — a gzipped tar archive of diff --git a/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs b/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs index d57b9c99a..aeea6769f 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/LayerStreams.cs @@ -1,6 +1,6 @@ using System.Security.Cryptography; -namespace Amazon.OCI.Builder.Images; +namespace AWS.OCI.Builder.Images; /// /// A write-only pass-through stream that feeds every byte written to an and counts diff --git a/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs index d50f4731d..06ee4a48a 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace Amazon.OCI.Builder.Images; +namespace AWS.OCI.Builder.Images; /// /// An image manifest (OCI image manifest or Docker V2 Schema 2): the single-platform document that references the diff --git a/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs b/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs index 3eeee7918..5b9c26c75 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/OciJsonContext.cs @@ -2,9 +2,9 @@ using System.Text.Json; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; -using Amazon.OCI.Builder.Registry; +using AWS.OCI.Builder.Registry; -namespace Amazon.OCI.Builder.Images; +namespace AWS.OCI.Builder.Images; /// /// The source-generated for the strongly-typed documents this library diff --git a/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs b/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs index 6ab112b43..660d67f98 100644 --- a/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs +++ b/Libraries/src/AWS.OCI.Builder/LayerPushProgress.cs @@ -1,4 +1,4 @@ -namespace Amazon.OCI.Builder; +namespace AWS.OCI.Builder; /// /// The lifecycle phase a layer is in when a event is reported. diff --git a/Libraries/src/AWS.OCI.Builder/MediaTypes.cs b/Libraries/src/AWS.OCI.Builder/MediaTypes.cs index 29235977a..5e5fba11e 100644 --- a/Libraries/src/AWS.OCI.Builder/MediaTypes.cs +++ b/Libraries/src/AWS.OCI.Builder/MediaTypes.cs @@ -1,4 +1,4 @@ -namespace Amazon.OCI.Builder; +namespace AWS.OCI.Builder; /// /// Well-known media type strings used by the OCI Image Specification and the legacy diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs index 73b807b7e..54b0f5920 100644 --- a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -1,7 +1,7 @@ -using Amazon.OCI.Builder.Images; -using Amazon.OCI.Builder.Registry; +using AWS.OCI.Builder.Images; +using AWS.OCI.Builder.Registry; -namespace Amazon.OCI.Builder; +namespace AWS.OCI.Builder; /// /// The programmatic entry point for building an OCI container image from a base image plus local files and diff --git a/Libraries/src/AWS.OCI.Builder/README.md b/Libraries/src/AWS.OCI.Builder/README.md index 7c9dd0791..5387828a2 100644 --- a/Libraries/src/AWS.OCI.Builder/README.md +++ b/Libraries/src/AWS.OCI.Builder/README.md @@ -37,7 +37,7 @@ string authorizationToken = auth.AuthorizationData[0].AuthorizationToken; // b ## Usage ```csharp -using Amazon.OCI.Builder; +using AWS.OCI.Builder; // Pull the base image (selects the right platform from a manifest list). using var builder = await OciImageBuilder.FromBaseImageAsync( diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index 7a74f28af..09f4e75bd 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -6,7 +6,7 @@ using System.Text.Json.Serialization; using System.Web; -namespace Amazon.OCI.Builder.Registry; +namespace AWS.OCI.Builder.Registry; /// /// A that performs the Docker/OCI registry authentication handshake. diff --git a/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs index d072f0a8d..bdf2f5c05 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs @@ -1,4 +1,4 @@ -namespace Amazon.OCI.Builder.Registry; +namespace AWS.OCI.Builder.Registry; /// /// Supplies the username/password a registry's authentication handshake needs for a given registry host. diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs b/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs index cde5f4457..04210103e 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RealmValidator.cs @@ -1,7 +1,7 @@ using System.Net; using System.Net.Sockets; -namespace Amazon.OCI.Builder.Registry; +namespace AWS.OCI.Builder.Registry; /// /// Validates the realm (token endpoint) URL returned in a registry's WWW-Authenticate diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs index 4f1056684..8719e7585 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -2,7 +2,7 @@ using System.Net.Http.Headers; using System.Security.Cryptography; -namespace Amazon.OCI.Builder.Registry; +namespace AWS.OCI.Builder.Registry; /// /// A client for the OCI Distribution / Docker Registry HTTP API (v2): pull and push of manifests and blobs. diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs index 8ffca17d9..5586c5630 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs @@ -1,4 +1,4 @@ -namespace Amazon.OCI.Builder.Registry; +namespace AWS.OCI.Builder.Registry; /// /// The base type for errors raised while interacting with a container registry — authentication, diff --git a/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs b/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs index 33a5faf55..ad77dfb87 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/SpillFile.cs @@ -1,4 +1,4 @@ -namespace Amazon.OCI.Builder.Registry; +namespace AWS.OCI.Builder.Registry; /// /// Helpers for creating the temporary "spill" files this library writes layer/blob bytes to, with owner-only diff --git a/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs b/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs index 929f83960..6bef6e3b9 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace Amazon.OCI.Builder.Registry; +namespace AWS.OCI.Builder.Registry; /// The JSON body returned by a registry token endpoint during the Bearer auth handshake. /// diff --git a/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs b/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs index f80c60022..0c7d98697 100644 --- a/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs +++ b/Libraries/src/AWS.OCI.Builder/UnixPermissions.cs @@ -1,4 +1,4 @@ -namespace Amazon.OCI.Builder; +namespace AWS.OCI.Builder; /// /// Helpers for expressing Unix file permissions the way chmod does — in octal — and converting them to the diff --git a/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs b/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs index 20396de0b..094b131ee 100644 --- a/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs +++ b/Libraries/test/AWS.OCI.Builder.IntegrationTests/EcrPushIntegrationTests.cs @@ -1,7 +1,7 @@ using Amazon; using Amazon.ECR; using Amazon.ECR.Model; -using Amazon.OCI.Builder; +using AWS.OCI.Builder; using Xunit; using Xunit.Abstractions; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs index 9ac1e5598..d3d9f615a 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/AuthHandshakeHandlerTests.cs @@ -1,6 +1,6 @@ using System.Net; using System.Net.Http.Headers; -using Amazon.OCI.Builder.Registry; +using AWS.OCI.Builder.Registry; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs index 162d24c2e..1959511d6 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/BlobRedirectAuthTests.cs @@ -1,6 +1,6 @@ using System.Net; using System.Net.Http.Headers; -using Amazon.OCI.Builder.Registry; +using AWS.OCI.Builder.Registry; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs index 91a179d4c..d3f19f4f5 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/DigestVerificationTests.cs @@ -1,5 +1,5 @@ using System.Text; -using Amazon.OCI.Builder.Registry; +using AWS.OCI.Builder.Registry; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs index 535213698..d76baeea6 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/FileManifestTests.cs @@ -1,4 +1,4 @@ -using Amazon.OCI.Builder; +using AWS.OCI.Builder; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs index 58c6cccd2..1f4aa1561 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageAssemblyTests.cs @@ -1,7 +1,7 @@ using System.Security.Cryptography; using System.Text.Json.Nodes; -using Amazon.OCI.Builder; -using Amazon.OCI.Builder.Images; +using AWS.OCI.Builder; +using AWS.OCI.Builder.Images; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs index dee70db00..164d215e5 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/ImageReferenceTests.cs @@ -1,4 +1,4 @@ -using Amazon.OCI.Builder; +using AWS.OCI.Builder; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs index d1359d6bb..6d299419d 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/LayerStreamingTests.cs @@ -1,7 +1,7 @@ using System.IO.Compression; using System.Security.Cryptography; -using Amazon.OCI.Builder; -using Amazon.OCI.Builder.Images; +using AWS.OCI.Builder; +using AWS.OCI.Builder.Images; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs index 98925d108..eea5a7bca 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/LayerTests.cs @@ -1,7 +1,7 @@ using System.Formats.Tar; using System.IO.Compression; -using Amazon.OCI.Builder; -using Amazon.OCI.Builder.Images; +using AWS.OCI.Builder; +using AWS.OCI.Builder.Images; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs index ba15584ce..17ebbfe77 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs @@ -1,8 +1,8 @@ using System.Net; using System.Text; -using Amazon.OCI.Builder; -using Amazon.OCI.Builder.Images; -using Amazon.OCI.Builder.Registry; +using AWS.OCI.Builder; +using AWS.OCI.Builder.Images; +using AWS.OCI.Builder.Registry; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs index 85fdf9118..2e0f448ff 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/RealmValidatorTests.cs @@ -1,4 +1,4 @@ -using Amazon.OCI.Builder.Registry; +using AWS.OCI.Builder.Registry; using Xunit; namespace AWS.OCI.Builder.Tests; diff --git a/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs index 8b17b125f..eb6260333 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/UnixPermissionsTests.cs @@ -1,4 +1,4 @@ -using Amazon.OCI.Builder; +using AWS.OCI.Builder; using Xunit; namespace AWS.OCI.Builder.Tests; From 20013b69b04966d51c1a94111617b08c68289375 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 28 Oct 2026 20:18:09 +0000 Subject: [PATCH 12/14] Harden AWS.OCI.Builder against untrusted registries and credential leaks - Bound reads from registries: manifests, configs and token responses are capped at 4 MiB, error and 401 bodies at 4 KiB, and verified blob downloads stop as soon as more bytes arrive than the manifest declared. - Validate digests, tags and repository names (OciNames) before they are used to build request URLs, including digests parsed from base images. - Redact secrets from ToString on BasicRegistryCredentials, TokenResponse and the cached auth entry. - Bind StaticRegistryCredentialSource to a single registry host. The auth handler looks up credentials by host and port, and Bearer token endpoints by their own host, so credentials are never sent to any other host, including across pushes. --- .../src/AWS.OCI.Builder/ImageReference.cs | 28 +- .../src/AWS.OCI.Builder/OciImageBuilder.cs | 36 +- Libraries/src/AWS.OCI.Builder/OciNames.cs | 52 +++ .../Registry/AuthHandshakeHandler.cs | 45 ++- .../AWS.OCI.Builder/Registry/BoundedRead.cs | 70 ++++ .../Registry/IRegistryCredentialSource.cs | 41 ++- .../Registry/RegistryClient.cs | 100 +++++- .../Registry/RegistryExceptions.cs | 22 +- .../AWS.OCI.Builder/Registry/TokenResponse.cs | 4 + .../CredentialBindingTests.cs | 111 ++++++ .../OciImageBuilderTests.cs | 87 +++++ .../SecurityHardeningTests.cs | 338 ++++++++++++++++++ 12 files changed, 892 insertions(+), 42 deletions(-) create mode 100644 Libraries/src/AWS.OCI.Builder/OciNames.cs create mode 100644 Libraries/src/AWS.OCI.Builder/Registry/BoundedRead.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/CredentialBindingTests.cs create mode 100644 Libraries/test/AWS.OCI.Builder.Tests/SecurityHardeningTests.cs diff --git a/Libraries/src/AWS.OCI.Builder/ImageReference.cs b/Libraries/src/AWS.OCI.Builder/ImageReference.cs index f17491caf..ac3ea5f59 100644 --- a/Libraries/src/AWS.OCI.Builder/ImageReference.cs +++ b/Libraries/src/AWS.OCI.Builder/ImageReference.cs @@ -148,14 +148,23 @@ public static ImageReference Parse(string reference) // Validate the repository against the OCI/Docker name grammar. Without this, an input such as // "123456789012.dkr.ecr.us-west-2.amazonaws.com///attacker.example/app" would be accepted and its // "//attacker.example/app" repository would override the authority of the request URIs the RegistryClient - // builds — causing the caller's credentials (a StaticRegistryCredentialSource answers for every host) to - // be sent to an attacker-controlled host. Rejecting malformed names here closes that vector. - if (!RepositoryRegex().IsMatch(repository)) + // builds, sending the push to an attacker-controlled host. Rejecting malformed names here closes that vector. + if (!OciNames.IsValidRepository(repository)) { throw new ArgumentException( - $"The image reference '{reference}' has an invalid repository '{repository}'. A repository is one or " + - "more lowercase path components (letters, digits, and separators '.', '_', '-') separated by single " + - "'/' characters.", + $"The image reference '{OciNames.ForDisplay(reference)}' has an invalid repository '{OciNames.ForDisplay(repository)}'. " + + "A repository is one or more lowercase path components (letters, digits, and separators '.', '_', '-') " + + "separated by single '/' characters.", + nameof(reference)); + } + + // The tag or digest is also spliced into request paths, so it must match its grammar too. + if (isDigest ? !OciNames.IsValidDigest(refValue) : !OciNames.IsValidTag(refValue)) + { + throw new ArgumentException( + isDigest + ? $"The image reference '{OciNames.ForDisplay(reference)}' has an invalid digest '{OciNames.ForDisplay(refValue)}'; expected 'sha256:' followed by 64 lowercase hex characters." + : $"The image reference '{OciNames.ForDisplay(reference)}' has an invalid tag '{OciNames.ForDisplay(refValue)}'.", nameof(reference)); } @@ -170,11 +179,4 @@ public override string ToString() // (also supports the FIPS and China/GovCloud partitions' domains). [GeneratedRegex(@"^\d{12}\.dkr\.ecr(-fips)?\.(?[a-z0-9-]+)\.amazonaws\.com(\.cn)?$", RegexOptions.IgnoreCase)] private static partial Regex EcrHostRegex(); - - // The OCI/Docker repository-name grammar: one or more path components separated by single '/' characters, - // each component lowercase alphanumerics with optional internal '.', '_', '__' or '-' separators. This rejects - // leading/trailing/double slashes, empty components, and uppercase — the shapes that could otherwise smuggle an - // alternate authority into a request URI. - [GeneratedRegex(@"^[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*)*$")] - private static partial Regex RepositoryRegex(); } diff --git a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs index 54b0f5920..aaa414a52 100644 --- a/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/OciImageBuilder.cs @@ -156,6 +156,20 @@ public static async Task FromBaseImageAsync( throw new RegistryException($"The base manifest for '{baseImage}' has no config descriptor."); } + // The base manifest is untrusted input: its digests go into request URLs (including authenticated ECR + // requests during the push) and its sizes bound how much is downloaded. Validate them all up front. + ValidateBaseDescriptor(baseImage, "config", manifest.Config.Digest, manifest.Config.Size); + if (manifest.Config.Size > BoundedRead.MaxMetadataBytes) + { + throw new RegistryException( + $"The base image '{baseImage}' declares a {manifest.Config.Size}-byte config, which exceeds the " + + $"{BoundedRead.MaxMetadataBytes}-byte limit for image configs."); + } + for (int i = 0; i < manifest.Layers.Count; i++) + { + ValidateBaseDescriptor(baseImage, $"layer {i}", manifest.Layers[i].Digest, manifest.Layers[i].Size); + } + byte[] configBytes = await sourceClient .GetVerifiedBlobBytesAsync(baseReference.Repository, manifest.Config.Digest, "base image config", manifest.Config.Size, cancellationToken) .ConfigureAwait(false); @@ -311,7 +325,7 @@ public async Task PushToEcrAsync( // Decode the ECR authorization token into the Basic username/password the registry handshake expects. (string username, string password) = DecodeEcrAuthorizationToken(authorizationToken); - var credentialSource = new StaticRegistryCredentialSource(username, password); + var credentialSource = new StaticRegistryCredentialSource(target.Registry, username, password); IProgress? progress = options?.Progress; @@ -537,6 +551,8 @@ private static async Task ResolvePlatformManifestAsync( $"The base image '{baseReference}' has no manifest for platform '{os}/{architecture}'. " + $"Available: {DescribePlatforms(index)}."); + ValidateBaseDescriptor(baseReference.ToString(), $"{os}/{architecture} manifest", selected.Digest, selected.Size); + ManifestResponse platformManifest = await client.GetManifestAsync(baseReference.Repository, selected.Digest, cancellationToken).ConfigureAwait(false); @@ -577,6 +593,24 @@ private static (string Os, string Architecture) ParsePlatform(string platform) private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this); + /// + /// Rejects a descriptor from the (untrusted) base image whose digest is not a well-formed sha256 digest or + /// whose size is negative. + /// + internal static void ValidateBaseDescriptor(string baseImage, string what, string? digest, long size) + { + if (!OciNames.IsValidDigest(digest)) + { + throw new RegistryException( + $"The base image '{baseImage}' has an invalid {what} digest '{OciNames.ForDisplay(digest)}'; expected " + + "'sha256:' followed by 64 lowercase hex characters."); + } + if (size < 0) + { + throw new RegistryException($"The base image '{baseImage}' declares a negative size ({size}) for its {what}."); + } + } + /// /// Decodes an ECR authorization token (base64 of username:password, where the username is AWS) /// into its username/password parts. diff --git a/Libraries/src/AWS.OCI.Builder/OciNames.cs b/Libraries/src/AWS.OCI.Builder/OciNames.cs new file mode 100644 index 000000000..189f78a58 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/OciNames.cs @@ -0,0 +1,52 @@ +using System.Text.RegularExpressions; + +namespace AWS.OCI.Builder; + +/// +/// Validation for the names that end up in registry request URLs: repositories, tags and digests. +/// +/// +/// These values are spliced into URL paths such as /v2/<repository>/blobs/<digest>, and some come +/// from untrusted documents (a base image's manifest). A value like sha256:x/../../other/manifests/latest +/// would otherwise redirect an authenticated request to a different path, and a repository starting with +/// // would replace the request's host. All patterns are anchored with \z so a trailing newline +/// cannot slip through. +/// +internal static partial class OciNames +{ + /// Returns whether is sha256: followed by 64 lowercase hex characters. + public static bool IsValidDigest(string? digest) => digest is not null && DigestRegex().IsMatch(digest); + + /// Returns whether matches the OCI tag grammar. + public static bool IsValidTag(string? tag) => tag is not null && TagRegex().IsMatch(tag); + + /// Returns whether matches the OCI repository-name grammar. + public static bool IsValidRepository(string? repository) => repository is not null && RepositoryRegex().IsMatch(repository); + + /// Returns whether is a valid tag or a valid digest. + public static bool IsValidReference(string? reference) => IsValidTag(reference) || IsValidDigest(reference); + + /// Formats an untrusted value for an error message: shortened, with control characters escaped. + public static string ForDisplay(string? value) + { + if (value is null) + { + return "(null)"; + } + string shortened = value.Length <= 100 ? value : value[..100] + "…"; + return Regex.Replace(shortened, @"\p{C}", m => $"\\u{(int)m.Value[0]:x4}"); + } + + [GeneratedRegex(@"^sha256:[a-f0-9]{64}\z")] + private static partial Regex DigestRegex(); + + // OCI distribution spec tag grammar. + [GeneratedRegex(@"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}\z")] + private static partial Regex TagRegex(); + + // One or more path components separated by single '/' characters, each component lowercase alphanumerics with + // optional internal '.', '_', '__' or '-' separators. Rejects leading/trailing/double slashes, empty components + // and uppercase: the shapes that could smuggle an alternate authority or path into a request URI. + [GeneratedRegex(@"^[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*)*\z")] + private static partial Regex RepositoryRegex(); +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index 09f4e75bd..a7a2af533 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -53,7 +53,11 @@ public AuthHandshakeHandler(IRegistryCredentialSource credentialSource, HttpMess : base(innerHandler) => _credentialSource = credentialSource ?? throw new ArgumentNullException(nameof(credentialSource)); - private readonly record struct CachedAuth(AuthenticationHeaderValue Header, DateTimeOffset Expiry); + private readonly record struct CachedAuth(AuthenticationHeaderValue Header, DateTimeOffset Expiry) + { + // The header carries the credential (a bearer token or encoded username:password); never print it. + public override string ToString() => $"{nameof(CachedAuth)} {{ Scheme = {Header.Scheme}, Expiry = {Expiry:O} }}"; + } /// protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) @@ -79,7 +83,9 @@ protected override async Task SendAsync(HttpRequestMessage return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); } - string registryHost = request.RequestUri.Host; + // Host plus port (Authority omits the default 443), so credentials and cached headers never carry over to the + // same host name on a different port. + string registryHost = request.RequestUri.Authority; // Apply a cached, still-valid authorization header if we have one. if (_cache.TryGetValue(registryHost, out CachedAuth cached) && cached.Expiry > DateTimeOffset.UtcNow) @@ -99,8 +105,9 @@ protected override async Task SendAsync(HttpRequestMessage return response; // No usable challenge; let the caller handle the 401. } - // Drain the 401 body so the connection is freed for the token request. - _ = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + // Drain (a bounded prefix of) the 401 body so the connection can be reused for the token request. The read + // is capped because the body comes from the registry; anything beyond the cap is dropped with the response. + _ = await BoundedRead.ReadAsync(response.Content, BoundedRead.MaxErrorBodyBytes, cancellationToken).ConfigureAwait(false); response.Dispose(); (AuthenticationHeaderValue header, DateTimeOffset expiry) = @@ -118,11 +125,11 @@ protected override async Task SendAsync(HttpRequestMessage private async Task<(AuthenticationHeaderValue, DateTimeOffset)> AuthenticateAsync( string registryHost, string scheme, AuthInfo? authInfo, CancellationToken cancellationToken) { - BasicRegistryCredentials? creds = - await _credentialSource.GetCredentialsAsync(registryHost, cancellationToken).ConfigureAwait(false); - if (scheme.Equals(BasicScheme, StringComparison.OrdinalIgnoreCase)) { + // Basic credentials go to the registry host itself, so ask for that host's credentials. + BasicRegistryCredentials? creds = + await _credentialSource.GetCredentialsAsync(registryHost, cancellationToken).ConfigureAwait(false); if (creds is not { } basic) { throw new RegistryAuthenticationException(registryHost, "the registry requires Basic auth but no credentials were available."); @@ -136,18 +143,23 @@ protected override async Task SendAsync(HttpRequestMessage { throw new InvalidAuthResponseException(registryHost, "the Bearer challenge did not include a realm."); } - return await GetBearerTokenAsync(registryHost, info, creds, cancellationToken).ConfigureAwait(false); + return await GetBearerTokenAsync(registryHost, info, cancellationToken).ConfigureAwait(false); } throw new RegistryAuthenticationException(registryHost, $"unsupported authentication scheme '{scheme}'."); } private async Task<(AuthenticationHeaderValue, DateTimeOffset)> GetBearerTokenAsync( - string registryHost, AuthInfo info, BasicRegistryCredentials? creds, CancellationToken cancellationToken) + string registryHost, AuthInfo info, CancellationToken cancellationToken) { // SSRF guard: never send credentials to an unvalidated realm. Uri realm = RealmValidator.Validate(info.Realm, registryHost); + // The credentials are sent to the token endpoint, not the registry, so ask for the realm host's credentials. + // A challenge pointing the realm at some other host then gets none, rather than the registry's. + BasicRegistryCredentials? creds = + await _credentialSource.GetCredentialsAsync(realm.Authority, cancellationToken).ConfigureAwait(false); + var builder = new UriBuilder(realm); System.Collections.Specialized.NameValueCollection query = HttpUtility.ParseQueryString(realm.Query); if (info.Service is { } svc) @@ -177,9 +189,20 @@ protected override async Task SendAsync(HttpRequestMessage } TokenResponse? token; - await using (Stream s = await tokenResponse.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) + (byte[] tokenBytes, bool exceeded) = + await BoundedRead.ReadAsync(tokenResponse.Content, BoundedRead.MaxMetadataBytes, cancellationToken).ConfigureAwait(false); + if (exceeded) + { + throw new InvalidAuthResponseException( + registryHost, $"the token endpoint response is larger than the {BoundedRead.MaxMetadataBytes}-byte limit."); + } + try + { + token = JsonSerializer.Deserialize(tokenBytes, Images.OciJsonContext.TokenResponseInfo); + } + catch (JsonException) { - token = await JsonSerializer.DeserializeAsync(s, Images.OciJsonContext.TokenResponseInfo, cancellationToken: cancellationToken).ConfigureAwait(false); + token = null; } if (token is null || token.ResolvedToken is null) diff --git a/Libraries/src/AWS.OCI.Builder/Registry/BoundedRead.cs b/Libraries/src/AWS.OCI.Builder/Registry/BoundedRead.cs new file mode 100644 index 000000000..79e4da5e1 --- /dev/null +++ b/Libraries/src/AWS.OCI.Builder/Registry/BoundedRead.cs @@ -0,0 +1,70 @@ +using System.Buffers; + +namespace AWS.OCI.Builder.Registry; + +/// +/// Reads HTTP bodies with an upper bound, so a malicious or broken registry cannot exhaust memory by sending an +/// arbitrarily large manifest, token response or error body. +/// +internal static class BoundedRead +{ + /// + /// The limit for metadata documents: manifests, image indexes, image configs and token responses. Matches the + /// 4 MiB manifest limit used by containerd and the distribution registry; real documents are far smaller. + /// + public const int MaxMetadataBytes = 4 * 1024 * 1024; + + /// How much of an error response body is read for the exception message. + public const int MaxErrorBodyBytes = 4 * 1024; + + /// + /// Reads at most bytes of . Exceeded is + /// when the body is larger than the limit, either because its Content-Length + /// says so (nothing is read) or because more bytes arrived than allowed (reading stops right there). + /// + public static async Task<(byte[] Data, bool Exceeded)> ReadAsync(HttpContent content, long limit, CancellationToken cancellationToken) + { + if (content.Headers.ContentLength is long declared && declared > limit) + { + return (Array.Empty(), true); + } + + await using Stream stream = await content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + return await ReadAsync(stream, limit, cancellationToken).ConfigureAwait(false); + } + + /// + /// Reads at most bytes of , stopping as soon as more than + /// bytes are seen. Data holds the bytes up to the limit. + /// + public static async Task<(byte[] Data, bool Exceeded)> ReadAsync(Stream source, long limit, CancellationToken cancellationToken) + { + ArgumentOutOfRangeException.ThrowIfNegative(limit); + + using var buffer = new MemoryStream(); + byte[] chunk = ArrayPool.Shared.Rent(81920); + try + { + while (true) + { + int read = await source.ReadAsync(chunk.AsMemory(0, chunk.Length), cancellationToken).ConfigureAwait(false); + if (read == 0) + { + return (buffer.ToArray(), false); + } + + long room = limit - buffer.Length; + if (read > room) + { + buffer.Write(chunk, 0, (int)room); + return (buffer.ToArray(), true); + } + buffer.Write(chunk, 0, read); + } + } + finally + { + ArrayPool.Shared.Return(chunk); + } + } +} diff --git a/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs index bdf2f5c05..13843717f 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs @@ -13,17 +13,25 @@ public interface IRegistryCredentialSource { /// /// Returns the credentials for , or to attempt - /// anonymous access. + /// anonymous access. Return for any host the credentials are not meant for. /// - /// The registry host, for example 123456789012.dkr.ecr.us-west-2.amazonaws.com. + /// + /// The host the credentials would be sent to: the registry itself, or a Bearer token endpoint. It includes the + /// port only when it is not 443, for example 123456789012.dkr.ecr.us-west-2.amazonaws.com. + /// /// A cancellation token. Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default); } /// A plain username/password pair used for registry authentication. +/// never includes the password. /// The registry username. /// The registry password or token. -public readonly record struct BasicRegistryCredentials(string Username, string Password); +public readonly record struct BasicRegistryCredentials(string Username, string Password) +{ + /// Returns the username with the password redacted. + public override string ToString() => $"{nameof(BasicRegistryCredentials)} {{ Username = {Username}, Password = *** }}"; +} /// /// A credential source that always returns , causing the handshake to attempt @@ -38,23 +46,38 @@ public sealed class AnonymousRegistryCredentialSource : IRegistryCredentialSourc } /// -/// A credential source backed by a fixed username/password pair known up front (for example one decoded from an -/// Amazon ECR authorization token). It performs no network or credential resolution and has no dependency on the -/// AWS SDK; the caller is responsible for obtaining the token and keeping it fresh for the push's duration. +/// A credential source backed by a fixed username/password pair for a single registry host (for example one decoded +/// from an Amazon ECR authorization token). It performs no network or credential resolution and has no dependency on +/// the AWS SDK; the caller is responsible for obtaining the token and keeping it fresh for the push's duration. /// +/// +/// The credentials are returned only for the host they were created for, and for any other +/// host. A request that reaches an unexpected host (a forged upload location, a token endpoint on another host) +/// therefore never receives them. +/// public sealed class StaticRegistryCredentialSource : IRegistryCredentialSource { + private readonly string _registryHost; private readonly BasicRegistryCredentials _credentials; - /// Creates a source that returns the given username/password for every host. - public StaticRegistryCredentialSource(string username, string password) + /// Creates a source that returns the given username/password for only. + /// + /// The host the credentials belong to, with the port only when it is not 443, for example + /// 123456789012.dkr.ecr.us-west-2.amazonaws.com. + /// + /// The registry username. + /// The registry password or token. + public StaticRegistryCredentialSource(string registryHost, string username, string password) { + ArgumentException.ThrowIfNullOrWhiteSpace(registryHost); ArgumentNullException.ThrowIfNull(username); ArgumentNullException.ThrowIfNull(password); + _registryHost = registryHost; _credentials = new BasicRegistryCredentials(username, password); } /// public Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default) - => Task.FromResult(_credentials); + => Task.FromResult( + string.Equals(registryHost, _registryHost, StringComparison.OrdinalIgnoreCase) ? _credentials : null); } diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs index 8719e7585..391e247c9 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -147,15 +147,24 @@ private static async ValueTask GuardedConnectAsync(SocketsHttpConnection /// public async Task GetManifestAsync(string repository, string reference, CancellationToken cancellationToken = default) { + ValidateRepository(repository); + ValidateReference(reference); + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(Base, $"{repository}/manifests/{reference}")); AddManifestAcceptHeaders(request); - using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); + using HttpResponseMessage response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); await EnsureSuccessAsync(response, $"GET manifest {repository}:{reference}", cancellationToken).ConfigureAwait(false); // Read the exact bytes on the wire. Digests must be computed over these, not over a string that has been // decoded and re-encoded (which could differ by BOM/charset), so the raw bytes are exposed for verification. - byte[] raw = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + // The read is bounded so a hostile registry cannot exhaust memory with an enormous "manifest". + (byte[] raw, bool exceeded) = await BoundedRead.ReadAsync(response.Content, BoundedRead.MaxMetadataBytes, cancellationToken).ConfigureAwait(false); + if (exceeded) + { + throw new RegistryException( + $"The manifest {repository}:{reference} is larger than the {BoundedRead.MaxMetadataBytes}-byte limit for manifests."); + } string json = System.Text.Encoding.UTF8.GetString(raw); string mediaType = response.Content.Headers.ContentType?.MediaType ?? throw new RegistryRequestException((int)response.StatusCode, "manifest response had no Content-Type."); @@ -169,6 +178,9 @@ public async Task GetManifestAsync(string repository, string r /// Checks whether a manifest exists for the given reference without downloading it. public async Task ManifestExistsAsync(string repository, string reference, CancellationToken cancellationToken = default) { + ValidateRepository(repository); + ValidateReference(reference); + using var request = new HttpRequestMessage(HttpMethod.Head, new Uri(Base, $"{repository}/manifests/{reference}")); AddManifestAcceptHeaders(request); using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); @@ -178,6 +190,9 @@ public async Task ManifestExistsAsync(string repository, string reference, /// Uploads a manifest under the given tag or digest reference. public async Task PutManifestAsync(string repository, string reference, string manifestJson, string mediaType, CancellationToken cancellationToken = default) { + ValidateRepository(repository); + ValidateReference(reference); + using var request = new HttpRequestMessage(HttpMethod.Put, new Uri(Base, $"{repository}/manifests/{reference}")) { Content = new StringContent(manifestJson), @@ -193,6 +208,9 @@ public async Task PutManifestAsync(string repository, string reference, string m /// Checks whether a blob with the given digest already exists in the repository. public async Task BlobExistsAsync(string repository, string digest, CancellationToken cancellationToken = default) { + ValidateRepository(repository); + ValidateDigest(digest); + using var request = new HttpRequestMessage(HttpMethod.Head, new Uri(Base, $"{repository}/blobs/{digest}")); using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); return response.StatusCode == HttpStatusCode.OK; @@ -207,6 +225,9 @@ public async Task BlobExistsAsync(string repository, string digest, Cancel /// public async Task GetBlobAsync(string repository, string digest, CancellationToken cancellationToken = default) { + ValidateRepository(repository); + ValidateDigest(digest); + Uri uri = new(Base, $"{repository}/blobs/{digest}"); for (int hop = 0; ; hop++) @@ -342,6 +363,10 @@ public override async ValueTask DisposeAsync() /// public async Task TryMountBlobAsync(string repository, string digest, string fromRepository, CancellationToken cancellationToken = default) { + ValidateRepository(repository); + ValidateDigest(digest); + ValidateRepository(fromRepository, nameof(fromRepository)); + var uri = new Uri(Base, $"{repository}/blobs/uploads/?mount={Uri.EscapeDataString(digest)}&from={Uri.EscapeDataString(fromRepository)}"); using var request = new HttpRequestMessage(HttpMethod.Post, uri); using HttpResponseMessage response = await _http.SendAsync(request, cancellationToken).ConfigureAwait(false); @@ -400,6 +425,9 @@ public async Task UploadBlobAsync( Action? onBytesSent = null, long? contentLength = null) { + ValidateRepository(repository); + ValidateDigest(digest); + if (await BlobExistsAsync(repository, digest, cancellationToken).ConfigureAwait(false)) { return; @@ -501,7 +529,10 @@ private static async Task EnsureSuccessAsync(HttpResponseMessage response, strin string body = string.Empty; try { - body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + // Read only the start of the body: it is just context for the exception message, and an unbounded + // read would let a hostile registry exhaust memory with a huge error response. + (byte[] prefix, _) = await BoundedRead.ReadAsync(response.Content, BoundedRead.MaxErrorBodyBytes, cancellationToken).ConfigureAwait(false); + body = System.Text.Encoding.UTF8.GetString(prefix); } catch { @@ -513,6 +544,33 @@ private static async Task EnsureSuccessAsync(HttpResponseMessage response, strin $"{operation} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {Truncate(body, 512)}"); } + private static void ValidateRepository(string repository, string paramName = "repository") + { + if (!OciNames.IsValidRepository(repository)) + { + throw new ArgumentException($"'{OciNames.ForDisplay(repository)}' is not a valid repository name.", paramName); + } + } + + private static void ValidateDigest(string digest) + { + if (!OciNames.IsValidDigest(digest)) + { + throw new ArgumentException( + $"'{OciNames.ForDisplay(digest)}' is not a valid digest; expected 'sha256:' followed by 64 lowercase hex characters.", + nameof(digest)); + } + } + + private static void ValidateReference(string reference) + { + if (!OciNames.IsValidReference(reference)) + { + throw new ArgumentException( + $"'{OciNames.ForDisplay(reference)}' is not a valid tag or digest.", nameof(reference)); + } + } + private static string Truncate(string value, int max) => value.Length <= max ? value : value[..max] + "…"; @@ -591,12 +649,30 @@ public static void VerifyDigest(string what, string expectedDigest, ReadOnlySpan /// The downloaded length differs from . public async Task GetVerifiedBlobBytesAsync(string repository, string digest, string what, long? expectedSize = null, CancellationToken cancellationToken = default) { + if (expectedSize is < 0) + { + throw new ArgumentOutOfRangeException(nameof(expectedSize), expectedSize, "The expected size must not be negative."); + } + byte[] bytes; await using (Stream stream = await GetBlobAsync(repository, digest, cancellationToken).ConfigureAwait(false)) { - using var buffer = new MemoryStream(); - await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); - bytes = buffer.ToArray(); + if (expectedSize is { } limit) + { + // Stop as soon as the blob runs past its declared size, rather than buffering whatever the + // registry sends. + (bytes, bool exceeded) = await BoundedRead.ReadAsync(stream, limit, cancellationToken).ConfigureAwait(false); + if (exceeded) + { + throw BlobSizeMismatchException.LargerThanDeclared(what, digest, limit); + } + } + else + { + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + bytes = buffer.ToArray(); + } } VerifyDigest(what, digest, bytes); @@ -637,6 +713,11 @@ public async Task GetVerifiedBlobBytesAsync(string repository, string di /// The downloaded length differs from . public async Task GetVerifiedBlobToFileAsync(string repository, string digest, string what, long? expectedSize = null, CancellationToken cancellationToken = default) { + if (expectedSize is < 0) + { + throw new ArgumentOutOfRangeException(nameof(expectedSize), expectedSize, "The expected size must not be negative."); + } + // A unique temp path under the system temp dir (honors TMPDIR). Created now so we can always clean it up. string tempPath = Path.Combine(Path.GetTempPath(), $"aws-oci-layer-{Guid.NewGuid():N}.tmp"); @@ -655,6 +736,13 @@ public async Task GetVerifiedBlobToFileAsync(string repository int read; while ((read = await source.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0) { + // Stop as soon as the blob runs past its declared size, so a lying registry cannot fill + // the disk. The catch below deletes the partial file. + if (expectedSize is { } limit && downloadedLength + read > limit) + { + throw BlobSizeMismatchException.LargerThanDeclared(what, digest, limit); + } + hasher.AppendData(rented, 0, read); downloadedLength += read; await file.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false); diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs index 5586c5630..cddea40dd 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryExceptions.cs @@ -85,15 +85,33 @@ public sealed class BlobSizeMismatchException : RegistryException /// The size, in bytes, declared by the manifest descriptor. public long DeclaredSize { get; } - /// The actual size, in bytes, of the downloaded blob. + /// + /// The actual size, in bytes, of the downloaded blob. When the download was stopped because the blob ran past + /// its declared size, this is the number of bytes accepted before stopping (the declared size), and the real + /// size is larger. + /// public long ActualSize { get; } /// Creates the exception. public BlobSizeMismatchException(string what, string digest, long declaredSize, long actualSize) - : base($"Integrity check failed for {what} {digest}: the manifest declares a size of {declaredSize} bytes but the blob is {actualSize} bytes.") + : this( + $"Integrity check failed for {what} {digest}: the manifest declares a size of {declaredSize} bytes but the blob is {actualSize} bytes.", + declaredSize, actualSize, digest) + { + } + + private BlobSizeMismatchException(string message, long declaredSize, long actualSize, string digest) + : base(message) { Digest = digest; DeclaredSize = declaredSize; ActualSize = actualSize; } + + /// Creates the exception for a download stopped because the blob is larger than declared. + internal static BlobSizeMismatchException LargerThanDeclared(string what, string digest, long declaredSize) + => new( + $"Integrity check failed for {what} {digest}: the manifest declares a size of {declaredSize} bytes but the " + + "blob is larger, so the download was stopped.", + declaredSize, declaredSize, digest); } diff --git a/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs b/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs index 6bef6e3b9..1ac628448 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/TokenResponse.cs @@ -24,6 +24,10 @@ internal sealed record TokenResponse /// The usable token (registries populate either token or access_token). public string? ResolvedToken => Token ?? AccessToken; + /// Describes the response with the token values redacted. + public override string ToString() + => $"{nameof(TokenResponse)} {{ Token = {(ResolvedToken is null ? "(none)" : "***")}, ExpiresIn = {ExpiresIn}, IssuedAt = {IssuedAt:O} }}"; + /// /// The computed expiry. Per the token spec, default to 60 seconds of validity when the endpoint /// omits expires_in; subtract a small safety margin so a cached token is refreshed before it diff --git a/Libraries/test/AWS.OCI.Builder.Tests/CredentialBindingTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/CredentialBindingTests.cs new file mode 100644 index 000000000..f0c02b6f3 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/CredentialBindingTests.cs @@ -0,0 +1,111 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using AWS.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +/// +/// Credentials from a must only ever be sent to the host they were +/// created for, whatever host a request or a Bearer challenge points at. +/// +public class CredentialBindingTests +{ + private const string BoundHost = "111111111111.dkr.ecr.us-west-2.amazonaws.com"; + private static readonly BasicRegistryCredentials Creds = new("AWS", "bound-secret"); + + private static readonly string BoundBasicHeader = + "Basic " + Convert.ToBase64String(Encoding.ASCII.GetBytes("AWS:bound-secret")); + + /// Records the authority and Authorization header of every request it receives. + private sealed class RecordingHandler : HttpMessageHandler + { + private readonly Func _responder; + public List<(string Authority, string? Authorization)> Seen { get; } = new(); + + public RecordingHandler(Func responder) => _responder = responder; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Seen.Add((request.RequestUri!.Authority, request.Headers.Authorization?.ToString())); + return Task.FromResult(_responder(request)); + } + } + + private static HttpResponseMessage Challenge(string scheme, string parameter) + { + var response = new HttpResponseMessage(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue(scheme, parameter)); + return response; + } + + private static HttpClient CreateClient(RecordingHandler stub) + => new(new AuthHandshakeHandler(new StaticRegistryCredentialSource(BoundHost, Creds.Username, Creds.Password), stub)); + + [Fact] + public async Task Static_source_returns_credentials_only_for_its_own_host() + { + var source = new StaticRegistryCredentialSource(BoundHost, Creds.Username, Creds.Password); + + Assert.Equal(Creds, await source.GetCredentialsAsync(BoundHost)); + Assert.Equal(Creds, await source.GetCredentialsAsync(BoundHost.ToUpperInvariant())); + + Assert.Null(await source.GetCredentialsAsync("attacker.example")); + Assert.Null(await source.GetCredentialsAsync("222222222222.dkr.ecr.us-west-2.amazonaws.com")); + Assert.Null(await source.GetCredentialsAsync(BoundHost + ":8443")); + Assert.Null(await source.GetCredentialsAsync(BoundHost + ".attacker.example")); + } + + [Theory] + [InlineData("https://attacker.example/v2/")] + [InlineData("https://" + BoundHost + ":8443/v2/")] + public async Task Basic_challenge_from_any_other_host_gets_no_credentials(string url) + { + var stub = new RecordingHandler(_ => Challenge("Basic", "realm=\"registry\"")); + using HttpClient http = CreateClient(stub); + + await Assert.ThrowsAsync(() => http.GetAsync(url)); + + Assert.NotEmpty(stub.Seen); + Assert.All(stub.Seen, seen => Assert.Null(seen.Authorization)); + } + + [Fact] + public async Task Bearer_realm_on_another_host_gets_no_credentials() + { + // The bound host itself answers with a Bearer challenge whose token endpoint is some other host. + var stub = new RecordingHandler(request => request.RequestUri!.Host == BoundHost + ? Challenge("Bearer", "realm=\"https://attacker.example/token\",service=\"registry\"") + : new HttpResponseMessage(HttpStatusCode.Unauthorized)); + using HttpClient http = CreateClient(stub); + + await Assert.ThrowsAsync(() => http.GetAsync($"https://{BoundHost}/v2/")); + + (string Authority, string? Authorization) tokenRequest = Assert.Single(stub.Seen, s => s.Authority == "attacker.example"); + Assert.Null(tokenRequest.Authorization); + Assert.DoesNotContain(stub.Seen, s => s.Authorization == BoundBasicHeader); + } + + [Fact] + public async Task Bearer_realm_on_the_bound_host_still_gets_credentials() + { + var stub = new RecordingHandler(request => + { + if (request.RequestUri!.AbsolutePath == "/token") + { + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{\"token\":\"t0k3n\"}") }; + } + return request.Headers.Authorization is null + ? Challenge("Bearer", $"realm=\"https://{BoundHost}/token\"") + : new HttpResponseMessage(HttpStatusCode.OK); + }); + using HttpClient http = CreateClient(stub); + + using HttpResponseMessage response = await http.GetAsync($"https://{BoundHost}/v2/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(stub.Seen, s => s.Authorization == BoundBasicHeader); + Assert.Contains(stub.Seen, s => s.Authorization == "Bearer t0k3n"); + } +} diff --git a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs index 17ebbfe77..f735b683e 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs +++ b/Libraries/test/AWS.OCI.Builder.Tests/OciImageBuilderTests.cs @@ -588,6 +588,93 @@ private static async Task BuildTwoLayerImageAsync(FakeRegistry return builder; } + /// + /// Sits in front of , demands Basic auth on every ECR host, and records each + /// Authorization value an ECR host receives. + /// + private sealed class EcrAuthGate : HttpMessageHandler + { + private readonly HttpMessageInvoker _registry; + private readonly object _lock = new(); + private readonly List<(string Host, string? Authorization)> _seen = new(); + + public EcrAuthGate(FakeRegistry registry) => _registry = new HttpMessageInvoker(registry, disposeHandler: false); + + public IReadOnlyList<(string Host, string? Authorization)> Seen + { + get { lock (_lock) { return _seen.ToList(); } } + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string host = request.RequestUri!.Host; + if (host.Contains(".dkr.ecr.", StringComparison.Ordinal)) + { + lock (_lock) + { + _seen.Add((host, request.Headers.Authorization?.ToString())); + } + if (request.Headers.Authorization is null) + { + var challenge = new HttpResponseMessage(HttpStatusCode.Unauthorized); + challenge.Headers.WwwAuthenticate.Add( + new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", "realm=\"ecr\"")); + return Task.FromResult(challenge); + } + } + return _registry.SendAsync(request, cancellationToken); + } + } + + private static string EcrToken(string password) => Convert.ToBase64String(Encoding.UTF8.GetBytes("AWS:" + password)); + + private static string BasicHeader(string password) => "Basic " + Convert.ToBase64String(Encoding.ASCII.GetBytes("AWS:" + password)); + + [Fact] + public async Task A_later_push_never_receives_credentials_from_an_earlier_push() + { + // Push A goes to a legitimate target; push B, from the SAME builder, goes to a different host standing in for + // a malicious endpoint. Host B must only ever see push B's credentials, never push A's. + const string hostA = "111111111111.dkr.ecr.us-west-2.amazonaws.com"; + const string hostB = "222222222222.dkr.ecr.us-west-2.amazonaws.com"; + var registry = new FakeRegistry(); + var gate = new EcrAuthGate(registry); + + string tempDir = Path.Combine(Path.GetTempPath(), "oci-e2e-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + using OciImageBuilder builder = await BuildTwoLayerImageAsync(registry, "dotnet/aspnet", tempDir); + + // Use the real auth handler with whatever credential source each push hands the factory, over one shared + // transport, so any credential state that survived from push A would reach host B. + builder.UseTargetClientFactoryForTest((host, credentialSource) => + new RegistryClient(host, new HttpClient(new AuthHandshakeHandler(credentialSource, gate)))); + + await builder.PushToEcrAsync($"{hostA}/app:v1", EcrToken("password-A")); + await builder.PushToEcrAsync($"{hostB}/app:v1", EcrToken("password-B")); + + IReadOnlyList<(string Host, string? Authorization)> seen = gate.Seen; + var seenByA = seen.Where(s => s.Host == hostA).Select(s => s.Authorization).ToList(); + var seenByB = seen.Where(s => s.Host == hostB).Select(s => s.Authorization).ToList(); + + // Each push authenticated, with its own credentials. + Assert.Contains(BasicHeader("password-A"), seenByA); + Assert.Contains(BasicHeader("password-B"), seenByB); + + // Neither host ever saw the other push's credentials. + Assert.DoesNotContain(BasicHeader("password-A"), seenByB); + Assert.DoesNotContain(BasicHeader("password-B"), seenByA); + + // Push B's manifest really went to host B. + Assert.Equal(hostB, registry.PutManifest!.Value.Host); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + [Fact] public async Task PushToEcrAsync_reports_progress_for_every_layer_with_stable_index_total_and_kind() { diff --git a/Libraries/test/AWS.OCI.Builder.Tests/SecurityHardeningTests.cs b/Libraries/test/AWS.OCI.Builder.Tests/SecurityHardeningTests.cs new file mode 100644 index 000000000..cd0635ca8 --- /dev/null +++ b/Libraries/test/AWS.OCI.Builder.Tests/SecurityHardeningTests.cs @@ -0,0 +1,338 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Reflection; +using System.Text.Json; +using AWS.OCI.Builder.Registry; +using Xunit; + +namespace AWS.OCI.Builder.Tests; + +/// +/// Tests for the security hardening: secrets never appear in ToString, names and digests are validated before +/// they reach request URLs, and response bodies from a registry are read with an upper bound. +/// +public class SecurityHardeningTests +{ + private const string ValidDigest = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + // ---- Secrets never appear in ToString ----------------------------------------------------------------- + + [Fact] + public void BasicRegistryCredentials_ToString_redacts_password() + { + var creds = new BasicRegistryCredentials("AWS", "s3cr3t-token"); + + string text = creds.ToString(); + + Assert.DoesNotContain("s3cr3t-token", text); + Assert.Contains("AWS", text); + Assert.Equal(text, $"{creds}"); // interpolation goes through the same override + } + + [Theory] + [InlineData("{\"token\":\"tok-SECRET\",\"expires_in\":300}")] + [InlineData("{\"access_token\":\"tok-SECRET\",\"expires_in\":300}")] + [InlineData("{\"token\":\"tok-SECRET\",\"access_token\":\"tok-SECRET\"}")] + public void TokenResponse_ToString_redacts_tokens(string json) + { + TokenResponse token = JsonSerializer.Deserialize(json, Images.OciJsonContext.TokenResponseInfo)!; + + Assert.DoesNotContain("tok-SECRET", token.ToString()); + } + + [Fact] + public void CachedAuth_ToString_redacts_the_authorization_header() + { + // CachedAuth is a private nested type of the handler, so it is reached by reflection. + Type cachedAuth = typeof(AuthHandshakeHandler).GetNestedType("CachedAuth", BindingFlags.NonPublic)!; + object instance = Activator.CreateInstance( + cachedAuth, new AuthenticationHeaderValue("Bearer", "tok-SECRET"), DateTimeOffset.UnixEpoch)!; + + string text = instance.ToString()!; + + Assert.DoesNotContain("tok-SECRET", text); + Assert.Contains("Bearer", text); + } + + // ---- Names and digests are validated before they reach URLs ------------------------------------------ + + [Theory] + [InlineData(ValidDigest, true)] + [InlineData("sha256:E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", false)] // uppercase + [InlineData("sha256:e3b0", false)] // too short + [InlineData("sha512:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", false)] + [InlineData("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", false)] // no prefix + [InlineData("sha256:abc/../../../victim/manifests/latest", false)] // path traversal + [InlineData(ValidDigest + "\n", false)] // trailing newline + [InlineData(null, false)] + public void IsValidDigest(string? digest, bool expected) + => Assert.Equal(expected, OciNames.IsValidDigest(digest)); + + [Theory] + [InlineData("myapp", true)] + [InlineData("dotnet/aspnet", true)] + [InlineData("//attacker.example/app", false)] + [InlineData("myapp/../other", false)] + [InlineData("myapp\n", false)] + public void IsValidRepository(string repository, bool expected) + => Assert.Equal(expected, OciNames.IsValidRepository(repository)); + + [Theory] + [InlineData("latest", true)] + [InlineData("8.0-bookworm-slim", true)] + [InlineData("..", false)] + [InlineData("a?b", false)] + [InlineData("a#b", false)] + [InlineData("latest\n", false)] + public void IsValidTag(string tag, bool expected) + => Assert.Equal(expected, OciNames.IsValidTag(tag)); + + [Theory] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp@sha256:abc/../../victim")] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp@sha256:ABC")] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:..")] + [InlineData("123456789012.dkr.ecr.us-west-2.amazonaws.com/myapp:a?b")] + public void ImageReference_Parse_rejects_invalid_digest_or_tag(string reference) + => Assert.Throws(() => ImageReference.Parse(reference)); + + [Fact] + public void ImageReference_Parse_accepts_valid_digest() + => Assert.Equal(ValidDigest, ImageReference.Parse($"mcr.microsoft.com/dotnet/aspnet@{ValidDigest}").Reference); + + [Fact] + public async Task RegistryClient_rejects_traversal_digest_without_sending_a_request() + { + var handler = new CountingHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)); + var client = new RegistryClient("123456789012.dkr.ecr.us-west-2.amazonaws.com", new HttpClient(handler)); + + await Assert.ThrowsAsync( + () => client.BlobExistsAsync("myapp", "sha256:abc/../../../victim/manifests/latest")); + await Assert.ThrowsAsync( + () => client.GetBlobAsync("myapp", "sha256:abc/../../../victim/manifests/latest")); + await Assert.ThrowsAsync( + () => client.TryMountBlobAsync("myapp", "sha256:bad", "base")); + await Assert.ThrowsAsync( + () => client.GetManifestAsync("myapp", "../../victim")); + + Assert.Equal(0, handler.RequestCount); + } + + [Fact] + public async Task RegistryClient_rejects_repository_that_would_replace_the_host() + { + var handler = new CountingHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)); + var client = new RegistryClient("123456789012.dkr.ecr.us-west-2.amazonaws.com", new HttpClient(handler)); + + await Assert.ThrowsAsync(() => client.GetBlobAsync("//attacker.example/app", ValidDigest)); + await Assert.ThrowsAsync(() => client.TryMountBlobAsync("myapp", ValidDigest, "//attacker.example/x")); + + Assert.Equal(0, handler.RequestCount); + } + + [Theory] + [InlineData("sha256:abc/../../victim", 10)] + [InlineData("sha256:ABC", 10)] + [InlineData(ValidDigest, -1)] + public void ValidateBaseDescriptor_rejects_bad_digest_or_negative_size(string digest, long size) + => Assert.Throws( + () => OciImageBuilder.ValidateBaseDescriptor("example.com/base:1", "layer 0", digest, size)); + + [Fact] + public void ValidateBaseDescriptor_accepts_valid_descriptor() + => OciImageBuilder.ValidateBaseDescriptor("example.com/base:1", "layer 0", ValidDigest, 0); + + // ---- Response bodies are read with an upper bound --------------------------------------------------- + + /// + /// The most an endless body may be read past a limit: one pooled read buffer (ArrayPool rounds the + /// requested 80 KiB up to 128 KiB). + /// + private const long ReadSlack = 128 * 1024; + + [Fact] + public async Task GetManifestAsync_rejects_oversized_Content_Length_without_reading_the_body() + { + var body = new EndlessStream(); + var handler = new CountingHandler(_ => + { + var content = new StreamContent(body); + content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypes.OciManifestV1); + content.Headers.ContentLength = BoundedRead.MaxMetadataBytes + 1L; + return new HttpResponseMessage(HttpStatusCode.OK) { Content = content }; + }); + var client = new RegistryClient("example.com", new HttpClient(handler)); + + RegistryException ex = await Assert.ThrowsAsync(() => client.GetManifestAsync("repo", "latest")); + + Assert.Contains("limit", ex.Message); + Assert.Equal(0, body.BytesServed); + } + + [Fact] + public async Task GetManifestAsync_stops_reading_an_endless_body_at_the_limit() + { + var body = new EndlessStream(); + var handler = new CountingHandler(_ => + { + var content = new StreamContent(body); // no Content-Length: the size is only known by reading + content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypes.OciManifestV1); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = content }; + }); + var client = new RegistryClient("example.com", new HttpClient(handler)); + + await Assert.ThrowsAsync(() => client.GetManifestAsync("repo", "latest")); + + Assert.InRange(body.BytesServed, BoundedRead.MaxMetadataBytes, BoundedRead.MaxMetadataBytes + ReadSlack); + } + + [Fact] + public async Task Error_body_is_read_only_up_to_its_limit() + { + var body = new EndlessStream(); + var handler = new CountingHandler(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StreamContent(body), + }); + var client = new RegistryClient("example.com", new HttpClient(handler)); + + await Assert.ThrowsAsync(() => client.GetManifestAsync("repo", "latest")); + + Assert.InRange(body.BytesServed, 1, BoundedRead.MaxErrorBodyBytes + ReadSlack); + } + + [Fact] + public async Task GetVerifiedBlobBytesAsync_stops_when_blob_exceeds_declared_size() + { + var body = new EndlessStream(); + var handler = new CountingHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(body) }); + var client = new RegistryClient("example.com", new HttpClient(handler)); + + BlobSizeMismatchException ex = await Assert.ThrowsAsync( + () => client.GetVerifiedBlobBytesAsync("repo", ValidDigest, "base layer", expectedSize: 10)); + + Assert.Equal(10, ex.DeclaredSize); + Assert.Contains("larger", ex.Message); + Assert.InRange(body.BytesServed, 11, 10 + ReadSlack); + } + + [Fact] + public async Task GetVerifiedBlobToFileAsync_stops_when_blob_exceeds_declared_size() + { + var body = new EndlessStream(); + var handler = new CountingHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(body) }); + var client = new RegistryClient("example.com", new HttpClient(handler)); + + BlobSizeMismatchException ex = await Assert.ThrowsAsync( + () => client.GetVerifiedBlobToFileAsync("repo", ValidDigest, "base layer", expectedSize: 10)); + + Assert.Equal(10, ex.DeclaredSize); + Assert.InRange(body.BytesServed, 11, 10 + ReadSlack); + } + + [Fact] + public async Task Verified_downloads_reject_a_negative_expected_size() + { + var client = new RegistryClient("example.com", new HttpClient(new CountingHandler(_ => new HttpResponseMessage(HttpStatusCode.OK)))); + + await Assert.ThrowsAsync( + () => client.GetVerifiedBlobBytesAsync("repo", ValidDigest, "base layer", expectedSize: -1)); + await Assert.ThrowsAsync( + () => client.GetVerifiedBlobToFileAsync("repo", ValidDigest, "base layer", expectedSize: -1)); + } + + [Fact] + public async Task Oversized_token_response_is_rejected() + { + var tokenBody = new EndlessStream(); + var stub = new CountingHandler(request => + { + if (request.RequestUri!.Host == "auth.example.com") + { + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(tokenBody) }; + } + var challenge = new HttpResponseMessage(HttpStatusCode.Unauthorized); + challenge.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue( + "Bearer", "realm=\"https://auth.example.com/token\",service=\"registry.example.com\"")); + return challenge; + }); + var handler = new AuthHandshakeHandler(new AnonymousRegistryCredentialSource(), stub); + using var http = new HttpClient(handler); + + InvalidAuthResponseException ex = await Assert.ThrowsAsync( + () => http.GetAsync("https://registry.example.com/v2/")); + + Assert.Contains("limit", ex.Message); + Assert.InRange(tokenBody.BytesServed, BoundedRead.MaxMetadataBytes, BoundedRead.MaxMetadataBytes + ReadSlack); + } + + [Fact] + public async Task Unauthorized_body_is_drained_only_up_to_its_limit() + { + var challengeBody = new EndlessStream(); + var stub = new CountingHandler(request => + { + if (request.Headers.Authorization is null) + { + var challenge = new HttpResponseMessage(HttpStatusCode.Unauthorized) { Content = new StreamContent(challengeBody) }; + challenge.Headers.WwwAuthenticate.Add(new AuthenticationHeaderValue("Basic", "realm=\"registry\"")); + return challenge; + } + return new HttpResponseMessage(HttpStatusCode.OK); + }); + var handler = new AuthHandshakeHandler(new StaticRegistryCredentialSource("registry.example.com", "AWS", "secret"), stub); + using var http = new HttpClient(handler); + + using HttpResponseMessage response = await http.GetAsync("https://registry.example.com/v2/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.InRange(challengeBody.BytesServed, 1, BoundedRead.MaxErrorBodyBytes + ReadSlack); + } + + // ---- Helpers ------------------------------------------------------------------------------------------ + + private sealed class CountingHandler : HttpMessageHandler + { + private readonly Func _responder; + public int RequestCount { get; private set; } + + public CountingHandler(Func responder) => _responder = responder; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + RequestCount++; + return Task.FromResult(_responder(request)); + } + } + + /// A non-seekable stream that never ends, counting how many bytes were read from it. + private sealed class EndlessStream : Stream + { + public long BytesServed { get; private set; } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + + public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + buffer.Fill((byte)'a'); + BytesServed += buffer.Length; + return buffer.Length; + } + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => ValueTask.FromResult(Read(buffer.Span)); + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => Task.FromResult(Read(buffer.AsSpan(offset, count))); + + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} From 7bdb22f29225c466dc255f4ba0487392593dbd42 Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 28 Oct 2026 22:29:54 +0000 Subject: [PATCH 13/14] Make AWS.OCI.Builder plumbing types internal Reduce the public surface to the developer-facing types (OciImageBuilder, ImageConfig, FileManifest/FileManifestEntry/UnixPermissions, EcrPushOptions, push progress types) and the registry exceptions. Registry client, auth, credential source, image model and reference types are now internal, as are ImageConfig.Parse/AddLayer/ToJsonBytes/ToJsonString. --- Libraries/src/AWS.OCI.Builder/Descriptor.cs | 2 +- Libraries/src/AWS.OCI.Builder/ImageReference.cs | 2 +- Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs | 4 ++-- Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs | 8 ++++---- Libraries/src/AWS.OCI.Builder/Images/Layer.cs | 2 +- Libraries/src/AWS.OCI.Builder/Images/Manifest.cs | 8 ++++---- Libraries/src/AWS.OCI.Builder/MediaTypes.cs | 2 +- .../src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs | 2 +- .../AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs | 8 ++++---- Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs | 6 +++--- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Libraries/src/AWS.OCI.Builder/Descriptor.cs b/Libraries/src/AWS.OCI.Builder/Descriptor.cs index 9d7da4ca2..9c9951d03 100644 --- a/Libraries/src/AWS.OCI.Builder/Descriptor.cs +++ b/Libraries/src/AWS.OCI.Builder/Descriptor.cs @@ -7,7 +7,7 @@ namespace AWS.OCI.Builder; /// (a layer, a config, or a manifest) by its media type, size, and digest. /// See the OCI Image Specification "descriptor" definition. /// -public sealed record Descriptor +internal sealed record Descriptor { /// The media type of the referenced content (for example an image config or a gzipped layer). [JsonPropertyName("mediaType")] diff --git a/Libraries/src/AWS.OCI.Builder/ImageReference.cs b/Libraries/src/AWS.OCI.Builder/ImageReference.cs index ac3ea5f59..01b7ca6d2 100644 --- a/Libraries/src/AWS.OCI.Builder/ImageReference.cs +++ b/Libraries/src/AWS.OCI.Builder/ImageReference.cs @@ -22,7 +22,7 @@ namespace AWS.OCI.Builder; /// string is a Docker Hub repository. /// /// -public sealed partial class ImageReference +internal sealed partial class ImageReference { private const string DockerHubHost = "registry-1.docker.io"; diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs index d58153680..ecae55fea 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageBuilder.cs @@ -21,7 +21,7 @@ namespace AWS.OCI.Builder.Images; /// producing a manifest whose media types are inconsistent with its config, which some registries reject. /// /// -public sealed class ImageBuilder +internal sealed class ImageBuilder { private readonly ImageConfig _config; private readonly Manifest _manifest; @@ -164,7 +164,7 @@ private static string ComputeDigest(byte[] data) /// The canonical manifest bytes to upload (and PUT under the target tag). /// The sha256:... digest of (the image digest). /// The manifest media type to send as the Content-Type on upload. -public sealed record BuiltImage( +internal sealed record BuiltImage( byte[] ConfigBytes, string ConfigDigest, string ConfigMediaType, diff --git a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs index e399041b5..e4f7fa7e8 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/ImageConfig.cs @@ -41,7 +41,7 @@ public sealed class ImageConfig /// Parses an existing image config document (typically the base image's config blob). /// The config JSON. /// The JSON is null, empty, or not a JSON object. - public static ImageConfig Parse(string json) + internal static ImageConfig Parse(string json) { if (string.IsNullOrWhiteSpace(json)) { @@ -78,7 +78,7 @@ public static ImageConfig Parse(string json) /// /// The layer's uncompressed-tar digest (). /// An optional free-text description recorded in the history entry. - public void AddLayer(string diffId, string? createdBy = null) + internal void AddLayer(string diffId, string? createdBy = null) { ArgumentException.ThrowIfNullOrWhiteSpace(diffId); @@ -197,7 +197,7 @@ public void ExposePort(string port) } /// Serializes the config to its canonical, compact UTF-8 bytes (what the config digest is computed over). - public byte[] ToJsonBytes() + internal byte[] ToJsonBytes() { var buffer = new System.Buffers.ArrayBufferWriter(); using (var writer = new Utf8JsonWriter(buffer, WriterOptions)) @@ -208,7 +208,7 @@ public byte[] ToJsonBytes() } /// Serializes the config to its canonical, compact JSON string. - public string ToJsonString() => System.Text.Encoding.UTF8.GetString(ToJsonBytes()); + internal string ToJsonString() => System.Text.Encoding.UTF8.GetString(ToJsonBytes()); /// Returns the config sub-object, creating it if absent. private JsonObject ConfigObject() diff --git a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs index 47c77420c..6cc677464 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/Layer.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/Layer.cs @@ -32,7 +32,7 @@ namespace AWS.OCI.Builder.Images; /// guaranteed to be identical across builds, even for identical input. /// /// -public sealed class Layer +internal sealed class Layer { /// A fixed timestamp used for every tar entry. private static readonly DateTimeOffset ReproducibleTimestamp = DateTimeOffset.UnixEpoch; diff --git a/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs index 06ee4a48a..9f47e9eb1 100644 --- a/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs +++ b/Libraries/src/AWS.OCI.Builder/Images/Manifest.cs @@ -11,7 +11,7 @@ namespace AWS.OCI.Builder.Images; /// The manifest is serialized compactly and its property order is fixed (schema version, media type, config, /// layers), because the image's digest/tag is computed over these exact bytes. /// -public sealed class Manifest +internal sealed class Manifest { /// The manifest schema version. Always 2 for OCI and Docker V2 Schema 2. [JsonPropertyName("schemaVersion")] @@ -68,7 +68,7 @@ public static Manifest Parse(string json) /// single-platform for a specific OS/architecture. Base images are frequently published as /// an index, so the builder must select the right per-platform manifest from it. /// -public sealed class ImageIndex +internal sealed class ImageIndex { /// The schema version (always 2). [JsonPropertyName("schemaVersion")] @@ -120,7 +120,7 @@ m.Platform is { } p && } /// A manifest descriptor within an , carrying the target platform. -public sealed class PlatformManifestDescriptor +internal sealed class PlatformManifestDescriptor { /// The media type of the referenced manifest. [JsonPropertyName("mediaType")] @@ -140,7 +140,7 @@ public sealed class PlatformManifestDescriptor } /// An OS/architecture platform descriptor within an image index. -public sealed class Platform +internal sealed class Platform { /// The operating system, for example linux. [JsonPropertyName("os")] diff --git a/Libraries/src/AWS.OCI.Builder/MediaTypes.cs b/Libraries/src/AWS.OCI.Builder/MediaTypes.cs index 5e5fba11e..7562c27c0 100644 --- a/Libraries/src/AWS.OCI.Builder/MediaTypes.cs +++ b/Libraries/src/AWS.OCI.Builder/MediaTypes.cs @@ -5,7 +5,7 @@ namespace AWS.OCI.Builder; /// Docker Image Manifest V2 Schema 2. A base image may use either family; the builder /// preserves the base image's family when producing the output manifest. /// -public static class MediaTypes +internal static class MediaTypes { // --- OCI (application/vnd.oci.*) --- diff --git a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs index a7a2af533..65687d889 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/AuthHandshakeHandler.cs @@ -29,7 +29,7 @@ namespace AWS.OCI.Builder.Registry; /// source returns (public base-image pulls do not need credentials to fetch a token). /// /// -public sealed partial class AuthHandshakeHandler : DelegatingHandler +internal sealed partial class AuthHandshakeHandler : DelegatingHandler { private const string BasicScheme = "Basic"; private const string BearerScheme = "Bearer"; diff --git a/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs index 13843717f..09be45595 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/IRegistryCredentialSource.cs @@ -9,7 +9,7 @@ namespace AWS.OCI.Builder.Registry; /// from a caller-supplied ECR authorization token), while pulls of public base images use /// . /// -public interface IRegistryCredentialSource +internal interface IRegistryCredentialSource { /// /// Returns the credentials for , or to attempt @@ -27,7 +27,7 @@ public interface IRegistryCredentialSource /// never includes the password. /// The registry username. /// The registry password or token. -public readonly record struct BasicRegistryCredentials(string Username, string Password) +internal readonly record struct BasicRegistryCredentials(string Username, string Password) { /// Returns the username with the password redacted. public override string ToString() => $"{nameof(BasicRegistryCredentials)} {{ Username = {Username}, Password = *** }}"; @@ -38,7 +38,7 @@ public readonly record struct BasicRegistryCredentials(string Username, string P /// anonymous access. Suitable for pulling public base images (for example from /// mcr.microsoft.com or Docker Hub). /// -public sealed class AnonymousRegistryCredentialSource : IRegistryCredentialSource +internal sealed class AnonymousRegistryCredentialSource : IRegistryCredentialSource { /// public Task GetCredentialsAsync(string registryHost, CancellationToken cancellationToken = default) @@ -55,7 +55,7 @@ public sealed class AnonymousRegistryCredentialSource : IRegistryCredentialSourc /// host. A request that reaches an unexpected host (a forged upload location, a token endpoint on another host) /// therefore never receives them. /// -public sealed class StaticRegistryCredentialSource : IRegistryCredentialSource +internal sealed class StaticRegistryCredentialSource : IRegistryCredentialSource { private readonly string _registryHost; private readonly BasicRegistryCredentials _credentials; diff --git a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs index 391e247c9..c65cce80f 100644 --- a/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs +++ b/Libraries/src/AWS.OCI.Builder/Registry/RegistryClient.cs @@ -19,7 +19,7 @@ namespace AWS.OCI.Builder.Registry; /// are deliberately avoided. /// /// -public sealed class RegistryClient : IDisposable +internal sealed class RegistryClient : IDisposable { private readonly HttpClient _http; private readonly string _registryHost; @@ -866,13 +866,13 @@ public void Dispose() /// The exact bytes received on the wire — digests must be computed over these. /// The manifest media type (OCI or Docker; single image or index). /// The server-reported Docker-Content-Digest, if present. -public readonly record struct ManifestResponse(string Json, byte[] RawBytes, string MediaType, string? Digest); +internal readonly record struct ManifestResponse(string Json, byte[] RawBytes, string MediaType, string? Digest); /// /// A verified blob that has been spilled to a temporary file on disk. Owns both the open read stream (positioned /// at the start, ready to upload) and the temp file; disposing it closes the stream and deletes the file. /// -public sealed class VerifiedBlobFile : IAsyncDisposable, IDisposable +internal sealed class VerifiedBlobFile : IAsyncDisposable, IDisposable { private readonly string _path; private bool _disposed; From 172877304db2dda852e8dffb713758dd29d9445a Mon Sep 17 00:00:00 2001 From: Norm Johanson Date: Wed, 28 Oct 2026 15:37:15 -0700 Subject: [PATCH 14/14] Remove unused AWSSDK.ECR reference --- .../test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj | 2 -- 1 file changed, 2 deletions(-) diff --git a/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj b/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj index 2578b3bb6..ea33250e6 100644 --- a/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj +++ b/Libraries/test/AWS.OCI.Builder.Tests/AWS.OCI.Builder.Tests.csproj @@ -16,8 +16,6 @@ - -