From 189a4bc5057bf9241899ee1bdde671ea67620d61 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Mon, 12 Oct 2026 17:03:01 -0400 Subject: [PATCH 01/37] fix --- e2e/config/vhd.go | 32 ++++++++ e2e/runner/filter_test.go | 3 +- e2e/scenario/scenario_cvm.go | 128 ++++++++++++++++++++++++++++++ e2e/scenario/scenario_cvm_test.go | 97 ++++++++++++++++++++++ e2e/scenario/types.go | 1 + 5 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 e2e/scenario/scenario_cvm.go create mode 100644 e2e/scenario/scenario_cvm_test.go diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index dd44c1e13be..4b54fc2b4fc 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -35,6 +35,14 @@ var ( ) var ( + VHDUbuntu2004CVMGen2Containerd = &Image{ + Name: "2004gen2CVMcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2004CVMGen2, + Gallery: &Config.GalleryLinux, + } + VHDUbuntu2204Gen2Arm64Containerd = &Image{ Name: "2204gen2arm64containerd", OS: OSUbuntu, @@ -115,6 +123,14 @@ var ( Gallery: &Config.GalleryLinux, } + VHDAzureLinuxV3CVMGen2 = &Image{ + Name: "V3gen2CVM", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3CVMGen2, + Gallery: &Config.GalleryLinux, + } + // VHDAzureLinuxV3Gen2Kata is the AzureLinux V3 Gen2 VHD built with FEATURE_FLAGS=kata. // The image definition name mirrors the SIG_IMAGE_NAME produced by the VHD builder for // OS_VERSION=V3kata + HYPERV_GENERATION=V2 (SKU_NAME=V3katagen2, prefixed with "AzureLinux"). @@ -154,6 +170,14 @@ var ( Gallery: &Config.GalleryLinux, } + VHDUbuntu2404CVMGen2Containerd = &Image{ + Name: "2404gen2CVMcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2404CVMGen2, + Gallery: &Config.GalleryLinux, + } + VHDUbuntu2404ArmContainerd = &Image{ Name: "2404gen2arm64containerd", OS: OSUbuntu, @@ -170,6 +194,14 @@ var ( Gallery: &Config.GalleryLinux, } + VHDUbuntu2604MinimalCVMGen2Containerd = &Image{ + Name: "2604minimalgen2CVMcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuMinimalContainerd2604CVMGen2, + Gallery: &Config.GalleryLinux, + } + VHDUbuntu2604MinimalArm64Gen2Containerd = &Image{ Name: "2604minimalgen2arm64containerd", OS: OSUbuntu, diff --git a/e2e/runner/filter_test.go b/e2e/runner/filter_test.go index 2cf0bbffa1e..9fff63a8792 100644 --- a/e2e/runner/filter_test.go +++ b/e2e/runner/filter_test.go @@ -122,7 +122,7 @@ func restoreRunnerConfig(t *testing.T) { } func TestMatchFiltersPreservesTagPolicy(t *testing.T) { - tags := scenario.Tags{Name: "Ubuntu2204", OS: "linux", GPU: true} + tags := scenario.Tags{Name: "Ubuntu2204", OS: "linux", GPU: true, CVM: true} for _, test := range []struct { filter string all bool @@ -132,6 +132,7 @@ func TestMatchFiltersPreservesTagPolicy(t *testing.T) { {filter: "Name=Ubuntu2204,GPU=false", all: true, match: false}, {filter: "Name=Ubuntu2204,GPU=false", all: false, match: true}, {filter: " os = LINUX , gpu = true ", all: true, match: true}, + {filter: "CVM=true", all: true, match: true}, {filter: "", all: true, match: true}, } { t.Run(test.filter, func(t *testing.T) { diff --git a/e2e/scenario/scenario_cvm.go b/e2e/scenario/scenario_cvm.go new file mode 100644 index 00000000000..45f7063436a --- /dev/null +++ b/e2e/scenario/scenario_cvm.go @@ -0,0 +1,128 @@ +package scenario + +import ( + "context" + + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v8" +) + +const ( + cvmLocation = "westeurope" + cvmSNPSKU = "Standard_DC2ads_v5" + cvmTDXSKU = "Standard_DC2eds_v6" +) + +var ( + _ = Register(newCVMScenario( + "Ubuntu2004_CVM_SEVSNP", + "Tests that an Ubuntu 20.04 Confidential VM node can be provisioned and pass standard node validation", + config.VHDUbuntu2004CVMGen2Containerd, + ClusterKubenet, + cvmSNPSKU, + false, + )) + _ = Register(newCVMScenario( + "Ubuntu2404_CVM_SEVSNP", + "Tests that an Ubuntu 24.04 Confidential VM node can be provisioned and pass standard node validation", + config.VHDUbuntu2404CVMGen2Containerd, + ClusterKubenet, + cvmSNPSKU, + false, + )) + _ = Register(newCVMScenario( + "AzureLinuxV3_CVM_SEVSNP", + "Tests that an Azure Linux V3 Confidential VM node can be provisioned and pass standard node validation", + config.VHDAzureLinuxV3CVMGen2, + ClusterKubenet, + cvmSNPSKU, + false, + )) + _ = Register(newCVMScenario( + "Ubuntu2604Minimal_CVM_SEVSNP", + "Tests that an Ubuntu 26.04 minimal Confidential VM node can be provisioned on SEV-SNP and pass standard node validation", + config.VHDUbuntu2604MinimalCVMGen2Containerd, + ClusterLatestKubernetesVersionKubenet, + cvmSNPSKU, + false, + )) + _ = Register(newCVMScenario( + "Ubuntu2604Minimal_CVM_TDX", + "Tests that an Ubuntu 26.04 minimal Confidential VM node can be provisioned on TDX and pass standard node validation", + config.VHDUbuntu2604MinimalCVMGen2Containerd, + ClusterLatestKubernetesVersionKubenet, + cvmTDXSKU, + true, + )) +) + +func newCVMScenario( + name string, + description string, + vhd *config.Image, + cluster func(context.Context, ClusterRequest) (*Cluster, error), + vmSize string, + useNVMe bool, +) *Scenario { + return &Scenario{ + Name: name, + Description: description, + Location: cvmLocation, + Tags: Tags{ + CVM: true, + }, + Config: Config{ + Cluster: cluster, + VHD: vhd, + UseNVMe: useNVMe, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = vmSize + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + configureConfidentialVM(vmss, vmSize) + }, + }, + } +} + +func configureConfidentialVM(vmss *armcompute.VirtualMachineScaleSet, vmSize string) { + if vmss.SKU == nil { + vmss.SKU = &armcompute.SKU{} + } + vmss.SKU.Name = to.Ptr(vmSize) + + if vmss.Properties == nil { + vmss.Properties = &armcompute.VirtualMachineScaleSetProperties{} + } + if vmss.Properties.VirtualMachineProfile == nil { + vmss.Properties.VirtualMachineProfile = &armcompute.VirtualMachineScaleSetVMProfile{} + } + profile := vmss.Properties.VirtualMachineProfile + if profile.SecurityProfile == nil { + profile.SecurityProfile = &armcompute.SecurityProfile{} + } + profile.SecurityProfile.SecurityType = to.Ptr(armcompute.SecurityTypesConfidentialVM) + profile.SecurityProfile.EncryptionAtHost = to.Ptr(true) + if profile.SecurityProfile.UefiSettings == nil { + profile.SecurityProfile.UefiSettings = &armcompute.UefiSettings{} + } + profile.SecurityProfile.UefiSettings.SecureBootEnabled = to.Ptr(true) + profile.SecurityProfile.UefiSettings.VTpmEnabled = to.Ptr(true) + + if profile.StorageProfile == nil { + profile.StorageProfile = &armcompute.VirtualMachineScaleSetStorageProfile{} + } + if profile.StorageProfile.OSDisk == nil { + profile.StorageProfile.OSDisk = &armcompute.VirtualMachineScaleSetOSDisk{} + } + if profile.StorageProfile.OSDisk.ManagedDisk == nil { + profile.StorageProfile.OSDisk.ManagedDisk = &armcompute.VirtualMachineScaleSetManagedDiskParameters{} + } + if profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile == nil { + profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile = &armcompute.VMDiskSecurityProfile{} + } + profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType = + to.Ptr(armcompute.SecurityEncryptionTypesVMGuestStateOnly) +} diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go new file mode 100644 index 00000000000..9da6696cd3d --- /dev/null +++ b/e2e/scenario/scenario_cvm_test.go @@ -0,0 +1,97 @@ +package scenario + +import ( + "testing" + + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v8" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewCVMScenarioUsesWestEuropeAndDefaultValidation(t *testing.T) { + s := newCVMScenario( + "test", + "description", + config.VHDUbuntu2404CVMGen2Containerd, + ClusterKubenet, + cvmSNPSKU, + false, + ) + + assert.Equal(t, cvmLocation, s.Location) + assert.True(t, s.Tags.CVM) + assert.Equal(t, config.VHDUbuntu2404CVMGen2Containerd, s.VHD) + assert.False(t, s.SkipDefaultValidation) + assert.Nil(t, s.Validator) + + nbc := &datamodel.NodeBootstrappingConfiguration{ + AgentPoolProfile: &datamodel.AgentPoolProfile{}, + } + s.BootstrapConfigMutator(nil, nbc) + assert.Equal(t, cvmSNPSKU, nbc.AgentPoolProfile.VMSize) +} + +func TestRegisteredCVMScenarios(t *testing.T) { + expected := map[string]struct { + vhd *config.Image + useNVMe bool + }{ + "Ubuntu2004_CVM_SEVSNP": {vhd: config.VHDUbuntu2004CVMGen2Containerd}, + "Ubuntu2404_CVM_SEVSNP": {vhd: config.VHDUbuntu2404CVMGen2Containerd}, + "AzureLinuxV3_CVM_SEVSNP": {vhd: config.VHDAzureLinuxV3CVMGen2}, + "Ubuntu2604Minimal_CVM_SEVSNP": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd}, + "Ubuntu2604Minimal_CVM_TDX": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, useNVMe: true}, + } + + var found int + for _, s := range List() { + if !s.Tags.CVM { + continue + } + found++ + want, ok := expected[s.Name] + require.True(t, ok, "unexpected CVM scenario %q", s.Name) + assert.Equal(t, cvmLocation, s.Location) + assert.Equal(t, want.vhd, s.VHD) + assert.Equal(t, want.useNVMe, s.UseNVMe) + assert.False(t, s.SkipDefaultValidation) + } + assert.Equal(t, len(expected), found) +} + +func TestConfigureConfidentialVM(t *testing.T) { + vmss := &armcompute.VirtualMachineScaleSet{} + + configureConfidentialVM(vmss, cvmTDXSKU) + + require.NotNil(t, vmss.SKU) + require.NotNil(t, vmss.SKU.Name) + assert.Equal(t, cvmTDXSKU, *vmss.SKU.Name) + + require.NotNil(t, vmss.Properties) + require.NotNil(t, vmss.Properties.VirtualMachineProfile) + profile := vmss.Properties.VirtualMachineProfile + require.NotNil(t, profile.SecurityProfile) + require.NotNil(t, profile.SecurityProfile.SecurityType) + assert.Equal(t, armcompute.SecurityTypesConfidentialVM, *profile.SecurityProfile.SecurityType) + require.NotNil(t, profile.SecurityProfile.EncryptionAtHost) + assert.True(t, *profile.SecurityProfile.EncryptionAtHost) + require.NotNil(t, profile.SecurityProfile.UefiSettings) + require.NotNil(t, profile.SecurityProfile.UefiSettings.SecureBootEnabled) + assert.True(t, *profile.SecurityProfile.UefiSettings.SecureBootEnabled) + require.NotNil(t, profile.SecurityProfile.UefiSettings.VTpmEnabled) + assert.True(t, *profile.SecurityProfile.UefiSettings.VTpmEnabled) + + require.NotNil(t, profile.StorageProfile) + require.NotNil(t, profile.StorageProfile.OSDisk) + require.NotNil(t, profile.StorageProfile.OSDisk.ManagedDisk) + require.NotNil(t, profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile) + require.NotNil(t, profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType) + assert.Equal( + t, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, + *profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType, + ) +} diff --git a/e2e/scenario/types.go b/e2e/scenario/types.go index 50fd3dd57f0..ad0d3f757ea 100644 --- a/e2e/scenario/types.go +++ b/e2e/scenario/types.go @@ -24,6 +24,7 @@ type Tags struct { NetworkIsolated bool NonAnonymousACR bool GPU bool + CVM bool WASM bool Kata bool BootstrapTokenFallback bool From d02bf9d1bcb18cfad7f1dce9c4f20f59079e1b2f Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 13 Oct 2026 14:12:55 -0400 Subject: [PATCH 02/37] fix --- .pipelines/.vsts-vhd-builder-release.yaml | 4 ++++ .pipelines/templates/.builder-release-template.yaml | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index 0f52c4a2ad1..51b345cf5bd 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -383,6 +383,7 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: azurelinuxv3-cvm-gen2 + e2eImageName: V3gen2CVM - job: buildAzureLinuxOSGuardV3gen2fipsTL condition: eq('${{ parameters.buildAzureLinuxOSGuardV3gen2fipsTL }}', true) dependsOn: [ ] @@ -690,6 +691,7 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2404-cvm-gen2-containerd + e2eImageName: 2404gen2CVMcontainerd - job: build2404gen2containerd condition: eq('${{ parameters.build2404gen2containerd }}', true) dependsOn: [ ] @@ -765,6 +767,7 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2004-cvm-gen2-containerd + e2eImageName: 2004gen2CVMcontainerd - job: build2204tlgen2containerd condition: eq('${{ parameters.build2204tlgen2containerd }}', true) dependsOn: [ ] @@ -864,6 +867,7 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2604-minimal-cvm-gen2-containerd + e2eImageName: 2604minimalgen2CVMcontainerd - job: buildacltlgen2 condition: eq('${{ parameters.buildacltlgen2 }}', true) dependsOn: [ ] diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 682a81b5bf3..c21c59922bd 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -2,6 +2,9 @@ parameters: - name: artifactName type: string default: 2404-gen2-containerd + - name: e2eImageName + type: string + default: '' - name: useOverrides type: boolean default: false @@ -293,8 +296,10 @@ steps: --arg replications "${REPLICATIONS:-}" \ '([$buildLocation] + ($replications | split(",") | map(split("=")[0] | gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)))) | unique')" + metadata_image_name="${E2E_IMAGE_NAME:-${SIG_IMAGE_NAME}}" + jq -n \ - --arg imageName "${SIG_IMAGE_NAME}" \ + --arg imageName "$metadata_image_name" \ --arg resourceId "${MANAGED_SIG_ID}" \ --arg version "${CAPTURED_SIG_VERSION}" \ --argjson regions "$regions" \ @@ -303,6 +308,7 @@ steps: displayName: Create E2E VHD metadata env: VHD_ARTIFACT_NAME: ${{ parameters.artifactName }} + E2E_IMAGE_NAME: ${{ parameters.e2eImageName }} SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) MANAGED_SIG_ID: $(MANAGED_SIG_ID) From 6aa4672937eecb570d5f187a87a408dd1e387a25 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 13 Oct 2026 15:24:11 -0400 Subject: [PATCH 03/37] fix --- .pipelines/.vsts-vhd-builder-release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index 51b345cf5bd..24c7f8adb48 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -370,7 +370,7 @@ stages: echo '##vso[task.setvariable variable=IMG_SKU]azure-linux-3-cvm' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC16ads_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC4ads_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]cvm' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' @@ -679,7 +679,7 @@ stages: echo '##vso[task.setvariable variable=IMG_SKU]cvm' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC16ads_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC4ads_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]cvm' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' @@ -755,7 +755,7 @@ stages: echo '##vso[task.setvariable variable=IMG_SKU]20_04-lts-cvm' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC16ads_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC4ads_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]cvm' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' @@ -856,7 +856,7 @@ stages: echo '##vso[task.setvariable variable=IMG_SKU]server-cvm' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC16ads_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC4ads_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]minimal,cvm' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' From 14c98ea37ee4b5924fc14240db438727e63d5c25 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 13 Oct 2026 16:38:39 -0400 Subject: [PATCH 04/37] fix --- e2e/config/vhd.go | 21 ++++++++++++++++--- e2e/config/vhd_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 e2e/config/vhd_test.go diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 4b54fc2b4fc..9c7160c48c2 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -345,6 +345,19 @@ func (i *Image) SupportsScriptless() bool { return !i.Flatcar && !i.Distro.IsWindowsDistro() && i.Distro != datamodel.AKSAzureLinuxV2Gen2 } +func (i Image) vhdBuilderGalleryImageName() string { + switch i.Distro { + case datamodel.AKSUbuntuContainerd2004CVMGen2, + datamodel.AKSUbuntuContainerd2404CVMGen2, + datamodel.AKSUbuntuMinimalContainerd2604CVMGen2: + return i.Name + "Specialized" + case datamodel.AKSAzureLinuxV3CVMGen2: + return "AzureLinux" + i.Name + default: + return i.Name + } +} + func GetVHDResourceID(ctx context.Context, i Image, location string) (VHDResourceID, error) { if i.Version == "" && Config.vhdMetadata != nil { vhd, err := getVHDResourceIDFromMetadata(Config.vhdMetadata, i, location) @@ -364,14 +377,16 @@ func GetVHDResourceID(ctx context.Context, i Image, location string) (VHDResourc logging.Logf(ctx, "Got image by version: %s", i.azurePortalImageVersionUrl()) return vhd, nil default: - vhd, err := Azure.LatestSIGImageVersionByTag(ctx, &i, Config.SIGVersionTagName, Config.SIGVersionTagValue, location) + builderImage := i + builderImage.Name = i.vhdBuilderGalleryImageName() + vhd, err := Azure.LatestSIGImageVersionByTag(ctx, &builderImage, Config.SIGVersionTagName, Config.SIGVersionTagValue, location) if err != nil { return "", fmt.Errorf("failed to get latest image by tag %s=%s: %w", Config.SIGVersionTagName, Config.SIGVersionTagValue, err) } if vhd != "" { - logging.Logf(ctx, "got version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, i.azurePortalImageVersionUrl()) + logging.Logf(ctx, "got version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, builderImage.azurePortalImageVersionUrl()) } else { - logging.Logf(ctx, "Could not find version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, i.azurePortalImageUrl()) + logging.Logf(ctx, "Could not find version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, builderImage.azurePortalImageUrl()) } return vhd, nil } diff --git a/e2e/config/vhd_test.go b/e2e/config/vhd_test.go new file mode 100644 index 00000000000..392f6a0c5ba --- /dev/null +++ b/e2e/config/vhd_test.go @@ -0,0 +1,47 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestVHDBuilderGalleryImageName(t *testing.T) { + tests := []struct { + name string + image Image + expected string + }{ + { + name: "Ubuntu 2004 CVM", + image: *VHDUbuntu2004CVMGen2Containerd, + expected: "2004gen2CVMcontainerdSpecialized", + }, + { + name: "Ubuntu 2404 CVM", + image: *VHDUbuntu2404CVMGen2Containerd, + expected: "2404gen2CVMcontainerdSpecialized", + }, + { + name: "Ubuntu 2604 minimal CVM", + image: *VHDUbuntu2604MinimalCVMGen2Containerd, + expected: "2604minimalgen2CVMcontainerdSpecialized", + }, + { + name: "Azure Linux V3 CVM", + image: *VHDAzureLinuxV3CVMGen2, + expected: "AzureLinuxV3gen2CVM", + }, + { + name: "non-CVM image", + image: *VHDUbuntu2404Gen2Containerd, + expected: "2404gen2containerd", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, tt.image.vhdBuilderGalleryImageName()) + }) + } +} From 953ae9b37be4b1fed12bfa6f0fc0e4ed8a9a8762 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 13 Oct 2026 17:08:12 -0400 Subject: [PATCH 05/37] fix --- .pipelines/.vsts-vhd-builder-release.yaml | 4 -- .../templates/.builder-release-template.yaml | 8 +--- e2e/config/vhd.go | 11 ++--- e2e/config/vhd_test.go | 46 +++++++++++++++++++ 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index 24c7f8adb48..fc6f667bafb 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -383,7 +383,6 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: azurelinuxv3-cvm-gen2 - e2eImageName: V3gen2CVM - job: buildAzureLinuxOSGuardV3gen2fipsTL condition: eq('${{ parameters.buildAzureLinuxOSGuardV3gen2fipsTL }}', true) dependsOn: [ ] @@ -691,7 +690,6 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2404-cvm-gen2-containerd - e2eImageName: 2404gen2CVMcontainerd - job: build2404gen2containerd condition: eq('${{ parameters.build2404gen2containerd }}', true) dependsOn: [ ] @@ -767,7 +765,6 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2004-cvm-gen2-containerd - e2eImageName: 2004gen2CVMcontainerd - job: build2204tlgen2containerd condition: eq('${{ parameters.build2204tlgen2containerd }}', true) dependsOn: [ ] @@ -867,7 +864,6 @@ stages: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: 2604-minimal-cvm-gen2-containerd - e2eImageName: 2604minimalgen2CVMcontainerd - job: buildacltlgen2 condition: eq('${{ parameters.buildacltlgen2 }}', true) dependsOn: [ ] diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index c21c59922bd..682a81b5bf3 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -2,9 +2,6 @@ parameters: - name: artifactName type: string default: 2404-gen2-containerd - - name: e2eImageName - type: string - default: '' - name: useOverrides type: boolean default: false @@ -296,10 +293,8 @@ steps: --arg replications "${REPLICATIONS:-}" \ '([$buildLocation] + ($replications | split(",") | map(split("=")[0] | gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)))) | unique')" - metadata_image_name="${E2E_IMAGE_NAME:-${SIG_IMAGE_NAME}}" - jq -n \ - --arg imageName "$metadata_image_name" \ + --arg imageName "${SIG_IMAGE_NAME}" \ --arg resourceId "${MANAGED_SIG_ID}" \ --arg version "${CAPTURED_SIG_VERSION}" \ --argjson regions "$regions" \ @@ -308,7 +303,6 @@ steps: displayName: Create E2E VHD metadata env: VHD_ARTIFACT_NAME: ${{ parameters.artifactName }} - E2E_IMAGE_NAME: ${{ parameters.e2eImageName }} SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) MANAGED_SIG_ID: $(MANAGED_SIG_ID) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 9c7160c48c2..f04d78439f7 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -377,16 +377,14 @@ func GetVHDResourceID(ctx context.Context, i Image, location string) (VHDResourc logging.Logf(ctx, "Got image by version: %s", i.azurePortalImageVersionUrl()) return vhd, nil default: - builderImage := i - builderImage.Name = i.vhdBuilderGalleryImageName() - vhd, err := Azure.LatestSIGImageVersionByTag(ctx, &builderImage, Config.SIGVersionTagName, Config.SIGVersionTagValue, location) + vhd, err := Azure.LatestSIGImageVersionByTag(ctx, &i, Config.SIGVersionTagName, Config.SIGVersionTagValue, location) if err != nil { return "", fmt.Errorf("failed to get latest image by tag %s=%s: %w", Config.SIGVersionTagName, Config.SIGVersionTagValue, err) } if vhd != "" { - logging.Logf(ctx, "got version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, builderImage.azurePortalImageVersionUrl()) + logging.Logf(ctx, "got version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, i.azurePortalImageVersionUrl()) } else { - logging.Logf(ctx, "Could not find version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, builderImage.azurePortalImageUrl()) + logging.Logf(ctx, "Could not find version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, i.azurePortalImageUrl()) } return vhd, nil } @@ -416,8 +414,9 @@ func loadVHDMetadata(path string) (map[string]vhdMetadataEntry, error) { func getVHDResourceIDFromMetadata(metadata map[string]vhdMetadataEntry, image Image, location string) (VHDResourceID, error) { var entry vhdMetadataEntry var found bool + builderGalleryImageName := image.vhdBuilderGalleryImageName() for imageName, candidate := range metadata { - if strings.EqualFold(imageName, image.Name) { + if strings.EqualFold(imageName, image.Name) || strings.EqualFold(imageName, builderGalleryImageName) { entry = candidate found = true break diff --git a/e2e/config/vhd_test.go b/e2e/config/vhd_test.go index 392f6a0c5ba..748ff66d4e2 100644 --- a/e2e/config/vhd_test.go +++ b/e2e/config/vhd_test.go @@ -45,3 +45,49 @@ func TestVHDBuilderGalleryImageName(t *testing.T) { }) } } + +func TestGetVHDResourceIDFromMetadataUsesVHDBuilderGalleryImageName(t *testing.T) { + tests := []struct { + name string + image Image + metadataImageName string + }{ + { + name: "Ubuntu 2004 CVM", + image: *VHDUbuntu2004CVMGen2Containerd, + metadataImageName: "2004gen2CVMcontainerdSpecialized", + }, + { + name: "Ubuntu 2404 CVM", + image: *VHDUbuntu2404CVMGen2Containerd, + metadataImageName: "2404gen2CVMcontainerdSpecialized", + }, + { + name: "Ubuntu 2604 minimal CVM", + image: *VHDUbuntu2604MinimalCVMGen2Containerd, + metadataImageName: "2604minimalgen2CVMcontainerdSpecialized", + }, + { + name: "Azure Linux V3 CVM", + image: *VHDAzureLinuxV3CVMGen2, + metadataImageName: "AzureLinuxV3gen2CVM", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expectedResourceID := VHDResourceID("/subscriptions/test/resourceGroups/test/providers/Microsoft.Compute/galleries/test/images/test/versions/1.0.0") + metadata := map[string]vhdMetadataEntry{ + tt.metadataImageName: { + ResourceID: expectedResourceID, + Regions: []string{"west europe"}, + }, + } + + resourceID, err := getVHDResourceIDFromMetadata(metadata, tt.image, "West Europe") + + require.NoError(t, err) + require.Equal(t, expectedResourceID, resourceID) + }) + } +} From 791bbd60c42906725b7ca6886e4758aa6210bbec Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 13 Oct 2026 17:21:59 -0400 Subject: [PATCH 06/37] fix --- e2e/scenario/scenario_cvm.go | 1 + e2e/scenario/scenario_cvm_test.go | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/e2e/scenario/scenario_cvm.go b/e2e/scenario/scenario_cvm.go index 45f7063436a..b39d4206e1d 100644 --- a/e2e/scenario/scenario_cvm.go +++ b/e2e/scenario/scenario_cvm.go @@ -100,6 +100,7 @@ func configureConfidentialVM(vmss *armcompute.VirtualMachineScaleSet, vmSize str vmss.Properties.VirtualMachineProfile = &armcompute.VirtualMachineScaleSetVMProfile{} } profile := vmss.Properties.VirtualMachineProfile + profile.OSProfile = nil if profile.SecurityProfile == nil { profile.SecurityProfile = &armcompute.SecurityProfile{} } diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go index 9da6696cd3d..0894e2158c5 100644 --- a/e2e/scenario/scenario_cvm_test.go +++ b/e2e/scenario/scenario_cvm_test.go @@ -62,7 +62,13 @@ func TestRegisteredCVMScenarios(t *testing.T) { } func TestConfigureConfidentialVM(t *testing.T) { - vmss := &armcompute.VirtualMachineScaleSet{} + vmss := &armcompute.VirtualMachineScaleSet{ + Properties: &armcompute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ + OSProfile: &armcompute.VirtualMachineScaleSetOSProfile{}, + }, + }, + } configureConfidentialVM(vmss, cvmTDXSKU) @@ -73,6 +79,7 @@ func TestConfigureConfidentialVM(t *testing.T) { require.NotNil(t, vmss.Properties) require.NotNil(t, vmss.Properties.VirtualMachineProfile) profile := vmss.Properties.VirtualMachineProfile + assert.Nil(t, profile.OSProfile) require.NotNil(t, profile.SecurityProfile) require.NotNil(t, profile.SecurityProfile.SecurityType) assert.Equal(t, armcompute.SecurityTypesConfidentialVM, *profile.SecurityProfile.SecurityType) From bc412880945991e5a1575ac253cd2ebb22de440a Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 14 Oct 2026 14:45:42 -0400 Subject: [PATCH 07/37] fix --- e2e/config/vhd.go | 16 +--------- e2e/config/vhd_test.go | 50 ++++--------------------------- e2e/scenario/scenario_cvm.go | 1 - e2e/scenario/scenario_cvm_test.go | 6 ++-- 4 files changed, 10 insertions(+), 63 deletions(-) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index f04d78439f7..4b54fc2b4fc 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -345,19 +345,6 @@ func (i *Image) SupportsScriptless() bool { return !i.Flatcar && !i.Distro.IsWindowsDistro() && i.Distro != datamodel.AKSAzureLinuxV2Gen2 } -func (i Image) vhdBuilderGalleryImageName() string { - switch i.Distro { - case datamodel.AKSUbuntuContainerd2004CVMGen2, - datamodel.AKSUbuntuContainerd2404CVMGen2, - datamodel.AKSUbuntuMinimalContainerd2604CVMGen2: - return i.Name + "Specialized" - case datamodel.AKSAzureLinuxV3CVMGen2: - return "AzureLinux" + i.Name - default: - return i.Name - } -} - func GetVHDResourceID(ctx context.Context, i Image, location string) (VHDResourceID, error) { if i.Version == "" && Config.vhdMetadata != nil { vhd, err := getVHDResourceIDFromMetadata(Config.vhdMetadata, i, location) @@ -414,9 +401,8 @@ func loadVHDMetadata(path string) (map[string]vhdMetadataEntry, error) { func getVHDResourceIDFromMetadata(metadata map[string]vhdMetadataEntry, image Image, location string) (VHDResourceID, error) { var entry vhdMetadataEntry var found bool - builderGalleryImageName := image.vhdBuilderGalleryImageName() for imageName, candidate := range metadata { - if strings.EqualFold(imageName, image.Name) || strings.EqualFold(imageName, builderGalleryImageName) { + if strings.EqualFold(imageName, image.Name) { entry = candidate found = true break diff --git a/e2e/config/vhd_test.go b/e2e/config/vhd_test.go index 748ff66d4e2..1ee0e34a9fb 100644 --- a/e2e/config/vhd_test.go +++ b/e2e/config/vhd_test.go @@ -6,47 +6,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestVHDBuilderGalleryImageName(t *testing.T) { - tests := []struct { - name string - image Image - expected string - }{ - { - name: "Ubuntu 2004 CVM", - image: *VHDUbuntu2004CVMGen2Containerd, - expected: "2004gen2CVMcontainerdSpecialized", - }, - { - name: "Ubuntu 2404 CVM", - image: *VHDUbuntu2404CVMGen2Containerd, - expected: "2404gen2CVMcontainerdSpecialized", - }, - { - name: "Ubuntu 2604 minimal CVM", - image: *VHDUbuntu2604MinimalCVMGen2Containerd, - expected: "2604minimalgen2CVMcontainerdSpecialized", - }, - { - name: "Azure Linux V3 CVM", - image: *VHDAzureLinuxV3CVMGen2, - expected: "AzureLinuxV3gen2CVM", - }, - { - name: "non-CVM image", - image: *VHDUbuntu2404Gen2Containerd, - expected: "2404gen2containerd", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require.Equal(t, tt.expected, tt.image.vhdBuilderGalleryImageName()) - }) - } -} - -func TestGetVHDResourceIDFromMetadataUsesVHDBuilderGalleryImageName(t *testing.T) { +func TestGetVHDResourceIDFromMetadataUsesLogicalImageName(t *testing.T) { tests := []struct { name string image Image @@ -55,22 +15,22 @@ func TestGetVHDResourceIDFromMetadataUsesVHDBuilderGalleryImageName(t *testing.T { name: "Ubuntu 2004 CVM", image: *VHDUbuntu2004CVMGen2Containerd, - metadataImageName: "2004gen2CVMcontainerdSpecialized", + metadataImageName: "2004gen2CVMcontainerd", }, { name: "Ubuntu 2404 CVM", image: *VHDUbuntu2404CVMGen2Containerd, - metadataImageName: "2404gen2CVMcontainerdSpecialized", + metadataImageName: "2404gen2CVMcontainerd", }, { name: "Ubuntu 2604 minimal CVM", image: *VHDUbuntu2604MinimalCVMGen2Containerd, - metadataImageName: "2604minimalgen2CVMcontainerdSpecialized", + metadataImageName: "2604minimalgen2CVMcontainerd", }, { name: "Azure Linux V3 CVM", image: *VHDAzureLinuxV3CVMGen2, - metadataImageName: "AzureLinuxV3gen2CVM", + metadataImageName: "V3gen2CVM", }, } diff --git a/e2e/scenario/scenario_cvm.go b/e2e/scenario/scenario_cvm.go index b39d4206e1d..45f7063436a 100644 --- a/e2e/scenario/scenario_cvm.go +++ b/e2e/scenario/scenario_cvm.go @@ -100,7 +100,6 @@ func configureConfidentialVM(vmss *armcompute.VirtualMachineScaleSet, vmSize str vmss.Properties.VirtualMachineProfile = &armcompute.VirtualMachineScaleSetVMProfile{} } profile := vmss.Properties.VirtualMachineProfile - profile.OSProfile = nil if profile.SecurityProfile == nil { profile.SecurityProfile = &armcompute.SecurityProfile{} } diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go index 0894e2158c5..95115c22dfb 100644 --- a/e2e/scenario/scenario_cvm_test.go +++ b/e2e/scenario/scenario_cvm_test.go @@ -57,15 +57,17 @@ func TestRegisteredCVMScenarios(t *testing.T) { assert.Equal(t, want.vhd, s.VHD) assert.Equal(t, want.useNVMe, s.UseNVMe) assert.False(t, s.SkipDefaultValidation) + assert.Nil(t, s.Validator) } assert.Equal(t, len(expected), found) } func TestConfigureConfidentialVM(t *testing.T) { + osProfile := &armcompute.VirtualMachineScaleSetOSProfile{} vmss := &armcompute.VirtualMachineScaleSet{ Properties: &armcompute.VirtualMachineScaleSetProperties{ VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ - OSProfile: &armcompute.VirtualMachineScaleSetOSProfile{}, + OSProfile: osProfile, }, }, } @@ -79,7 +81,7 @@ func TestConfigureConfidentialVM(t *testing.T) { require.NotNil(t, vmss.Properties) require.NotNil(t, vmss.Properties.VirtualMachineProfile) profile := vmss.Properties.VirtualMachineProfile - assert.Nil(t, profile.OSProfile) + assert.Same(t, osProfile, profile.OSProfile) require.NotNil(t, profile.SecurityProfile) require.NotNil(t, profile.SecurityProfile.SecurityType) assert.Equal(t, armcompute.SecurityTypesConfidentialVM, *profile.SecurityProfile.SecurityType) From 154eec91d10387952cefe5bae31694a754bfa463 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 14 Oct 2026 15:22:06 -0400 Subject: [PATCH 08/37] fix --- .pipelines/.vsts-vhd-builder-release.yaml | 8 ++-- e2e/config/vhd_test.go | 53 ----------------------- e2e/scenario/scenario_cvm_test.go | 13 +++--- 3 files changed, 11 insertions(+), 63 deletions(-) delete mode 100644 e2e/config/vhd_test.go diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index 5b42a709375..efd1abb1c65 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -370,7 +370,7 @@ stages: echo '##vso[task.setvariable variable=IMG_SKU]azure-linux-3-cvm' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC4ads_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC16ads_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]cvm' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' @@ -678,7 +678,7 @@ stages: echo '##vso[task.setvariable variable=IMG_SKU]cvm' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC4ads_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC16ads_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]cvm' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' @@ -753,7 +753,7 @@ stages: echo '##vso[task.setvariable variable=IMG_SKU]20_04-lts-cvm' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC4ads_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC16ads_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]cvm' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' @@ -853,7 +853,7 @@ stages: echo '##vso[task.setvariable variable=IMG_SKU]server-cvm' echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' - echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC4ads_v5' + echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_DC16ads_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]minimal,cvm' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' diff --git a/e2e/config/vhd_test.go b/e2e/config/vhd_test.go deleted file mode 100644 index 1ee0e34a9fb..00000000000 --- a/e2e/config/vhd_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package config - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestGetVHDResourceIDFromMetadataUsesLogicalImageName(t *testing.T) { - tests := []struct { - name string - image Image - metadataImageName string - }{ - { - name: "Ubuntu 2004 CVM", - image: *VHDUbuntu2004CVMGen2Containerd, - metadataImageName: "2004gen2CVMcontainerd", - }, - { - name: "Ubuntu 2404 CVM", - image: *VHDUbuntu2404CVMGen2Containerd, - metadataImageName: "2404gen2CVMcontainerd", - }, - { - name: "Ubuntu 2604 minimal CVM", - image: *VHDUbuntu2604MinimalCVMGen2Containerd, - metadataImageName: "2604minimalgen2CVMcontainerd", - }, - { - name: "Azure Linux V3 CVM", - image: *VHDAzureLinuxV3CVMGen2, - metadataImageName: "V3gen2CVM", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - expectedResourceID := VHDResourceID("/subscriptions/test/resourceGroups/test/providers/Microsoft.Compute/galleries/test/images/test/versions/1.0.0") - metadata := map[string]vhdMetadataEntry{ - tt.metadataImageName: { - ResourceID: expectedResourceID, - Regions: []string{"west europe"}, - }, - } - - resourceID, err := getVHDResourceIDFromMetadata(metadata, tt.image, "West Europe") - - require.NoError(t, err) - require.Equal(t, expectedResourceID, resourceID) - }) - } -} diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go index 95115c22dfb..1cb989c65db 100644 --- a/e2e/scenario/scenario_cvm_test.go +++ b/e2e/scenario/scenario_cvm_test.go @@ -45,21 +45,22 @@ func TestRegisteredCVMScenarios(t *testing.T) { "Ubuntu2604Minimal_CVM_TDX": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, useNVMe: true}, } - var found int + found := make(map[string]bool, len(expected)) for _, s := range List() { - if !s.Tags.CVM { + want, ok := expected[s.Name] + if !ok { continue } - found++ - want, ok := expected[s.Name] - require.True(t, ok, "unexpected CVM scenario %q", s.Name) + found[s.Name] = true assert.Equal(t, cvmLocation, s.Location) assert.Equal(t, want.vhd, s.VHD) assert.Equal(t, want.useNVMe, s.UseNVMe) assert.False(t, s.SkipDefaultValidation) assert.Nil(t, s.Validator) } - assert.Equal(t, len(expected), found) + for name := range expected { + assert.True(t, found[name], "CVM scenario %q is not registered", name) + } } func TestConfigureConfidentialVM(t *testing.T) { From 3adf3cfdc421c2b42ffa8172397b594a00e14b4f Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 14 Oct 2026 17:17:30 -0400 Subject: [PATCH 09/37] fix --- .pipelines/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/e2e.yaml b/.pipelines/e2e.yaml index c621ed1596c..d8244fbf39b 100644 --- a/.pipelines/e2e.yaml +++ b/.pipelines/e2e.yaml @@ -1,6 +1,6 @@ name: $(Date:yyyyMMdd)$(Rev:.r) variables: - TAGS_TO_SKIP: "os=windows,gpu=true" + TAGS_TO_SKIP: "os=windows,gpu=true,cvm=true" SKIP_E2E_TESTS: false E2E_FAILED_TESTS_RETRY_COUNT: 0 schedules: From a78abbf949fc0e9176845d05dae8595a750be49a Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Thu, 15 Oct 2026 09:45:37 -0400 Subject: [PATCH 10/37] fix --- e2e/scenario/scenario_cvm.go | 8 ++++---- e2e/scenario/scenario_cvm_test.go | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/e2e/scenario/scenario_cvm.go b/e2e/scenario/scenario_cvm.go index 45f7063436a..ef542f658b2 100644 --- a/e2e/scenario/scenario_cvm.go +++ b/e2e/scenario/scenario_cvm.go @@ -17,7 +17,7 @@ const ( var ( _ = Register(newCVMScenario( - "Ubuntu2004_CVM_SEVSNP", + "Ubuntu2004_CVM_SNP", "Tests that an Ubuntu 20.04 Confidential VM node can be provisioned and pass standard node validation", config.VHDUbuntu2004CVMGen2Containerd, ClusterKubenet, @@ -25,7 +25,7 @@ var ( false, )) _ = Register(newCVMScenario( - "Ubuntu2404_CVM_SEVSNP", + "Ubuntu2404_CVM_SNP", "Tests that an Ubuntu 24.04 Confidential VM node can be provisioned and pass standard node validation", config.VHDUbuntu2404CVMGen2Containerd, ClusterKubenet, @@ -33,7 +33,7 @@ var ( false, )) _ = Register(newCVMScenario( - "AzureLinuxV3_CVM_SEVSNP", + "AzureLinuxV3_CVM_SNP", "Tests that an Azure Linux V3 Confidential VM node can be provisioned and pass standard node validation", config.VHDAzureLinuxV3CVMGen2, ClusterKubenet, @@ -41,7 +41,7 @@ var ( false, )) _ = Register(newCVMScenario( - "Ubuntu2604Minimal_CVM_SEVSNP", + "Ubuntu2604Minimal_CVM_SNP", "Tests that an Ubuntu 26.04 minimal Confidential VM node can be provisioned on SEV-SNP and pass standard node validation", config.VHDUbuntu2604MinimalCVMGen2Containerd, ClusterLatestKubernetesVersionKubenet, diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go index 1cb989c65db..4bc440fb1ef 100644 --- a/e2e/scenario/scenario_cvm_test.go +++ b/e2e/scenario/scenario_cvm_test.go @@ -38,11 +38,11 @@ func TestRegisteredCVMScenarios(t *testing.T) { vhd *config.Image useNVMe bool }{ - "Ubuntu2004_CVM_SEVSNP": {vhd: config.VHDUbuntu2004CVMGen2Containerd}, - "Ubuntu2404_CVM_SEVSNP": {vhd: config.VHDUbuntu2404CVMGen2Containerd}, - "AzureLinuxV3_CVM_SEVSNP": {vhd: config.VHDAzureLinuxV3CVMGen2}, - "Ubuntu2604Minimal_CVM_SEVSNP": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd}, - "Ubuntu2604Minimal_CVM_TDX": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, useNVMe: true}, + "Ubuntu2004_CVM_SNP": {vhd: config.VHDUbuntu2004CVMGen2Containerd}, + "Ubuntu2404_CVM_SNP": {vhd: config.VHDUbuntu2404CVMGen2Containerd}, + "AzureLinuxV3_CVM_SNP": {vhd: config.VHDAzureLinuxV3CVMGen2}, + "Ubuntu2604Minimal_CVM_SNP": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd}, + "Ubuntu2604Minimal_CVM_TDX": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, useNVMe: true}, } found := make(map[string]bool, len(expected)) From 2440c4a421fe4df07091f2688ddad86d7e5b3f6e Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Thu, 15 Oct 2026 17:01:17 -0400 Subject: [PATCH 11/37] fix --- .../templates/.builder-release-template.yaml | 15 ++ .../publish_generalized_cvm_if_needed_spec.sh | 171 ++++++++++++++ .../publish-generalized-cvm-if-needed.sh | 219 ++++++++++++++++++ vhdbuilder/prefetch/scripts/optimize.sh | 3 + 4 files changed, 408 insertions(+) create mode 100644 spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh create mode 100755 vhdbuilder/packer/publish-generalized-cvm-if-needed.sh diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 682a81b5bf3..47b9fc9954c 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -335,15 +335,19 @@ steps: env: SUBSCRIPTION_ID: $(SUBSCRIPTION_ID) LOCATION: $(PACKER_BUILD_LOCATION) + PACKER_BUILD_LOCATION: $(PACKER_BUILD_LOCATION) SIG_GALLERY_RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) + AZURE_RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) SIG_GALLERY_NAME: $(SIG_GALLERY_NAME) SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) SKU_NAME: $(SKU_NAME) + OS_SKU: $(OS_SKU) STORAGE_ACCOUNT_BLOB_URL: $(CLASSIC_BLOB) STORAGE_ACCOUNT_BLOB_URL_STAGING: $(CLASSIC_BLOB_STAGING) VHD_NAME: $(VHD_NAME) IMAGE_BUILDER_IDENTITY_ID: $(AZURE_MSI_RESOURCE_STRING) BUILD_RUN_NUMBER: $(Build.BuildNumber) + BUILD_BUILDID: $(Build.BuildId) CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) HYPERV_GENERATION: $(HYPERV_GENERATION) FEATURE_FLAGS: $(FEATURE_FLAGS) @@ -453,6 +457,8 @@ steps: export AZCOPY_CONCURRENCY_VALUE="AUTO" az storage blob copy start --account-name "$STORAGE_ACCOUNT_NAME" --destination-blob "${CAPTURED_SIG_VERSION}.vhd" --destination-container "$VHD_CONTAINER_NAME" --source-uri "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --auth-mode login || exit 1 echo "Successfully copied to immutable container" + ./vhdbuilder/packer/publish-generalized-cvm-if-needed.sh \ + "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" || exit 1 # Remove old VHD from staging container after copy to immutable container is complete azcopy remove "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true condition: | @@ -466,6 +472,15 @@ steps: ) ) displayName: Copy to Immutable Storage Container + env: + OS_SKU: $(OS_SKU) + FEATURE_FLAGS: $(FEATURE_FLAGS) + AZURE_RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) + SIG_GALLERY_NAME: $(SIG_GALLERY_NAME) + SKU_NAME: $(SKU_NAME) + CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) + PACKER_BUILD_LOCATION: $(PACKER_BUILD_LOCATION) + BUILD_BUILDID: $(Build.BuildId) - task: DownloadPipelineArtifact@2 displayName: Download Build Performance Program diff --git a/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh b/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh new file mode 100644 index 00000000000..768bc7dbbd8 --- /dev/null +++ b/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh @@ -0,0 +1,171 @@ +#!/bin/bash + +Describe 'publish-generalized-cvm-if-needed' + SCRIPT="./vhdbuilder/packer/publish-generalized-cvm-if-needed.sh" + ORIGINAL_PATH="${PATH}" + + setup() { + TEST_ROOT="$(mktemp -d)" + AZ_CALLS="${TEST_ROOT}/az-calls" + AZ_VERSION_STATE_FILE="${TEST_ROOT}/version-state" + touch "${AZ_CALLS}" + + export AZ_CALLS + export AZ_VERSION_STATE_FILE + export MOCK_DEFINITION_EXISTS=false + export MOCK_CREATED_VERSION_STATE=Succeeded + + export OS_SKU=Ubuntu + export FEATURE_FLAGS=cvm + export AZURE_RESOURCE_GROUP_NAME=resource-group + export SIG_GALLERY_NAME=gallery + export SKU_NAME=2404gen2CVMcontainerd + export CAPTURED_SIG_VERSION=202610.15.0 + export PACKER_BUILD_LOCATION=eastus + export BUILD_BUILDID=12345 + + cat >"${TEST_ROOT}/az" <<'EOF' +#!/usr/bin/env bash + +printf '<%s>\n' "$*" >>"${AZ_CALLS}" + +case "$1 $2" in + "sig image-definition") + case "$3" in + show) + if [[ "$*" == *"--query id"* ]]; then + echo "/subscriptions/test/resourceGroups/resource-group/providers/Microsoft.Compute/galleries/gallery/images/2404gen2CVMcontainerd" + exit 0 + fi + if [[ "${MOCK_DEFINITION_EXISTS}" == "true" ]]; then + echo '{}' + exit 0 + fi + exit 1 + ;; + create) + exit 0 + ;; + esac + ;; + "sig image-version") + case "$3" in + show) + if [[ -s "${AZ_VERSION_STATE_FILE}" ]]; then + cat "${AZ_VERSION_STATE_FILE}" + exit 0 + fi + exit 1 + ;; + create) + printf '%s' "${MOCK_CREATED_VERSION_STATE}" >"${AZ_VERSION_STATE_FILE}" + exit 0 + ;; + delete) + rm -f "${AZ_VERSION_STATE_FILE}" + exit 0 + ;; + wait) + exit 0 + ;; + esac + ;; + "image create") + echo "/subscriptions/test/resourceGroups/resource-group/providers/Microsoft.Compute/images/temporary-image" + exit 0 + ;; + "image show") + exit 1 + ;; + "image delete"|"resource update") + exit 0 + ;; +esac + +echo "Unexpected az invocation: $*" >&2 +exit 1 +EOF + + cat >"${TEST_ROOT}/sleep" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + + chmod +x "${TEST_ROOT}/az" "${TEST_ROOT}/sleep" + export PATH="${TEST_ROOT}:${ORIGINAL_PATH}" + } + + cleanup() { + PATH="${ORIGINAL_PATH}" + rm -rf "${TEST_ROOT}" + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + It 'does nothing for a non-CVM build' + FEATURE_FLAGS="" + + When run "${SCRIPT}" "https://storage.example/vhd.vhd" + The status should be success + The output should include "Generalized CVM publication is not required" + The contents of file "${AZ_CALLS}" should equal "" + End + + It 'does nothing for Azure Container Linux CVM' + OS_SKU=AzureContainerLinux + + When run "${SCRIPT}" "https://storage.example/vhd.vhd" + The status should be success + The output should include "Generalized CVM publication is not required" + The contents of file "${AZ_CALLS}" should equal "" + End + + It 'publishes an Ubuntu CVM staging VHD' + When run "${SCRIPT}" "https://storage.example/ubuntu.vhd" + The status should be success + The output should include "provisioned successfully" + The contents of file "${AZ_CALLS}" should include ""${AZ_VERSION_STATE_FILE}" + + When run "${SCRIPT}" "https://storage.example/ubuntu.vhd" + The status should be success + The output should include "is already provisioned" + The contents of file "${AZ_CALLS}" should not include ""${AZ_VERSION_STATE_FILE}" + + When run "${SCRIPT}" "https://storage.example/ubuntu.vhd" + The status should be success + The output should include "Deleting failed image version" + The contents of file "${AZ_CALLS}" should include "&2 + exit 1 + fi +done + +ensure_generalized_image_definition() { + if az sig image-definition show \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" \ + --only-show-errors >/dev/null 2>&1; then + return + fi + + az sig image-definition create \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" \ + --publisher microsoft-aks \ + --offer "${SIG_GALLERY_NAME}" \ + --sku "${SKU_NAME}" \ + --os-type Linux \ + --os-state Generalized \ + --architecture x64 \ + --hyper-v-generation V2 \ + --location "${PACKER_BUILD_LOCATION}" \ + --features \ + "DiskControllerTypes=SCSI,NVMe" \ + "SecurityType=ConfidentialVMSupported" \ + --only-show-errors >/dev/null + + az resource update \ + --ids "$( + az sig image-definition show \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" \ + --query id \ + --output tsv \ + --only-show-errors + )" \ + --set properties.allowUpdateImage=true \ + --only-show-errors >/dev/null +} + +wait_for_successful_provisioning() { + local provisioning_state + + echo "Waiting five minutes before checking image version provisioning" + sleep 300 + + while true; do + if provisioning_state="$( + az sig image-version show \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" \ + --gallery-image-version "${CAPTURED_SIG_VERSION}" \ + --query provisioningState \ + --output tsv \ + --only-show-errors 2>/dev/null + )"; then + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioning state: ${provisioning_state:-unknown}" + + case "${provisioning_state,,}" in + succeeded) + return 0 + ;; + failed) + return 1 + ;; + esac + else + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} does not exist yet" + fi + + sleep 60 + done +} + +delete_failed_image_version() { + echo "Deleting failed image version ${SKU_NAME}/${CAPTURED_SIG_VERSION}" + + az sig image-version delete \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" \ + --gallery-image-version "${CAPTURED_SIG_VERSION}" \ + --only-show-errors + + az sig image-version wait \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" \ + --gallery-image-version "${CAPTURED_SIG_VERSION}" \ + --deleted \ + --only-show-errors +} + +cleanup_managed_image() { + if az image show \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --name "${SKU_NAME}-${CAPTURED_SIG_VERSION}-generalized-${BUILD_BUILDID}" \ + --only-show-errors >/dev/null 2>&1; then + echo "Deleting temporary managed image" + + az image delete \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --name "${SKU_NAME}-${CAPTURED_SIG_VERSION}-generalized-${BUILD_BUILDID}" \ + --only-show-errors + fi +} + +ensure_generalized_image_definition + +if provisioning_state="$( + az sig image-version show \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" \ + --gallery-image-version "${CAPTURED_SIG_VERSION}" \ + --query provisioningState \ + --output tsv \ + --only-show-errors 2>/dev/null +)"; then + case "${provisioning_state,,}" in + succeeded) + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} is already provisioned" + exit 0 + ;; + failed) + delete_failed_image_version + ;; + *) + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} already exists and is provisioning" + + if wait_for_successful_provisioning; then + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioned successfully" + exit 0 + fi + + delete_failed_image_version + ;; + esac +fi + +managed_image_id="$( + az image create \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --name "${SKU_NAME}-${CAPTURED_SIG_VERSION}-generalized-${BUILD_BUILDID}" \ + --location "${PACKER_BUILD_LOCATION}" \ + --source "${STAGING_VHD_URI}" \ + --os-type Linux \ + --hyper-v-generation V2 \ + --storage-sku Standard_LRS \ + --tags \ + "sourceBuildId=${BUILD_BUILDID}" \ + "createdBy=aks-vhd-pipeline" \ + --query id \ + --output tsv \ + --only-show-errors +)" + +trap cleanup_managed_image EXIT + +az sig image-version create \ + --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ + --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" \ + --gallery-image-version "${CAPTURED_SIG_VERSION}" \ + --managed-image "${managed_image_id}" \ + --target-regions eastus westeurope \ + --tags \ + "sourceBuildId=${BUILD_BUILDID}" \ + "createdBy=aks-vhd-pipeline" \ + --only-show-errors >/dev/null + +if ! wait_for_successful_provisioning; then + delete_failed_image_version + echo "Generalized CVM image version failed to provision" >&2 + exit 1 +fi + +echo "Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioned successfully" diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index f7ca0890757..3d78ac6986d 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -356,6 +356,9 @@ copy_optimized_vhd() { wait_for_vhd_copy || return $? + "$(dirname "${BASH_SOURCE[0]}")/../../packer/publish-generalized-cvm-if-needed.sh" \ + "${STAGING_VHD_URI}" || return $? + # the optimized VHD is now safely in the immutable vhd container, so the staging copy is no longer # needed - delete it as best-effort cleanup so staging blobs do not accumulate in the destination account. # Use no-wait since we don't need to wait for the blob to be deleted in this case. From bb35c1d7a76b24b6b3b2c5753ac951979b1fd431 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Thu, 15 Oct 2026 17:18:00 -0400 Subject: [PATCH 12/37] fix --- .../packer/publish_generalized_cvm_if_needed_spec.sh | 2 ++ vhdbuilder/packer/publish-generalized-cvm-if-needed.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh b/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh index 768bc7dbbd8..e3a7a728b3e 100644 --- a/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh +++ b/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh @@ -132,6 +132,8 @@ EOF The contents of file "${AZ_CALLS}" should include "--source https://storage.example/ubuntu.vhd" The contents of file "${AZ_CALLS}" should include "/dev/null From 58e210e129f4216d654aacfd160af10af71ecdef Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Thu, 15 Oct 2026 17:33:20 -0400 Subject: [PATCH 13/37] fix --- e2e/config/vhd.go | 10 +++++++ e2e/scenario/scenario_cvm.go | 25 ++++++++++++++-- e2e/scenario/scenario_cvm_test.go | 48 +++++++++++++++++++++++++------ 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 4b54fc2b4fc..43c3bd311bd 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -229,6 +229,16 @@ var ( OSDiskSizeGB: 60, } + VHDACLCVMGen2 = &Image{ + Name: "aclgen2CVM", + OS: OSACL, + Arch: "amd64", + Distro: datamodel.AKSACLCVMGen2, + Gallery: &Config.GalleryLinux, + Flatcar: true, + OSDiskSizeGB: 60, + } + VHDACLArm64Gen2TL = &Image{ Name: "aclgen2arm64TL", OS: OSACL, diff --git a/e2e/scenario/scenario_cvm.go b/e2e/scenario/scenario_cvm.go index ef542f658b2..f281f098279 100644 --- a/e2e/scenario/scenario_cvm.go +++ b/e2e/scenario/scenario_cvm.go @@ -23,6 +23,7 @@ var ( ClusterKubenet, cvmSNPSKU, false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, )) _ = Register(newCVMScenario( "Ubuntu2404_CVM_SNP", @@ -31,6 +32,7 @@ var ( ClusterKubenet, cvmSNPSKU, false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, )) _ = Register(newCVMScenario( "AzureLinuxV3_CVM_SNP", @@ -39,6 +41,16 @@ var ( ClusterKubenet, cvmSNPSKU, false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, + )) + _ = Register(newCVMScenario( + "AzureContainerLinux_CVM_SNP", + "Tests that an Azure Container Linux Confidential VM node can be provisioned and pass standard node validation", + config.VHDACLCVMGen2, + ClusterKubenet, + cvmSNPSKU, + false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, )) _ = Register(newCVMScenario( "Ubuntu2604Minimal_CVM_SNP", @@ -47,6 +59,7 @@ var ( ClusterLatestKubernetesVersionKubenet, cvmSNPSKU, false, + armcompute.SecurityEncryptionTypesDiskWithVMGuestState, )) _ = Register(newCVMScenario( "Ubuntu2604Minimal_CVM_TDX", @@ -55,6 +68,7 @@ var ( ClusterLatestKubernetesVersionKubenet, cvmTDXSKU, true, + armcompute.SecurityEncryptionTypesDiskWithVMGuestState, )) ) @@ -65,6 +79,7 @@ func newCVMScenario( cluster func(context.Context, ClusterRequest) (*Cluster, error), vmSize string, useNVMe bool, + securityEncryptionType armcompute.SecurityEncryptionTypes, ) *Scenario { return &Scenario{ Name: name, @@ -81,13 +96,17 @@ func newCVMScenario( nbc.AgentPoolProfile.VMSize = vmSize }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - configureConfidentialVM(vmss, vmSize) + configureConfidentialVM(vmss, vmSize, securityEncryptionType) }, }, } } -func configureConfidentialVM(vmss *armcompute.VirtualMachineScaleSet, vmSize string) { +func configureConfidentialVM( + vmss *armcompute.VirtualMachineScaleSet, + vmSize string, + securityEncryptionType armcompute.SecurityEncryptionTypes, +) { if vmss.SKU == nil { vmss.SKU = &armcompute.SKU{} } @@ -124,5 +143,5 @@ func configureConfidentialVM(vmss *armcompute.VirtualMachineScaleSet, vmSize str profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile = &armcompute.VMDiskSecurityProfile{} } profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType = - to.Ptr(armcompute.SecurityEncryptionTypesVMGuestStateOnly) + to.Ptr(securityEncryptionType) } diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go index 4bc440fb1ef..dafdc7bcdec 100644 --- a/e2e/scenario/scenario_cvm_test.go +++ b/e2e/scenario/scenario_cvm_test.go @@ -18,6 +18,7 @@ func TestNewCVMScenarioUsesWestEuropeAndDefaultValidation(t *testing.T) { ClusterKubenet, cvmSNPSKU, false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, ) assert.Equal(t, cvmLocation, s.Location) @@ -35,14 +36,35 @@ func TestNewCVMScenarioUsesWestEuropeAndDefaultValidation(t *testing.T) { func TestRegisteredCVMScenarios(t *testing.T) { expected := map[string]struct { - vhd *config.Image - useNVMe bool + vhd *config.Image + useNVMe bool + securityEncryptionType armcompute.SecurityEncryptionTypes }{ - "Ubuntu2004_CVM_SNP": {vhd: config.VHDUbuntu2004CVMGen2Containerd}, - "Ubuntu2404_CVM_SNP": {vhd: config.VHDUbuntu2404CVMGen2Containerd}, - "AzureLinuxV3_CVM_SNP": {vhd: config.VHDAzureLinuxV3CVMGen2}, - "Ubuntu2604Minimal_CVM_SNP": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd}, - "Ubuntu2604Minimal_CVM_TDX": {vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, useNVMe: true}, + "Ubuntu2004_CVM_SNP": { + vhd: config.VHDUbuntu2004CVMGen2Containerd, + securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, + }, + "Ubuntu2404_CVM_SNP": { + vhd: config.VHDUbuntu2404CVMGen2Containerd, + securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, + }, + "AzureLinuxV3_CVM_SNP": { + vhd: config.VHDAzureLinuxV3CVMGen2, + securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, + }, + "AzureContainerLinux_CVM_SNP": { + vhd: config.VHDACLCVMGen2, + securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, + }, + "Ubuntu2604Minimal_CVM_SNP": { + vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, + securityEncryptionType: armcompute.SecurityEncryptionTypesDiskWithVMGuestState, + }, + "Ubuntu2604Minimal_CVM_TDX": { + vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, + useNVMe: true, + securityEncryptionType: armcompute.SecurityEncryptionTypesDiskWithVMGuestState, + }, } found := make(map[string]bool, len(expected)) @@ -57,6 +79,14 @@ func TestRegisteredCVMScenarios(t *testing.T) { assert.Equal(t, want.useNVMe, s.UseNVMe) assert.False(t, s.SkipDefaultValidation) assert.Nil(t, s.Validator) + + vmss := &armcompute.VirtualMachineScaleSet{} + s.VMConfigMutator(vmss) + assert.Equal( + t, + want.securityEncryptionType, + *vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType, + ) } for name := range expected { assert.True(t, found[name], "CVM scenario %q is not registered", name) @@ -73,7 +103,7 @@ func TestConfigureConfidentialVM(t *testing.T) { }, } - configureConfidentialVM(vmss, cvmTDXSKU) + configureConfidentialVM(vmss, cvmTDXSKU, armcompute.SecurityEncryptionTypesDiskWithVMGuestState) require.NotNil(t, vmss.SKU) require.NotNil(t, vmss.SKU.Name) @@ -101,7 +131,7 @@ func TestConfigureConfidentialVM(t *testing.T) { require.NotNil(t, profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType) assert.Equal( t, - armcompute.SecurityEncryptionTypesVMGuestStateOnly, + armcompute.SecurityEncryptionTypesDiskWithVMGuestState, *profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType, ) } From eff563ca1c666c2c6c120ee85e368458b4607fdf Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Thu, 15 Oct 2026 17:38:40 -0400 Subject: [PATCH 14/37] fix --- .../templates/.builder-release-template.yaml | 2 +- .../publish_generalized_cvm_if_needed_spec.sh | 173 ------------------ ...=> publish-generalized-cvm-if-required.sh} | 0 vhdbuilder/prefetch/scripts/optimize.sh | 2 +- 4 files changed, 2 insertions(+), 175 deletions(-) delete mode 100644 spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh rename vhdbuilder/packer/{publish-generalized-cvm-if-needed.sh => publish-generalized-cvm-if-required.sh} (100%) diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 47b9fc9954c..5dc724a584d 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -457,7 +457,7 @@ steps: export AZCOPY_CONCURRENCY_VALUE="AUTO" az storage blob copy start --account-name "$STORAGE_ACCOUNT_NAME" --destination-blob "${CAPTURED_SIG_VERSION}.vhd" --destination-container "$VHD_CONTAINER_NAME" --source-uri "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --auth-mode login || exit 1 echo "Successfully copied to immutable container" - ./vhdbuilder/packer/publish-generalized-cvm-if-needed.sh \ + ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" || exit 1 # Remove old VHD from staging container after copy to immutable container is complete azcopy remove "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true diff --git a/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh b/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh deleted file mode 100644 index e3a7a728b3e..00000000000 --- a/spec/vhdbuilder/packer/publish_generalized_cvm_if_needed_spec.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash - -Describe 'publish-generalized-cvm-if-needed' - SCRIPT="./vhdbuilder/packer/publish-generalized-cvm-if-needed.sh" - ORIGINAL_PATH="${PATH}" - - setup() { - TEST_ROOT="$(mktemp -d)" - AZ_CALLS="${TEST_ROOT}/az-calls" - AZ_VERSION_STATE_FILE="${TEST_ROOT}/version-state" - touch "${AZ_CALLS}" - - export AZ_CALLS - export AZ_VERSION_STATE_FILE - export MOCK_DEFINITION_EXISTS=false - export MOCK_CREATED_VERSION_STATE=Succeeded - - export OS_SKU=Ubuntu - export FEATURE_FLAGS=cvm - export AZURE_RESOURCE_GROUP_NAME=resource-group - export SIG_GALLERY_NAME=gallery - export SKU_NAME=2404gen2CVMcontainerd - export CAPTURED_SIG_VERSION=202610.15.0 - export PACKER_BUILD_LOCATION=eastus - export BUILD_BUILDID=12345 - - cat >"${TEST_ROOT}/az" <<'EOF' -#!/usr/bin/env bash - -printf '<%s>\n' "$*" >>"${AZ_CALLS}" - -case "$1 $2" in - "sig image-definition") - case "$3" in - show) - if [[ "$*" == *"--query id"* ]]; then - echo "/subscriptions/test/resourceGroups/resource-group/providers/Microsoft.Compute/galleries/gallery/images/2404gen2CVMcontainerd" - exit 0 - fi - if [[ "${MOCK_DEFINITION_EXISTS}" == "true" ]]; then - echo '{}' - exit 0 - fi - exit 1 - ;; - create) - exit 0 - ;; - esac - ;; - "sig image-version") - case "$3" in - show) - if [[ -s "${AZ_VERSION_STATE_FILE}" ]]; then - cat "${AZ_VERSION_STATE_FILE}" - exit 0 - fi - exit 1 - ;; - create) - printf '%s' "${MOCK_CREATED_VERSION_STATE}" >"${AZ_VERSION_STATE_FILE}" - exit 0 - ;; - delete) - rm -f "${AZ_VERSION_STATE_FILE}" - exit 0 - ;; - wait) - exit 0 - ;; - esac - ;; - "image create") - echo "/subscriptions/test/resourceGroups/resource-group/providers/Microsoft.Compute/images/temporary-image" - exit 0 - ;; - "image show") - exit 1 - ;; - "image delete"|"resource update") - exit 0 - ;; -esac - -echo "Unexpected az invocation: $*" >&2 -exit 1 -EOF - - cat >"${TEST_ROOT}/sleep" <<'EOF' -#!/usr/bin/env bash -exit 0 -EOF - - chmod +x "${TEST_ROOT}/az" "${TEST_ROOT}/sleep" - export PATH="${TEST_ROOT}:${ORIGINAL_PATH}" - } - - cleanup() { - PATH="${ORIGINAL_PATH}" - rm -rf "${TEST_ROOT}" - } - - BeforeEach 'setup' - AfterEach 'cleanup' - - It 'does nothing for a non-CVM build' - FEATURE_FLAGS="" - - When run "${SCRIPT}" "https://storage.example/vhd.vhd" - The status should be success - The output should include "Generalized CVM publication is not required" - The contents of file "${AZ_CALLS}" should equal "" - End - - It 'does nothing for Azure Container Linux CVM' - OS_SKU=AzureContainerLinux - - When run "${SCRIPT}" "https://storage.example/vhd.vhd" - The status should be success - The output should include "Generalized CVM publication is not required" - The contents of file "${AZ_CALLS}" should equal "" - End - - It 'publishes an Ubuntu CVM staging VHD' - When run "${SCRIPT}" "https://storage.example/ubuntu.vhd" - The status should be success - The output should include "provisioned successfully" - The contents of file "${AZ_CALLS}" should include ""${AZ_VERSION_STATE_FILE}" - - When run "${SCRIPT}" "https://storage.example/ubuntu.vhd" - The status should be success - The output should include "is already provisioned" - The contents of file "${AZ_CALLS}" should not include ""${AZ_VERSION_STATE_FILE}" - - When run "${SCRIPT}" "https://storage.example/ubuntu.vhd" - The status should be success - The output should include "Deleting failed image version" - The contents of file "${AZ_CALLS}" should include " Date: Fri, 16 Oct 2026 13:20:24 -0400 Subject: [PATCH 15/37] fix --- .../templates/.builder-release-template.yaml | 2 + spec/vhdbuilder/prefetch/publication_spec.sh | 388 ++++++++++++++++++ .../publish-generalized-cvm-if-required.sh | 149 +++---- vhdbuilder/prefetch/README.md | 17 +- vhdbuilder/prefetch/scripts/cleanup.sh | 14 +- vhdbuilder/prefetch/scripts/optimize.sh | 96 +++-- 6 files changed, 561 insertions(+), 105 deletions(-) create mode 100644 spec/vhdbuilder/prefetch/publication_spec.sh diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 5e27f39dbd5..f8fa6823305 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -382,6 +382,7 @@ steps: condition: | and( succeeded(), + eq(variables.PREFETCH_DELIVERY_COMPLETE, 'True'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables.RUN_PREFETCH_OPTIMIZATION, 'True'), eq(variables.PREFETCH_COMPATIBLE, 'True') @@ -390,6 +391,7 @@ steps: env: BUILD_RUN_NUMBER: $(Build.BuildNumber) CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) + PREFETCH_DELIVERY_COMPLETE: $(PREFETCH_DELIVERY_COMPLETE) - task: PublishPipelineArtifact@0 condition: always() diff --git a/spec/vhdbuilder/prefetch/publication_spec.sh b/spec/vhdbuilder/prefetch/publication_spec.sh new file mode 100644 index 00000000000..4510f32cc6e --- /dev/null +++ b/spec/vhdbuilder/prefetch/publication_spec.sh @@ -0,0 +1,388 @@ +#!/bin/bash +# shellcheck disable=SC2329 + +Describe 'Post-prefetch CVM publication' + setup_environment() { + TEST_ROOT="$(mktemp -d "${SHELLSPEC_TMPBASE}/publication.XXXXXX")" + export TEST_ROOT + export AZ_MOCK_LOG="${TEST_ROOT}/az.log" + : > "${AZ_MOCK_LOG}" + export SUBSCRIPTION_ID=test-subscription LOCATION=westus PACKER_BUILD_LOCATION=westus + export SIG_GALLERY_RESOURCE_GROUP_NAME=gallery-rg AZURE_RESOURCE_GROUP_NAME=gallery-rg + export SIG_GALLERY_NAME=gallery SIG_IMAGE_NAME=specialized SKU_NAME=generalized + export STORAGE_ACCOUNT_BLOB_URL=https://destination.blob.core.windows.net/vhd + export STORAGE_ACCOUNT_BLOB_URL_STAGING=https://destination.blob.core.windows.net/vhdstaging + export VHD_NAME=test.vhd IMAGE_BUILDER_IDENTITY_ID=identity BUILD_RUN_NUMBER=run + export CAPTURED_SIG_VERSION=1.2.3 HYPERV_GENERATION=V2 FEATURE_FLAGS=cvm + export ENABLE_TRUSTED_LAUNCH=False WORKLOAD_OPTIMIZATION_SCRIPT_URL=https://example.test/optimize.sh + export OS_SKU=Ubuntu BUILD_BUILDID=123 + export SOURCE_URI=https://sourceaccount.blob.core.windows.net/vhds/original.vhd + export MOCK_SOURCE_LOCATION=westus MOCK_STORAGE_ACCOUNT_MISSING=false + export MOCK_FINAL_EXISTS=false MOCK_STAGING_COMPLETE=false MOCK_VERSION_STATE=missing + export MOCK_STORAGE_FAILURE=false MOCK_GALLERY_FAILURE=false MOCK_REQUIRE_OVERLAP=false + export MOCK_GALLERY_WAIT_FOR_STORAGE=false MOCK_REPLICATION_PENDING_ONCE=false + export MOCK_ARTIFACT_MISSING=false MOCK_TEMPLATE_DELETE_FAILURE=false + export MOCK_DEFINITION_EXISTS=true MOCK_VERSION_READ_FAILURE=false + export MOCK_REPLICATION_FAILURE=false + export MOCK_COPY_PENDING_ONCE=false MOCK_COPY_STATUS=success + export PREFETCH_DELIVERY_COMPLETE=False + + wait_for_marker() { + local attempt + for ((attempt=0; attempt<200; attempt++)); do + [ -f "${TEST_ROOT}/$1" ] && return 0 + command sleep 0.01 + done + echo "Timed out waiting for mocked operation: $1" >&2 + return 90 + } + + az() { + printf '%s\n' "$*" >> "${AZ_MOCK_LOG}" + case "$1 $2 $3" in + 'sig image-definition show') + if [ "${MOCK_DEFINITION_EXISTS}" = false ] && [ ! -f "${TEST_ROOT}/definition-created" ]; then + return 3 + fi + echo definition + ;; + 'sig image-definition create') + touch "${TEST_ROOT}/definition-created" + ;; + 'resource update --ids') + ;; + 'sig image-version show') + if [ "${MOCK_VERSION_READ_FAILURE}" = true ]; then + echo '(AuthorizationFailed) Access denied' >&2 + return 4 + fi + if [ "${MOCK_VERSION_STATE}" = succeeded ] || [ -f "${TEST_ROOT}/version-created" ]; then + if [ "${MOCK_REPLICATION_FAILURE}" = true ]; then + echo '{"provisioningState":"Succeeded","replicationStatus":{"aggregatedState":"Failed"}}' + elif [ "${MOCK_REPLICATION_PENDING_ONCE}" = true ] && [ ! -f "${TEST_ROOT}/replication-ready" ]; then + echo '{"provisioningState":"Succeeded","replicationStatus":{"aggregatedState":"InProgress"}}' + else + echo '{"provisioningState":"Succeeded","replicationStatus":{"aggregatedState":"Completed"}}' + fi + elif [ "${MOCK_VERSION_STATE}" = failed ]; then + echo '{"provisioningState":"Failed","replicationStatus":{"aggregatedState":"Failed"}}' + else + echo '(ResourceNotFound) Image version not found' >&2 + return 3 + fi + ;; + 'sig image-version create') + touch "${TEST_ROOT}/gallery-started" + if [ "${MOCK_REQUIRE_OVERLAP}" = true ]; then + wait_for_marker storage-started || return $? + fi + if [ "${MOCK_GALLERY_WAIT_FOR_STORAGE}" = true ]; then + wait_for_marker storage-finished || return $? + fi + touch "${TEST_ROOT}/gallery-finished" + [ "${MOCK_GALLERY_FAILURE}" = true ] && return 7 + touch "${TEST_ROOT}/version-created" + ;; + 'sig image-version delete'|'sig image-version wait') + ;; + 'storage account list') + if [ "${MOCK_STORAGE_ACCOUNT_MISSING}" = true ]; then + echo null + else + printf '{"id":"/subscriptions/test-subscription/resourceGroups/IT_staging/providers/Microsoft.Storage/storageAccounts/sourceaccount","location":"%s"}\n' "${MOCK_SOURCE_LOCATION}" + fi + ;; + 'storage blob exists') + printf '{"exists":%s}\n' "${MOCK_FINAL_EXISTS}" + ;; + 'storage blob show') + if [[ "$*" = *'/vhdstaging/'* ]]; then + if [ "${MOCK_STAGING_COMPLETE}" = true ]; then + echo '{"name":"test.vhd","metadata":{"prefetched":"true"}}' + else + echo '{}' + fi + else + if [ "${MOCK_COPY_PENDING_ONCE}" = true ] && [ ! -f "${TEST_ROOT}/copy-pending-checked" ]; then + touch "${TEST_ROOT}/copy-pending-checked" + echo '{"properties":{"copy":{"status":"pending"}}}' + else + touch "${TEST_ROOT}/storage-finished" + printf '{"properties":{"copy":{"status":"%s"}}}\n' "${MOCK_COPY_STATUS}" + fi + fi + ;; + 'storage blob metadata'|'storage blob copy') + ;; + 'storage blob delete') + if [ "${MOCK_VERSION_STATE}" != succeeded ] && [ "${FEATURE_FLAGS}" = cvm ]; then + [ -f "${TEST_ROOT}/gallery-finished" ] || return 91 + fi + ;; + 'resource show --ids') + if [ "${MOCK_ARTIFACT_MISSING}" = true ]; then + echo null + else + echo "${SOURCE_URI}" + fi + ;; + 'group exists -g') + echo true + ;; + 'image builder show') + echo '{"provisioningState":"Succeeded","lastRunStatus":{"runState":"Succeeded"}}' + ;; + 'image builder wait') + ;; + 'image builder delete') + [ "${MOCK_TEMPLATE_DELETE_FAILURE}" = true ] && return 8 + touch "${TEST_ROOT}/template-deleted" + return 0 + ;; + 'group delete -g') + [ -f "${TEST_ROOT}/template-deleted" ] || return 92 + ;; + *) + echo "Unexpected Azure call: $*" >&2 + return 99 + ;; + esac + } + + azcopy() { + printf 'azcopy %s\n' "$*" >> "${AZ_MOCK_LOG}" + touch "${TEST_ROOT}/storage-started" + if [ "${MOCK_REQUIRE_OVERLAP}" = true ]; then + wait_for_marker gallery-started || return $? + fi + if [ "${MOCK_STORAGE_FAILURE}" = true ]; then + touch "${TEST_ROOT}/storage-finished" + return 6 + fi + } + + sleep() { + printf 'sleep %s\n' "$*" >> "${AZ_MOCK_LOG}" + touch "${TEST_ROOT}/replication-ready" + } + + export -f az azcopy sleep wait_for_marker + } + BeforeEach 'setup_environment' + + run_script() { + local status=0 + bash "$@" 2>&1 || status=$? + cat "${AZ_MOCK_LOG}" + return "${status}" + } + + Describe 'Direct VHD gallery versions' + It 'uses the original VHD, account ID and actual account region without a managed image' + MOCK_SOURCE_LOCATION=northeurope + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be success + The output should include "--os-vhd-uri ${SOURCE_URI}" + The output should include '--os-vhd-storage-account /subscriptions/test-subscription/resourceGroups/IT_staging/providers/Microsoft.Storage/storageAccounts/sourceaccount' + The output should include '--location northeurope' + The output should include '--target-regions eastus westeurope northeurope' + The output should include 'buildId=123' + The output should not include '--managed-image' + The output should not include 'image create' + The output should not include 'image delete' + End + + It 'does not duplicate the source region when it is already a replication target' + MOCK_SOURCE_LOCATION=eastus + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be success + The output should include '--target-regions eastus westeurope --tags' + End + + It 'creates a generalized Gen2 confidential-VM-supported definition when needed' + MOCK_DEFINITION_EXISTS=false + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be success + The output should include '--os-state Generalized' + The output should include '--hyper-v-generation V2' + The output should include 'SecurityType=ConfidentialVMSupported' + End + + It 'waits for replication even when provisioning has succeeded' + MOCK_REPLICATION_PENDING_ONCE=true + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be success + The output should include 'replication: InProgress' + The output should include 'sleep 60' + The output should include 'replication: Completed' + End + + It 'fails explicitly if the source storage metadata is unavailable' + MOCK_STORAGE_ACCOUNT_MISSING=true + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be failure + The output should include 'Unable to resolve source storage account' + The output should not include 'image-version create' + End + + It 'checks completion without mutating Azure resources' + MOCK_VERSION_STATE=missing + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh --check-only + The status should be failure + The output should include 'publication is not complete' + The output should not include 'image-definition show' + The output should not include 'image-version create' + The output should not include 'image-version delete' + End + + It 'surfaces authorization failures rather than treating them as missing versions' + MOCK_VERSION_READ_FAILURE=true + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be failure + The output should include 'AuthorizationFailed' + The output should not include 'image-version create' + The output should not include 'image-version delete' + End + + It 'does not succeed when gallery replication fails' + MOCK_REPLICATION_FAILURE=true + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be failure + The output should include 'replication: Failed' + The output should include 'did not complete publication' + End + + It 'does not treat incomplete replication as completed delivery' + MOCK_VERSION_STATE=succeeded MOCK_REPLICATION_PENDING_ONCE=true + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh --check-only + The status should be failure + The output should include 'publication is not complete' + End + + It 'skips unsupported OS flavors without storage lookups' + OS_SKU=AzureContainerLinux + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be success + The output should include 'publication is not required' + The output should not include 'storage account list' + End + End + + Describe 'Parallel delivery and retries' + It 'reads the original blob concurrently and joins both operations before staging cleanup' + MOCK_REQUIRE_OVERLAP=true MOCK_GALLERY_WAIT_FOR_STORAGE=true MOCK_COPY_PENDING_ONCE=true + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be success + The output should include "azcopy copy ${SOURCE_URI} ${STORAGE_ACCOUNT_BLOB_URL_STAGING}/${VHD_NAME}" + The output should include "--os-vhd-uri ${SOURCE_URI}" + The output should include "storage blob copy start" + The output should include 'sleep 30s' + The output should include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should include 'storage blob delete' + End + + It 'waits for gallery publication even if the storage copy fails and blocks cleanup' + MOCK_STORAGE_FAILURE=true MOCK_GALLERY_WAIT_FOR_STORAGE=true + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be failure + The output should include 'Post-prefetch delivery failed: storage=6, gallery=0' + The output should not include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'storage blob delete' + End + + It 'waits for immutable storage delivery even if gallery publication fails' + MOCK_GALLERY_FAILURE=true MOCK_REQUIRE_OVERLAP=true + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be failure + The output should include 'server-side copy to https://destination.blob.core.windows.net/vhd/test.vhd completed successfully' + The output should include 'Post-prefetch delivery failed: storage=0, gallery=7' + The output should not include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'storage blob delete' + End + + It 'blocks cleanup if the server-side immutable copy fails' + MOCK_COPY_STATUS=failed + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be failure + The output should include 'finished with unexpected status: failed' + The output should include 'Post-prefetch delivery failed: storage=1, gallery=0' + The output should not include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'storage blob delete' + End + + It 'waits for a pending immutable copy on retry before confirming completed delivery' + MOCK_FINAL_EXISTS=true MOCK_VERSION_STATE=succeeded MOCK_COPY_PENDING_ONCE=true + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be success + The output should include 'is currently being copied' + The output should include 'completed successfully' + The output should include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'storage blob copy start' + The output should not include 'resource show' + End + + It 'retries missing gallery publication without recopying or rebuilding an immutable VHD' + MOCK_FINAL_EXISTS=true + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be success + The output should include "--os-vhd-uri ${SOURCE_URI}" + The output should include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'azcopy copy' + The output should not include 'storage blob copy start' + The output should not include 'image builder show' + End + + It 'resolves the original blob even when the destination staging copy is reusable' + MOCK_STAGING_COMPLETE=true + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be success + The output should include "--os-vhd-uri ${SOURCE_URI}" + The output should include 'storage blob copy start' + The output should not include 'azcopy copy' + End + + It 'allows an already-completed retry after Image Builder resources have been removed' + MOCK_FINAL_EXISTS=true MOCK_VERSION_STATE=succeeded MOCK_ARTIFACT_MISSING=true + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be success + The output should include 'Both VHD delivery and gallery publication are already complete' + The output should include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'resource show' + End + + It 'fails rather than rebuilding different bits if the original source is missing on retry' + MOCK_FINAL_EXISTS=true MOCK_ARTIFACT_MISSING=true + When call run_script vhdbuilder/prefetch/scripts/optimize.sh + The status should be failure + The output should include 'unable to determine artifactUri' + The output should not include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'image builder show' + The output should not include 'image-version create' + End + End + + Describe 'Image Builder cleanup' + It 'rejects cleanup without confirmation that both deliveries completed' + When call run_script vhdbuilder/prefetch/scripts/cleanup.sh + The status should be failure + The output should include 'must both complete before cleanup' + The output should not include 'image builder delete' + The output should not include 'group delete' + End + + It 'deletes the template before requesting resource group deletion' + PREFETCH_DELIVERY_COMPLETE=True + When call run_script vhdbuilder/prefetch/scripts/cleanup.sh + The status should be success + The output should include 'image builder delete -g image-builder-1.2.3-run -n template-1.2.3-run' + The output should include 'group delete -g image-builder-1.2.3-run --yes --no-wait' + End + + It 'does not delete the resource group if template deletion fails' + PREFETCH_DELIVERY_COMPLETE=True MOCK_TEMPLATE_DELETE_FAILURE=true + When call run_script vhdbuilder/prefetch/scripts/cleanup.sh + The status should be failure + The output should not include 'group delete' + End + End +End diff --git a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh index 46c5e52a114..ebf4fa70207 100755 --- a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh +++ b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh @@ -2,7 +2,7 @@ set -euo pipefail -STAGING_VHD_URI="${1:?staging VHD URI is required}" +SOURCE_VHD_URI="${1:?source VHD URI or --check-only is required}" cvm_requires_publishing() { grep -q "cvm" <<<"${FEATURE_FLAGS,,}" || return 1 @@ -79,34 +79,34 @@ ensure_generalized_image_definition() { } wait_for_successful_provisioning() { - local provisioning_state - - echo "Waiting five minutes before checking image version provisioning" - sleep 300 + local version_info provisioning_state replication_state while true; do - if provisioning_state="$( + if version_info="$( az sig image-version show \ --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ --gallery-name "${SIG_GALLERY_NAME}" \ --gallery-image-definition "${SKU_NAME}" \ --gallery-image-version "${CAPTURED_SIG_VERSION}" \ - --query provisioningState \ - --output tsv \ - --only-show-errors 2>/dev/null + --expand ReplicationStatus \ + --output json \ + --only-show-errors )"; then - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioning state: ${provisioning_state:-unknown}" + provisioning_state="$(jq -r '.provisioningState' <<<"${version_info}")" + replication_state="$(jq -r '.replicationStatus.aggregatedState' <<<"${version_info}")" + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioning: ${provisioning_state}, replication: ${replication_state}" - case "${provisioning_state,,}" in - succeeded) + case "${provisioning_state,,}/${replication_state,,}" in + succeeded/completed) return 0 ;; - failed) + failed/*|*/failed) return 1 ;; esac else - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} does not exist yet" + echo "Unable to read image version status" >&2 + return 1 fi sleep 60 @@ -132,87 +132,98 @@ delete_failed_image_version() { --only-show-errors } -cleanup_managed_image() { - if az image show \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --name "${SKU_NAME}-${CAPTURED_SIG_VERSION}-generalized-${BUILD_BUILDID}" \ - --only-show-errors >/dev/null 2>&1; then - echo "Deleting temporary managed image" - - az image delete \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --name "${SKU_NAME}-${CAPTURED_SIG_VERSION}-generalized-${BUILD_BUILDID}" \ - --only-show-errors - fi -} - -ensure_generalized_image_definition - -if provisioning_state="$( +if version_info="$( az sig image-version show \ --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ --gallery-name "${SIG_GALLERY_NAME}" \ --gallery-image-definition "${SKU_NAME}" \ --gallery-image-version "${CAPTURED_SIG_VERSION}" \ - --query provisioningState \ - --output tsv \ - --only-show-errors 2>/dev/null + --expand ReplicationStatus \ + --output json \ + --only-show-errors 2>&1 )"; then - case "${provisioning_state,,}" in - succeeded) - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} is already provisioned" + provisioning_state="$(jq -r '.provisioningState' <<<"${version_info}")" + replication_state="$(jq -r '.replicationStatus.aggregatedState' <<<"${version_info}")" + case "${provisioning_state,,}/${replication_state,,}" in + succeeded/completed) + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} is already provisioned and replicated" exit 0 ;; - failed) - delete_failed_image_version - ;; - *) - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} already exists and is provisioning" - - if wait_for_successful_provisioning; then - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioned successfully" - exit 0 - fi - - delete_failed_image_version - ;; esac +else + if [[ "${version_info}" != *"(ResourceNotFound)"* && "${version_info}" != *"(ParentResourceNotFound)"* ]]; then + echo "Unable to read image version: ${version_info}" >&2 + exit 1 + fi + provisioning_state="missing" + replication_state="missing" fi -managed_image_id="$( - az image create \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --name "${SKU_NAME}-${CAPTURED_SIG_VERSION}-generalized-${BUILD_BUILDID}" \ - --location "${PACKER_BUILD_LOCATION}" \ - --source "${STAGING_VHD_URI}" \ - --os-type Linux \ - --hyper-v-generation V2 \ - --storage-sku Standard_LRS \ - --tags \ - "buildId=${BUILD_BUILDID}" \ - "createdBy=aks-vhd-pipeline" \ - --query id \ - --output tsv \ +if [[ "${SOURCE_VHD_URI}" = "--check-only" ]]; then + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} publication is not complete" + exit 1 +fi + +ensure_generalized_image_definition + +case "${provisioning_state,,}/${replication_state,,}" in + missing/*) + ;; + failed/*|*/failed) + delete_failed_image_version + ;; + *) + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} already exists and is provisioning" + + if wait_for_successful_provisioning; then + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioned successfully" + exit 0 + fi + + echo "Generalized CVM image version did not complete publication" >&2 + exit 1 + ;; +esac + +if [[ ! "${SOURCE_VHD_URI}" =~ ^https://([a-z0-9]{3,24})\.blob\.[^/]+/.+ ]]; then + echo "Source VHD must be an Azure Storage blob URL" >&2 + exit 1 +fi +source_storage_account="${BASH_REMATCH[1]}" +source_storage_info="$( + az storage account list \ + --query "[?name=='${source_storage_account}'].{id:id,location:location} | [0]" \ + --output json \ --only-show-errors )" +source_storage_id="$(jq -r '.id // empty' <<<"${source_storage_info}")" +source_location="$(jq -r '.location // empty' <<<"${source_storage_info}")" +if [[ -z "${source_storage_id}" || -z "${source_location}" ]]; then + echo "Unable to resolve source storage account ${source_storage_account} and its location" >&2 + exit 1 +fi -trap cleanup_managed_image EXIT +target_regions=(eastus westeurope) +if [[ "${source_location}" != "eastus" && "${source_location}" != "westeurope" ]]; then + target_regions+=("${source_location}") +fi az sig image-version create \ --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ --gallery-name "${SIG_GALLERY_NAME}" \ --gallery-image-definition "${SKU_NAME}" \ --gallery-image-version "${CAPTURED_SIG_VERSION}" \ - --managed-image "${managed_image_id}" \ - --target-regions eastus westeurope \ + --location "${source_location}" \ + --os-vhd-uri "${SOURCE_VHD_URI}" \ + --os-vhd-storage-account "${source_storage_id}" \ + --target-regions "${target_regions[@]}" \ --tags \ "buildId=${BUILD_BUILDID}" \ "createdBy=aks-vhd-pipeline" \ --only-show-errors >/dev/null if ! wait_for_successful_provisioning; then - delete_failed_image_version - echo "Generalized CVM image version failed to provision" >&2 + echo "Generalized CVM image version did not complete publication" >&2 exit 1 fi diff --git a/vhdbuilder/prefetch/README.md b/vhdbuilder/prefetch/README.md index 6080ef1cf7e..17353ea67eb 100644 --- a/vhdbuilder/prefetch/README.md +++ b/vhdbuilder/prefetch/README.md @@ -1,3 +1,18 @@ # Prefetch -Provides functionality to generate scripts used for prefetch optimization. \ No newline at end of file +Provides functionality to generate scripts used for prefetch optimization. + +After Image Builder finishes, its original post-prefetch VHD is used concurrently +for generalized CVM gallery publication and delivery to the final storage account +(via the mutable staging container and then the immutable VHD container). Gallery +versions are created directly from that VHD, without an intermediate managed image, +in the source storage account's region. Replication includes East US, West Europe, +and the source region when different. The managed image used as Image Builder's +input for TL/CVM builds is unchanged. + +The prefetch task waits for both storage delivery and gallery replication to finish +before setting `PREFETCH_DELIVERY_COMPLETE=True`. The cleanup task requires this +signal and deletes the Image Builder template before its resource group, releasing +the service-managed staging storage and original VHD. On failure, cleanup is +blocked; retries reuse the original output and do not skip missing gallery +publication just because the immutable VHD already exists. \ No newline at end of file diff --git a/vhdbuilder/prefetch/scripts/cleanup.sh b/vhdbuilder/prefetch/scripts/cleanup.sh index 8b6146b0d5b..ea03378e2be 100755 --- a/vhdbuilder/prefetch/scripts/cleanup.sh +++ b/vhdbuilder/prefetch/scripts/cleanup.sh @@ -3,18 +3,28 @@ set -uxo pipefail [ -z "${CAPTURED_SIG_VERSION:-}" ] && echo "CAPTURED_SIG_VERSION is not set" && exit 1 [ -z "${BUILD_RUN_NUMBER:-}" ] && echo "BUILD_RUN_NUMBER is not set" && exit 1 +[ "${PREFETCH_DELIVERY_COMPLETE:-}" != "True" ] && echo "VHD delivery and gallery publication must both complete before cleanup" && exit 1 main() { cleanup_image_builder_rg } cleanup_image_builder_rg() { + local group_exists IMAGE_BUILDER_RG_NAME="image-builder-${CAPTURED_SIG_VERSION}-${BUILD_RUN_NUMBER}" - if [ "$(az group exists -g "${IMAGE_BUILDER_RG_NAME}")" = "true" ]; then + group_exists="$(az group exists -g "${IMAGE_BUILDER_RG_NAME}")" || return $? + if [ "${group_exists}" = "true" ]; then + local template_name="template-${CAPTURED_SIG_VERSION}-${BUILD_RUN_NUMBER}" + template_name="${template_name:0:64}" + az image builder delete -g "${IMAGE_BUILDER_RG_NAME}" -n "${template_name}" || return $? echo "deleting image builder resource group: ${IMAGE_BUILDER_RG_NAME}" if ! az group delete -g "${IMAGE_BUILDER_RG_NAME}" --yes --no-wait; then - echo "unable to delete image builder resource group: ${IMAGE_BUILDER_RG_NAME}, will still proceed" + echo "unable to delete image builder resource group: ${IMAGE_BUILDER_RG_NAME}" >&2 + return 1 fi + elif [ "${group_exists}" != "false" ]; then + echo "Unable to determine whether image builder resource group exists" >&2 + return 1 fi } diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index 8493ff1ab53..1195607fd57 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -56,46 +56,62 @@ DESTINATION_STORAGE_ACCOUNT_NAME="${STORAGE_ACCOUNT_BLOB_URL#*://}" DESTINATION_STORAGE_ACCOUNT_NAME="${DESTINATION_STORAGE_ACCOUNT_NAME%%.*}" DESTINATION_CONTAINER_NAME="${STORAGE_ACCOUNT_BLOB_URL##*/}" STAGING_VHD_URI="${STORAGE_ACCOUNT_BLOB_URL_STAGING}/${VHD_NAME}" +GENERALIZED_CVM_PUBLISHER="$(dirname "${BASH_SOURCE[0]}")/../../packer/publish-generalized-cvm-if-required.sh" +VHD_COPY_COMPLETE=false main() { - # for idempotency, check whether the target VHD already exists. If a previous run already produced it - # successfully, exit early; if a server-side copy is still in progress, wait for it to finish. Because - # the destination container may be immutable, a VHD left in a bad state cannot be deleted and recreated - # and is treated as fatal. - check_for_existing_vhd - - # attempt to perform prefetch optimization, distributing the optimized image as a VHD into the - # image builder staging storage account, then copy that VHD into the target storage account - ensure_image_builder_rg || exit $? - run_image_builder_template || exit $? + echo "##vso[task.setvariable variable=PREFETCH_DELIVERY_COMPLETE]False" + check_for_existing_vhd || exit $? + + if [ "${VHD_COPY_COMPLETE}" = true ]; then + if "${GENERALIZED_CVM_PUBLISHER}" --check-only; then + echo "Both VHD delivery and gallery publication are already complete" + echo "##vso[task.setvariable variable=PREFETCH_DELIVERY_COMPLETE]True" + return 0 + fi + # Reuse the original output on publication retries; rebuilding could publish different + # contents from those already delivered to the immutable container. + else + ensure_image_builder_rg || exit $? + run_image_builder_template || exit $? + fi copy_optimized_vhd || exit $? + echo "##vso[task.setvariable variable=PREFETCH_DELIVERY_COMPLETE]True" } check_for_existing_vhd() { - if [ "$(az storage blob exists --blob-url "${VHD_URI}" --auth-mode login | jq -r '.exists')" = "false" ]; then + local exists copy_status + exists="$(az storage blob exists --blob-url "${VHD_URI}" --auth-mode login | jq -r '.exists')" || return $? + if [ "${exists}" = "false" ]; then echo "no existing VHD was found at: ${VHD_URI}, will proceed with optimization and VHD creation" return 0 fi + if [ "${exists}" != "true" ]; then + echo "Unable to determine whether VHD ${VHD_URI} exists" >&2 + return 1 + fi echo "VHD already exists at: ${VHD_URI}" # the VHD in a potentially immutable container is produced by a server-side blob copy, so its copy status # tells us whether a previous run already completed. Because the container may have an immutability policy, # a VHD in a bad state cannot be deleted and recreated, so any non-success terminal state is fatal. - copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" + copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" || return $? if [ "${copy_status,,}" = "success" ]; then - echo "VHD ${VHD_URI} was already produced by a previous prefetch optimization run, nothing to do" - exit 0 + echo "VHD ${VHD_URI} was already produced by a previous prefetch optimization run" + VHD_COPY_COMPLETE=true + return 0 fi if [ "${copy_status,,}" = "pending" ]; then echo "VHD ${VHD_URI} is currently being copied by a previous prefetch optimization run, will wait for completion" if wait_for_vhd_copy; then - exit 0 + VHD_COPY_COMPLETE=true + return 0 fi fi # the VHD exists in a non-success state within the immutable container. It cannot be deleted due to # the immutability policy, so there is nothing we can do to recover - fail the script. echo "VHD ${VHD_URI} exists in an unrecoverable copy state: '${copy_status}'" - exit 1 + return 1 } ensure_image_builder_rg() { @@ -308,6 +324,34 @@ create_temp_storage() { # 1. azcopy the optimized VHD into the staging container (STORAGE_ACCOUNT_BLOB_URL_STAGING) # 2. perform a single server-side blob copy from the staging container into the immutable vhd container copy_optimized_vhd() { + local artifact_uri gallery_pid gallery_status=0 storage_status=0 + artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv)" || return $? + if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then + echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" + return 1 + fi + + "${GENERALIZED_CVM_PUBLISHER}" "${artifact_uri}" & + gallery_pid=$! + copy_vhd_to_storage "${artifact_uri}" || storage_status=$? + # Always join the publisher, even when storage delivery fails, so cleanup cannot race either reader. + wait "${gallery_pid}" || gallery_status=$? + if [ "${storage_status}" -ne 0 ] || [ "${gallery_status}" -ne 0 ]; then + echo "Post-prefetch delivery failed: storage=${storage_status}, gallery=${gallery_status}" >&2 + return 1 + fi + + delete_staging_vhd true || echo "unable to delete staging VHD ${STAGING_VHD_URI}, will proceed" + echo "optimized VHD and gallery publication are complete: ${VHD_URI}" +} + +copy_vhd_to_storage() { + local artifact_uri="${1:?Image Builder artifact URI is required}" + local staging_info staging_blob_name azcopy_exit_code + if [ "${VHD_COPY_COMPLETE}" = true ]; then + return 0 + fi + # The staging container should never have an immutability policy, so copying from image builder staging storage is # fully retriable. If a previous run already produced a complete staging blob (marked with the # prefetched metadata flag) but failed before finishing the server-side copy into the @@ -323,11 +367,6 @@ copy_optimized_vhd() { delete_staging_vhd || return $? fi - artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv 2>/dev/null)" - if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then - echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" - return 1 - fi echo "setting azcopy environment variables with pool identity: ${IMAGE_BUILDER_IDENTITY_ID}" export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" export AZCOPY_CONCURRENCY_VALUE="AUTO" @@ -355,24 +394,15 @@ copy_optimized_vhd() { --source-uri "${STAGING_VHD_URI}" || return $? wait_for_vhd_copy || return $? - - "$(dirname "${BASH_SOURCE[0]}")/../../packer/publish-generalized-cvm-if-required.sh" \ - "${STAGING_VHD_URI}" || return $? - - # the optimized VHD is now safely in the immutable vhd container, so the staging copy is no longer - # needed - delete it as best-effort cleanup so staging blobs do not accumulate in the destination account. - # Use no-wait since we don't need to wait for the blob to be deleted in this case. - delete_staging_vhd true || echo "unable to delete staging VHD ${STAGING_VHD_URI}, will proceed" - - echo "optimized VHD has been published to: ${VHD_URI}" } wait_for_vhd_copy() { - copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" + local copy_status + copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" || return $? while [ "${copy_status,,}" = "pending" ]; do echo "server-side copy to ${VHD_URI} is still pending, will wait 30s before checking again" sleep 30s - copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" + copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" || return $? done if [ "${copy_status,,}" != "success" ]; then echo "server-side copy to ${VHD_URI} finished with unexpected status: ${copy_status}" From 01246527e89c2f4482845251a26c9928c160a9b6 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 13:34:13 -0400 Subject: [PATCH 16/37] fix --- .../templates/.builder-release-template.yaml | 2 - spec/vhdbuilder/prefetch/publication_spec.sh | 44 +++++-------------- vhdbuilder/prefetch/README.md | 9 ++-- vhdbuilder/prefetch/scripts/cleanup.sh | 14 +----- vhdbuilder/prefetch/scripts/optimize.sh | 3 -- 5 files changed, 16 insertions(+), 56 deletions(-) diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index f8fa6823305..5e27f39dbd5 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -382,7 +382,6 @@ steps: condition: | and( succeeded(), - eq(variables.PREFETCH_DELIVERY_COMPLETE, 'True'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables.RUN_PREFETCH_OPTIMIZATION, 'True'), eq(variables.PREFETCH_COMPATIBLE, 'True') @@ -391,7 +390,6 @@ steps: env: BUILD_RUN_NUMBER: $(Build.BuildNumber) CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) - PREFETCH_DELIVERY_COMPLETE: $(PREFETCH_DELIVERY_COMPLETE) - task: PublishPipelineArtifact@0 condition: always() diff --git a/spec/vhdbuilder/prefetch/publication_spec.sh b/spec/vhdbuilder/prefetch/publication_spec.sh index 4510f32cc6e..43e90a39396 100644 --- a/spec/vhdbuilder/prefetch/publication_spec.sh +++ b/spec/vhdbuilder/prefetch/publication_spec.sh @@ -21,11 +21,10 @@ Describe 'Post-prefetch CVM publication' export MOCK_FINAL_EXISTS=false MOCK_STAGING_COMPLETE=false MOCK_VERSION_STATE=missing export MOCK_STORAGE_FAILURE=false MOCK_GALLERY_FAILURE=false MOCK_REQUIRE_OVERLAP=false export MOCK_GALLERY_WAIT_FOR_STORAGE=false MOCK_REPLICATION_PENDING_ONCE=false - export MOCK_ARTIFACT_MISSING=false MOCK_TEMPLATE_DELETE_FAILURE=false + export MOCK_ARTIFACT_MISSING=false export MOCK_DEFINITION_EXISTS=true MOCK_VERSION_READ_FAILURE=false export MOCK_REPLICATION_FAILURE=false export MOCK_COPY_PENDING_ONCE=false MOCK_COPY_STATUS=success - export PREFETCH_DELIVERY_COMPLETE=False wait_for_marker() { local attempt @@ -134,13 +133,7 @@ Describe 'Post-prefetch CVM publication' ;; 'image builder wait') ;; - 'image builder delete') - [ "${MOCK_TEMPLATE_DELETE_FAILURE}" = true ] && return 8 - touch "${TEST_ROOT}/template-deleted" - return 0 - ;; 'group delete -g') - [ -f "${TEST_ROOT}/template-deleted" ] || return 92 ;; *) echo "Unexpected Azure call: $*" >&2 @@ -277,7 +270,7 @@ Describe 'Post-prefetch CVM publication' The output should include "--os-vhd-uri ${SOURCE_URI}" The output should include "storage blob copy start" The output should include 'sleep 30s' - The output should include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should include 'optimized VHD and gallery publication are complete:' The output should include 'storage blob delete' End @@ -286,7 +279,7 @@ Describe 'Post-prefetch CVM publication' When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be failure The output should include 'Post-prefetch delivery failed: storage=6, gallery=0' - The output should not include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'optimized VHD and gallery publication are complete:' The output should not include 'storage blob delete' End @@ -296,7 +289,7 @@ Describe 'Post-prefetch CVM publication' The status should be failure The output should include 'server-side copy to https://destination.blob.core.windows.net/vhd/test.vhd completed successfully' The output should include 'Post-prefetch delivery failed: storage=0, gallery=7' - The output should not include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'optimized VHD and gallery publication are complete:' The output should not include 'storage blob delete' End @@ -306,7 +299,7 @@ Describe 'Post-prefetch CVM publication' The status should be failure The output should include 'finished with unexpected status: failed' The output should include 'Post-prefetch delivery failed: storage=1, gallery=0' - The output should not include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'optimized VHD and gallery publication are complete:' The output should not include 'storage blob delete' End @@ -316,7 +309,7 @@ Describe 'Post-prefetch CVM publication' The status should be success The output should include 'is currently being copied' The output should include 'completed successfully' - The output should include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should include 'Both VHD delivery and gallery publication are already complete' The output should not include 'storage blob copy start' The output should not include 'resource show' End @@ -326,7 +319,7 @@ Describe 'Post-prefetch CVM publication' When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be success The output should include "--os-vhd-uri ${SOURCE_URI}" - The output should include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should include 'optimized VHD and gallery publication are complete:' The output should not include 'azcopy copy' The output should not include 'storage blob copy start' The output should not include 'image builder show' @@ -346,7 +339,6 @@ Describe 'Post-prefetch CVM publication' When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be success The output should include 'Both VHD delivery and gallery publication are already complete' - The output should include 'variable=PREFETCH_DELIVERY_COMPLETE]True' The output should not include 'resource show' End @@ -355,34 +347,18 @@ Describe 'Post-prefetch CVM publication' When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be failure The output should include 'unable to determine artifactUri' - The output should not include 'variable=PREFETCH_DELIVERY_COMPLETE]True' + The output should not include 'optimized VHD and gallery publication are complete:' The output should not include 'image builder show' The output should not include 'image-version create' End End Describe 'Image Builder cleanup' - It 'rejects cleanup without confirmation that both deliveries completed' - When call run_script vhdbuilder/prefetch/scripts/cleanup.sh - The status should be failure - The output should include 'must both complete before cleanup' - The output should not include 'image builder delete' - The output should not include 'group delete' - End - - It 'deletes the template before requesting resource group deletion' - PREFETCH_DELIVERY_COMPLETE=True + It 'retains resource group cleanup without additional prerequisites' When call run_script vhdbuilder/prefetch/scripts/cleanup.sh The status should be success - The output should include 'image builder delete -g image-builder-1.2.3-run -n template-1.2.3-run' The output should include 'group delete -g image-builder-1.2.3-run --yes --no-wait' - End - - It 'does not delete the resource group if template deletion fails' - PREFETCH_DELIVERY_COMPLETE=True MOCK_TEMPLATE_DELETE_FAILURE=true - When call run_script vhdbuilder/prefetch/scripts/cleanup.sh - The status should be failure - The output should not include 'group delete' + The output should not include 'image builder delete' End End End diff --git a/vhdbuilder/prefetch/README.md b/vhdbuilder/prefetch/README.md index 17353ea67eb..79d1bc8c05c 100644 --- a/vhdbuilder/prefetch/README.md +++ b/vhdbuilder/prefetch/README.md @@ -11,8 +11,7 @@ and the source region when different. The managed image used as Image Builder's input for TL/CVM builds is unchanged. The prefetch task waits for both storage delivery and gallery replication to finish -before setting `PREFETCH_DELIVERY_COMPLETE=True`. The cleanup task requires this -signal and deletes the Image Builder template before its resource group, releasing -the service-managed staging storage and original VHD. On failure, cleanup is -blocked; retries reuse the original output and do not skip missing gallery -publication just because the immutable VHD already exists. \ No newline at end of file +before returning success. The existing cleanup task runs afterward only on success; +no additional cleanup guard or wait loop is needed. On failure, cleanup is blocked; +retries reuse the original output and do not skip missing gallery publication just +because the immutable VHD already exists. \ No newline at end of file diff --git a/vhdbuilder/prefetch/scripts/cleanup.sh b/vhdbuilder/prefetch/scripts/cleanup.sh index ea03378e2be..8b6146b0d5b 100755 --- a/vhdbuilder/prefetch/scripts/cleanup.sh +++ b/vhdbuilder/prefetch/scripts/cleanup.sh @@ -3,28 +3,18 @@ set -uxo pipefail [ -z "${CAPTURED_SIG_VERSION:-}" ] && echo "CAPTURED_SIG_VERSION is not set" && exit 1 [ -z "${BUILD_RUN_NUMBER:-}" ] && echo "BUILD_RUN_NUMBER is not set" && exit 1 -[ "${PREFETCH_DELIVERY_COMPLETE:-}" != "True" ] && echo "VHD delivery and gallery publication must both complete before cleanup" && exit 1 main() { cleanup_image_builder_rg } cleanup_image_builder_rg() { - local group_exists IMAGE_BUILDER_RG_NAME="image-builder-${CAPTURED_SIG_VERSION}-${BUILD_RUN_NUMBER}" - group_exists="$(az group exists -g "${IMAGE_BUILDER_RG_NAME}")" || return $? - if [ "${group_exists}" = "true" ]; then - local template_name="template-${CAPTURED_SIG_VERSION}-${BUILD_RUN_NUMBER}" - template_name="${template_name:0:64}" - az image builder delete -g "${IMAGE_BUILDER_RG_NAME}" -n "${template_name}" || return $? + if [ "$(az group exists -g "${IMAGE_BUILDER_RG_NAME}")" = "true" ]; then echo "deleting image builder resource group: ${IMAGE_BUILDER_RG_NAME}" if ! az group delete -g "${IMAGE_BUILDER_RG_NAME}" --yes --no-wait; then - echo "unable to delete image builder resource group: ${IMAGE_BUILDER_RG_NAME}" >&2 - return 1 + echo "unable to delete image builder resource group: ${IMAGE_BUILDER_RG_NAME}, will still proceed" fi - elif [ "${group_exists}" != "false" ]; then - echo "Unable to determine whether image builder resource group exists" >&2 - return 1 fi } diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index 1195607fd57..dc52d8f4e34 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -60,13 +60,11 @@ GENERALIZED_CVM_PUBLISHER="$(dirname "${BASH_SOURCE[0]}")/../../packer/publish-g VHD_COPY_COMPLETE=false main() { - echo "##vso[task.setvariable variable=PREFETCH_DELIVERY_COMPLETE]False" check_for_existing_vhd || exit $? if [ "${VHD_COPY_COMPLETE}" = true ]; then if "${GENERALIZED_CVM_PUBLISHER}" --check-only; then echo "Both VHD delivery and gallery publication are already complete" - echo "##vso[task.setvariable variable=PREFETCH_DELIVERY_COMPLETE]True" return 0 fi # Reuse the original output on publication retries; rebuilding could publish different @@ -76,7 +74,6 @@ main() { run_image_builder_template || exit $? fi copy_optimized_vhd || exit $? - echo "##vso[task.setvariable variable=PREFETCH_DELIVERY_COMPLETE]True" } check_for_existing_vhd() { From 6ea2a1fdffff383b7a618abc3e4aa71acc4163ce Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 14:02:42 -0400 Subject: [PATCH 17/37] fi --- spec/vhdbuilder/prefetch/publication_spec.sh | 44 +++- .../publish-generalized-cvm-if-required.sh | 214 +++--------------- vhdbuilder/prefetch/README.md | 6 +- 3 files changed, 67 insertions(+), 197 deletions(-) diff --git a/spec/vhdbuilder/prefetch/publication_spec.sh b/spec/vhdbuilder/prefetch/publication_spec.sh index 43e90a39396..6f23370e2a5 100644 --- a/spec/vhdbuilder/prefetch/publication_spec.sh +++ b/spec/vhdbuilder/prefetch/publication_spec.sh @@ -39,14 +39,15 @@ Describe 'Post-prefetch CVM publication' az() { printf '%s\n' "$*" >> "${AZ_MOCK_LOG}" case "$1 $2 $3" in - 'sig image-definition show') + 'sig image-definition list') if [ "${MOCK_DEFINITION_EXISTS}" = false ] && [ ! -f "${TEST_ROOT}/definition-created" ]; then - return 3 + return 0 fi echo definition ;; 'sig image-definition create') touch "${TEST_ROOT}/definition-created" + echo definition ;; 'resource update --ids') ;; @@ -82,7 +83,15 @@ Describe 'Post-prefetch CVM publication' [ "${MOCK_GALLERY_FAILURE}" = true ] && return 7 touch "${TEST_ROOT}/version-created" ;; - 'sig image-version delete'|'sig image-version wait') + 'sig image-version wait') + if [ "${MOCK_REPLICATION_PENDING_ONCE}" = true ]; then + sleep 60 + fi + if [ "${MOCK_REPLICATION_FAILURE}" = true ]; then + echo 'Gallery replication did not complete' >&2 + return 8 + fi + touch "${TEST_ROOT}/replication-ready" ;; 'storage account list') if [ "${MOCK_STORAGE_ACCOUNT_MISSING}" = true ]; then @@ -115,7 +124,7 @@ Describe 'Post-prefetch CVM publication' ;; 'storage blob delete') if [ "${MOCK_VERSION_STATE}" != succeeded ] && [ "${FEATURE_FLAGS}" = cvm ]; then - [ -f "${TEST_ROOT}/gallery-finished" ] || return 91 + [ -f "${TEST_ROOT}/replication-ready" ] || return 91 fi ;; 'resource show --ids') @@ -205,9 +214,10 @@ Describe 'Post-prefetch CVM publication' MOCK_REPLICATION_PENDING_ONCE=true When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" The status should be success - The output should include 'replication: InProgress' + The output should include 'sig image-version wait' + The output should include '--expand ReplicationStatus --custom' + The output should include "provisioningState=='Succeeded' && replicationStatus.aggregatedState=='Completed'" The output should include 'sleep 60' - The output should include 'replication: Completed' End It 'fails explicitly if the source storage metadata is unavailable' @@ -222,27 +232,37 @@ Describe 'Post-prefetch CVM publication' MOCK_VERSION_STATE=missing When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh --check-only The status should be failure - The output should include 'publication is not complete' - The output should not include 'image-definition show' + The output should include 'ResourceNotFound' + The output should not include 'image-definition list' The output should not include 'image-version create' The output should not include 'image-version delete' End - It 'surfaces authorization failures rather than treating them as missing versions' + It 'surfaces authorization failures when checking completion' MOCK_VERSION_READ_FAILURE=true - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh --check-only The status should be failure The output should include 'AuthorizationFailed' The output should not include 'image-version create' The output should not include 'image-version delete' End + It 'does not inspect or delete failed versions before publishing' + MOCK_VERSION_STATE=failed + When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" + The status should be success + The output should include 'sig image-version create' + The output should not include 'sig image-version show' + The output should not include 'sig image-version delete' + The output should not include '--deleted' + End + It 'does not succeed when gallery replication fails' MOCK_REPLICATION_FAILURE=true When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" The status should be failure - The output should include 'replication: Failed' - The output should include 'did not complete publication' + The output should include 'Gallery replication did not complete' + The output should not include 'provisioned successfully' End It 'does not treat incomplete replication as completed delivery' diff --git a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh index ebf4fa70207..b4a0325d017 100755 --- a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh +++ b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh @@ -4,187 +4,28 @@ set -euo pipefail SOURCE_VHD_URI="${1:?source VHD URI or --check-only is required}" -cvm_requires_publishing() { - grep -q "cvm" <<<"${FEATURE_FLAGS,,}" || return 1 - - case "${OS_SKU,,}" in - ubuntu|azurelinux) - return 0 - ;; - *) - return 1 - ;; - esac -} - -if ! cvm_requires_publishing; then +if [[ "${FEATURE_FLAGS,,}" != *cvm* || ! "${OS_SKU,,}" =~ ^(ubuntu|azurelinux)$ ]]; then echo "Generalized CVM publication is not required for OS_SKU=${OS_SKU:-unset}, FEATURE_FLAGS=${FEATURE_FLAGS:-unset}" exit 0 fi -required_variables=( - AZURE_RESOURCE_GROUP_NAME - SIG_GALLERY_NAME - SKU_NAME - CAPTURED_SIG_VERSION - PACKER_BUILD_LOCATION - BUILD_BUILDID +gallery_args=( + --resource-group "${AZURE_RESOURCE_GROUP_NAME:?}" + --gallery-name "${SIG_GALLERY_NAME:?}" + --only-show-errors ) +definition_args=("${gallery_args[@]}" --gallery-image-definition "${SKU_NAME:?}") +version_args=("${definition_args[@]}" --gallery-image-version "${CAPTURED_SIG_VERSION:?}") -for variable in "${required_variables[@]}"; do - if [[ -z "${!variable:-}" ]]; then - echo "Required variable ${variable} is not set" >&2 - exit 1 - fi -done - -ensure_generalized_image_definition() { - if az sig image-definition show \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" \ - --only-show-errors >/dev/null 2>&1; then - return - fi - - az sig image-definition create \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" \ - --publisher microsoft-aks \ - --offer "${SIG_GALLERY_NAME}" \ - --sku "${SKU_NAME}" \ - --os-type Linux \ - --os-state Generalized \ - --architecture x64 \ - --hyper-v-generation V2 \ - --location "${PACKER_BUILD_LOCATION}" \ - --features \ - "DiskControllerTypes=SCSI,NVMe" \ - "SecurityType=ConfidentialVMSupported" \ - --only-show-errors >/dev/null - - az resource update \ - --ids "$( - az sig image-definition show \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" \ - --query id \ - --output tsv \ - --only-show-errors - )" \ - --set properties.allowUpdateImage=true \ - --only-show-errors >/dev/null -} - -wait_for_successful_provisioning() { - local version_info provisioning_state replication_state - - while true; do - if version_info="$( - az sig image-version show \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" \ - --gallery-image-version "${CAPTURED_SIG_VERSION}" \ - --expand ReplicationStatus \ - --output json \ - --only-show-errors - )"; then - provisioning_state="$(jq -r '.provisioningState' <<<"${version_info}")" - replication_state="$(jq -r '.replicationStatus.aggregatedState' <<<"${version_info}")" - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioning: ${provisioning_state}, replication: ${replication_state}" - - case "${provisioning_state,,}/${replication_state,,}" in - succeeded/completed) - return 0 - ;; - failed/*|*/failed) - return 1 - ;; - esac - else - echo "Unable to read image version status" >&2 - return 1 - fi - - sleep 60 - done -} - -delete_failed_image_version() { - echo "Deleting failed image version ${SKU_NAME}/${CAPTURED_SIG_VERSION}" - - az sig image-version delete \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" \ - --gallery-image-version "${CAPTURED_SIG_VERSION}" \ - --only-show-errors - - az sig image-version wait \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" \ - --gallery-image-version "${CAPTURED_SIG_VERSION}" \ - --deleted \ - --only-show-errors -} - -if version_info="$( - az sig image-version show \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" \ - --gallery-image-version "${CAPTURED_SIG_VERSION}" \ - --expand ReplicationStatus \ - --output json \ - --only-show-errors 2>&1 -)"; then - provisioning_state="$(jq -r '.provisioningState' <<<"${version_info}")" - replication_state="$(jq -r '.replicationStatus.aggregatedState' <<<"${version_info}")" - case "${provisioning_state,,}/${replication_state,,}" in - succeeded/completed) - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} is already provisioned and replicated" - exit 0 - ;; - esac -else - if [[ "${version_info}" != *"(ResourceNotFound)"* && "${version_info}" != *"(ParentResourceNotFound)"* ]]; then - echo "Unable to read image version: ${version_info}" >&2 +if [[ "${SOURCE_VHD_URI}" = "--check-only" ]]; then + version_info="$(az sig image-version show "${version_args[@]}" --expand ReplicationStatus --output json)" + if ! jq -e '.provisioningState == "Succeeded" and .replicationStatus.aggregatedState == "Completed"' <<<"${version_info}" >/dev/null; then + echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} publication is not complete" >&2 exit 1 fi - provisioning_state="missing" - replication_state="missing" -fi - -if [[ "${SOURCE_VHD_URI}" = "--check-only" ]]; then - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} publication is not complete" - exit 1 + exit 0 fi -ensure_generalized_image_definition - -case "${provisioning_state,,}/${replication_state,,}" in - missing/*) - ;; - failed/*|*/failed) - delete_failed_image_version - ;; - *) - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} already exists and is provisioning" - - if wait_for_successful_provisioning; then - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioned successfully" - exit 0 - fi - - echo "Generalized CVM image version did not complete publication" >&2 - exit 1 - ;; -esac - if [[ ! "${SOURCE_VHD_URI}" =~ ^https://([a-z0-9]{3,24})\.blob\.[^/]+/.+ ]]; then echo "Source VHD must be an Azure Storage blob URL" >&2 exit 1 @@ -203,28 +44,33 @@ if [[ -z "${source_storage_id}" || -z "${source_location}" ]]; then exit 1 fi +definition_id="$(az sig image-definition list "${gallery_args[@]}" --query "[?name=='${SKU_NAME}'].id | [0]" --output tsv)" +if [[ -z "${definition_id}" || "${definition_id}" = "None" ]]; then + definition_id="$( + az sig image-definition create "${definition_args[@]}" \ + --publisher microsoft-aks --offer "${SIG_GALLERY_NAME}" --sku "${SKU_NAME}" \ + --os-type Linux --os-state Generalized --architecture x64 --hyper-v-generation V2 \ + --location "${PACKER_BUILD_LOCATION:?}" \ + --features "DiskControllerTypes=SCSI,NVMe" "SecurityType=ConfidentialVMSupported" \ + --query id --output tsv + )" + az resource update --ids "${definition_id}" --set properties.allowUpdateImage=true --only-show-errors >/dev/null +fi + target_regions=(eastus westeurope) if [[ "${source_location}" != "eastus" && "${source_location}" != "westeurope" ]]; then target_regions+=("${source_location}") fi -az sig image-version create \ - --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \ - --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" \ - --gallery-image-version "${CAPTURED_SIG_VERSION}" \ +az sig image-version create "${version_args[@]}" \ --location "${source_location}" \ --os-vhd-uri "${SOURCE_VHD_URI}" \ --os-vhd-storage-account "${source_storage_id}" \ --target-regions "${target_regions[@]}" \ - --tags \ - "buildId=${BUILD_BUILDID}" \ - "createdBy=aks-vhd-pipeline" \ - --only-show-errors >/dev/null + --tags "buildId=${BUILD_BUILDID:?}" "createdBy=aks-vhd-pipeline" >/dev/null -if ! wait_for_successful_provisioning; then - echo "Generalized CVM image version did not complete publication" >&2 - exit 1 -fi +az sig image-version wait "${version_args[@]}" --expand ReplicationStatus \ + --custom "provisioningState=='Succeeded' && replicationStatus.aggregatedState=='Completed'" \ + --interval 60 echo "Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioned successfully" diff --git a/vhdbuilder/prefetch/README.md b/vhdbuilder/prefetch/README.md index 79d1bc8c05c..507d66947ef 100644 --- a/vhdbuilder/prefetch/README.md +++ b/vhdbuilder/prefetch/README.md @@ -14,4 +14,8 @@ The prefetch task waits for both storage delivery and gallery replication to fin before returning success. The existing cleanup task runs afterward only on success; no additional cleanup guard or wait loop is needed. On failure, cleanup is blocked; retries reuse the original output and do not skip missing gallery publication just -because the immutable VHD already exists. \ No newline at end of file +because the immutable VHD already exists. + +The publisher uses Azure CLI's built-in wait for provisioning and replication. +It does not inspect, delete, or recover failed image versions before publication; +Azure CLI failures propagate to the prefetch task. \ No newline at end of file From 0dfbed1f506b9a5cbc7d8f4026b1954123a14e47 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 14:09:06 -0400 Subject: [PATCH 18/37] fix --- vhdbuilder/prefetch/scripts/optimize.sh | 90 +++++++++---------------- 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index dc52d8f4e34..f7ca0890757 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -56,59 +56,46 @@ DESTINATION_STORAGE_ACCOUNT_NAME="${STORAGE_ACCOUNT_BLOB_URL#*://}" DESTINATION_STORAGE_ACCOUNT_NAME="${DESTINATION_STORAGE_ACCOUNT_NAME%%.*}" DESTINATION_CONTAINER_NAME="${STORAGE_ACCOUNT_BLOB_URL##*/}" STAGING_VHD_URI="${STORAGE_ACCOUNT_BLOB_URL_STAGING}/${VHD_NAME}" -GENERALIZED_CVM_PUBLISHER="$(dirname "${BASH_SOURCE[0]}")/../../packer/publish-generalized-cvm-if-required.sh" -VHD_COPY_COMPLETE=false main() { - check_for_existing_vhd || exit $? - - if [ "${VHD_COPY_COMPLETE}" = true ]; then - if "${GENERALIZED_CVM_PUBLISHER}" --check-only; then - echo "Both VHD delivery and gallery publication are already complete" - return 0 - fi - # Reuse the original output on publication retries; rebuilding could publish different - # contents from those already delivered to the immutable container. - else - ensure_image_builder_rg || exit $? - run_image_builder_template || exit $? - fi + # for idempotency, check whether the target VHD already exists. If a previous run already produced it + # successfully, exit early; if a server-side copy is still in progress, wait for it to finish. Because + # the destination container may be immutable, a VHD left in a bad state cannot be deleted and recreated + # and is treated as fatal. + check_for_existing_vhd + + # attempt to perform prefetch optimization, distributing the optimized image as a VHD into the + # image builder staging storage account, then copy that VHD into the target storage account + ensure_image_builder_rg || exit $? + run_image_builder_template || exit $? copy_optimized_vhd || exit $? } check_for_existing_vhd() { - local exists copy_status - exists="$(az storage blob exists --blob-url "${VHD_URI}" --auth-mode login | jq -r '.exists')" || return $? - if [ "${exists}" = "false" ]; then + if [ "$(az storage blob exists --blob-url "${VHD_URI}" --auth-mode login | jq -r '.exists')" = "false" ]; then echo "no existing VHD was found at: ${VHD_URI}, will proceed with optimization and VHD creation" return 0 fi - if [ "${exists}" != "true" ]; then - echo "Unable to determine whether VHD ${VHD_URI} exists" >&2 - return 1 - fi echo "VHD already exists at: ${VHD_URI}" # the VHD in a potentially immutable container is produced by a server-side blob copy, so its copy status # tells us whether a previous run already completed. Because the container may have an immutability policy, # a VHD in a bad state cannot be deleted and recreated, so any non-success terminal state is fatal. - copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" || return $? + copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" if [ "${copy_status,,}" = "success" ]; then - echo "VHD ${VHD_URI} was already produced by a previous prefetch optimization run" - VHD_COPY_COMPLETE=true - return 0 + echo "VHD ${VHD_URI} was already produced by a previous prefetch optimization run, nothing to do" + exit 0 fi if [ "${copy_status,,}" = "pending" ]; then echo "VHD ${VHD_URI} is currently being copied by a previous prefetch optimization run, will wait for completion" if wait_for_vhd_copy; then - VHD_COPY_COMPLETE=true - return 0 + exit 0 fi fi # the VHD exists in a non-success state within the immutable container. It cannot be deleted due to # the immutability policy, so there is nothing we can do to recover - fail the script. echo "VHD ${VHD_URI} exists in an unrecoverable copy state: '${copy_status}'" - return 1 + exit 1 } ensure_image_builder_rg() { @@ -321,34 +308,6 @@ create_temp_storage() { # 1. azcopy the optimized VHD into the staging container (STORAGE_ACCOUNT_BLOB_URL_STAGING) # 2. perform a single server-side blob copy from the staging container into the immutable vhd container copy_optimized_vhd() { - local artifact_uri gallery_pid gallery_status=0 storage_status=0 - artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv)" || return $? - if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then - echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" - return 1 - fi - - "${GENERALIZED_CVM_PUBLISHER}" "${artifact_uri}" & - gallery_pid=$! - copy_vhd_to_storage "${artifact_uri}" || storage_status=$? - # Always join the publisher, even when storage delivery fails, so cleanup cannot race either reader. - wait "${gallery_pid}" || gallery_status=$? - if [ "${storage_status}" -ne 0 ] || [ "${gallery_status}" -ne 0 ]; then - echo "Post-prefetch delivery failed: storage=${storage_status}, gallery=${gallery_status}" >&2 - return 1 - fi - - delete_staging_vhd true || echo "unable to delete staging VHD ${STAGING_VHD_URI}, will proceed" - echo "optimized VHD and gallery publication are complete: ${VHD_URI}" -} - -copy_vhd_to_storage() { - local artifact_uri="${1:?Image Builder artifact URI is required}" - local staging_info staging_blob_name azcopy_exit_code - if [ "${VHD_COPY_COMPLETE}" = true ]; then - return 0 - fi - # The staging container should never have an immutability policy, so copying from image builder staging storage is # fully retriable. If a previous run already produced a complete staging blob (marked with the # prefetched metadata flag) but failed before finishing the server-side copy into the @@ -364,6 +323,11 @@ copy_vhd_to_storage() { delete_staging_vhd || return $? fi + artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv 2>/dev/null)" + if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then + echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" + return 1 + fi echo "setting azcopy environment variables with pool identity: ${IMAGE_BUILDER_IDENTITY_ID}" export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" export AZCOPY_CONCURRENCY_VALUE="AUTO" @@ -391,15 +355,21 @@ copy_vhd_to_storage() { --source-uri "${STAGING_VHD_URI}" || return $? wait_for_vhd_copy || return $? + + # the optimized VHD is now safely in the immutable vhd container, so the staging copy is no longer + # needed - delete it as best-effort cleanup so staging blobs do not accumulate in the destination account. + # Use no-wait since we don't need to wait for the blob to be deleted in this case. + delete_staging_vhd true || echo "unable to delete staging VHD ${STAGING_VHD_URI}, will proceed" + + echo "optimized VHD has been published to: ${VHD_URI}" } wait_for_vhd_copy() { - local copy_status - copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" || return $? + copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" while [ "${copy_status,,}" = "pending" ]; do echo "server-side copy to ${VHD_URI} is still pending, will wait 30s before checking again" sleep 30s - copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" || return $? + copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" done if [ "${copy_status,,}" != "success" ]; then echo "server-side copy to ${VHD_URI} finished with unexpected status: ${copy_status}" From 216aa1f6b88c6c3ea03b3c692185e0dc837b617a Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 14:20:39 -0400 Subject: [PATCH 19/37] fix --- spec/vhdbuilder/prefetch/publication_spec.sh | 45 +++++++++----------- vhdbuilder/prefetch/README.md | 16 ++++--- vhdbuilder/prefetch/scripts/optimize.sh | 28 +++++++++--- 3 files changed, 52 insertions(+), 37 deletions(-) diff --git a/spec/vhdbuilder/prefetch/publication_spec.sh b/spec/vhdbuilder/prefetch/publication_spec.sh index 6f23370e2a5..e6726b6bb2c 100644 --- a/spec/vhdbuilder/prefetch/publication_spec.sh +++ b/spec/vhdbuilder/prefetch/publication_spec.sh @@ -123,9 +123,6 @@ Describe 'Post-prefetch CVM publication' 'storage blob metadata'|'storage blob copy') ;; 'storage blob delete') - if [ "${MOCK_VERSION_STATE}" != succeeded ] && [ "${FEATURE_FLAGS}" = cvm ]; then - [ -f "${TEST_ROOT}/replication-ready" ] || return 91 - fi ;; 'resource show --ids') if [ "${MOCK_ARTIFACT_MISSING}" = true ]; then @@ -282,7 +279,7 @@ Describe 'Post-prefetch CVM publication' End Describe 'Parallel delivery and retries' - It 'reads the original blob concurrently and joins both operations before staging cleanup' + It 'reads the original blob concurrently and joins both operations before returning' MOCK_REQUIRE_OVERLAP=true MOCK_GALLERY_WAIT_FOR_STORAGE=true MOCK_COPY_PENDING_ONCE=true When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be success @@ -290,27 +287,26 @@ Describe 'Post-prefetch CVM publication' The output should include "--os-vhd-uri ${SOURCE_URI}" The output should include "storage blob copy start" The output should include 'sleep 30s' - The output should include 'optimized VHD and gallery publication are complete:' + The output should include 'provisioned successfully' The output should include 'storage blob delete' End - It 'waits for gallery publication even if the storage copy fails and blocks cleanup' + It 'waits for gallery publication even if the storage copy fails' MOCK_STORAGE_FAILURE=true MOCK_GALLERY_WAIT_FOR_STORAGE=true When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be failure - The output should include 'Post-prefetch delivery failed: storage=6, gallery=0' - The output should not include 'optimized VHD and gallery publication are complete:' + The status should equal 6 + The output should include 'failed to copy optimized VHD to staging location' + The output should include 'provisioned successfully' The output should not include 'storage blob delete' End - It 'waits for immutable storage delivery even if gallery publication fails' + It 'warns without failing and waits for immutable storage delivery if gallery publication fails' MOCK_GALLERY_FAILURE=true MOCK_REQUIRE_OVERLAP=true When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be failure + The status should be success The output should include 'server-side copy to https://destination.blob.core.windows.net/vhd/test.vhd completed successfully' - The output should include 'Post-prefetch delivery failed: storage=0, gallery=7' - The output should not include 'optimized VHD and gallery publication are complete:' - The output should not include 'storage blob delete' + The output should include '##vso[task.logissue type=warning]Generalized CVM gallery publication failed (exit 7)' + The output should include 'storage blob delete' End It 'blocks cleanup if the server-side immutable copy fails' @@ -318,28 +314,26 @@ Describe 'Post-prefetch CVM publication' When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be failure The output should include 'finished with unexpected status: failed' - The output should include 'Post-prefetch delivery failed: storage=1, gallery=0' - The output should not include 'optimized VHD and gallery publication are complete:' + The output should include 'provisioned successfully' The output should not include 'storage blob delete' End - It 'waits for a pending immutable copy on retry before confirming completed delivery' + It 'preserves the existing wait for a pending immutable copy on retry' MOCK_FINAL_EXISTS=true MOCK_VERSION_STATE=succeeded MOCK_COPY_PENDING_ONCE=true When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be success The output should include 'is currently being copied' The output should include 'completed successfully' - The output should include 'Both VHD delivery and gallery publication are already complete' The output should not include 'storage blob copy start' The output should not include 'resource show' End - It 'retries missing gallery publication without recopying or rebuilding an immutable VHD' + It 'preserves the existing early exit when an immutable VHD already exists' MOCK_FINAL_EXISTS=true When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be success - The output should include "--os-vhd-uri ${SOURCE_URI}" - The output should include 'optimized VHD and gallery publication are complete:' + The output should include 'nothing to do' + The output should not include 'image-version create' The output should not include 'azcopy copy' The output should not include 'storage blob copy start' The output should not include 'image builder show' @@ -358,18 +352,17 @@ Describe 'Post-prefetch CVM publication' MOCK_FINAL_EXISTS=true MOCK_VERSION_STATE=succeeded MOCK_ARTIFACT_MISSING=true When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be success - The output should include 'Both VHD delivery and gallery publication are already complete' + The output should include 'nothing to do' The output should not include 'resource show' End - It 'fails rather than rebuilding different bits if the original source is missing on retry' - MOCK_FINAL_EXISTS=true MOCK_ARTIFACT_MISSING=true + It 'fails before starting either delivery if the original source URI is missing' + MOCK_ARTIFACT_MISSING=true When call run_script vhdbuilder/prefetch/scripts/optimize.sh The status should be failure The output should include 'unable to determine artifactUri' - The output should not include 'optimized VHD and gallery publication are complete:' - The output should not include 'image builder show' The output should not include 'image-version create' + The output should not include 'azcopy copy' End End diff --git a/vhdbuilder/prefetch/README.md b/vhdbuilder/prefetch/README.md index 507d66947ef..c38724df6c2 100644 --- a/vhdbuilder/prefetch/README.md +++ b/vhdbuilder/prefetch/README.md @@ -10,11 +10,17 @@ in the source storage account's region. Replication includes East US, West Europ and the source region when different. The managed image used as Image Builder's input for TL/CVM builds is unchanged. -The prefetch task waits for both storage delivery and gallery replication to finish -before returning success. The existing cleanup task runs afterward only on success; -no additional cleanup guard or wait loop is needed. On failure, cleanup is blocked; -retries reuse the original output and do not skip missing gallery publication just -because the immutable VHD already exists. +`copy_optimized_vhd` and `copy_optimized_vhd_to_packer_gallery` receive the same +Image Builder artifact URI and run in parallel. The prefetch task waits for both +storage delivery and gallery replication, even if either fails. Storage delivery +failure is fatal; gallery publication failure emits a pipeline warning but does +not fail the task. Cleanup can run once both operations finish and storage delivery +has succeeded. +The storage function deletes its own staging copy after immutable delivery; +Image Builder's original VHD remains available until the later cleanup task. + +The existing retry behavior is unchanged: an already-completed immutable VHD +causes an early exit without checking gallery publication. The publisher uses Azure CLI's built-in wait for provisioning and replication. It does not inspect, delete, or recover failed image versions before publication; diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index f7ca0890757..939209ec644 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -68,7 +68,22 @@ main() { # image builder staging storage account, then copy that VHD into the target storage account ensure_image_builder_rg || exit $? run_image_builder_template || exit $? - copy_optimized_vhd || exit $? + + local artifact_uri storage_pid gallery_pid storage_status=0 + artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv)" || return $? + if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then + echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" + return 1 + fi + + copy_optimized_vhd "${artifact_uri}" & + storage_pid=$! + copy_optimized_vhd_to_packer_gallery "${artifact_uri}" & + gallery_pid=$! + + wait "${storage_pid}" || storage_status=$? + wait "${gallery_pid}" || echo "##vso[task.logissue type=warning]Generalized CVM gallery publication failed (exit $?)" + return "${storage_status}" } check_for_existing_vhd() { @@ -308,6 +323,7 @@ create_temp_storage() { # 1. azcopy the optimized VHD into the staging container (STORAGE_ACCOUNT_BLOB_URL_STAGING) # 2. perform a single server-side blob copy from the staging container into the immutable vhd container copy_optimized_vhd() { + local artifact_uri="${1:?Image Builder VHD URI is required}" # The staging container should never have an immutability policy, so copying from image builder staging storage is # fully retriable. If a previous run already produced a complete staging blob (marked with the # prefetched metadata flag) but failed before finishing the server-side copy into the @@ -323,11 +339,6 @@ copy_optimized_vhd() { delete_staging_vhd || return $? fi - artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv 2>/dev/null)" - if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then - echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" - return 1 - fi echo "setting azcopy environment variables with pool identity: ${IMAGE_BUILDER_IDENTITY_ID}" export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" export AZCOPY_CONCURRENCY_VALUE="AUTO" @@ -364,6 +375,11 @@ copy_optimized_vhd() { echo "optimized VHD has been published to: ${VHD_URI}" } +copy_optimized_vhd_to_packer_gallery() { + local artifact_uri="${1:?Image Builder VHD URI is required}" + "$(dirname "${BASH_SOURCE[0]}")/../../packer/publish-generalized-cvm-if-required.sh" "${artifact_uri}" +} + wait_for_vhd_copy() { copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" while [ "${copy_status,,}" = "pending" ]; do From bac5fe62226494ae3bfe600a058c8839e09b664d Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 14:27:51 -0400 Subject: [PATCH 20/37] fix --- vhdbuilder/prefetch/README.md | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 vhdbuilder/prefetch/README.md diff --git a/vhdbuilder/prefetch/README.md b/vhdbuilder/prefetch/README.md deleted file mode 100644 index c38724df6c2..00000000000 --- a/vhdbuilder/prefetch/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Prefetch - -Provides functionality to generate scripts used for prefetch optimization. - -After Image Builder finishes, its original post-prefetch VHD is used concurrently -for generalized CVM gallery publication and delivery to the final storage account -(via the mutable staging container and then the immutable VHD container). Gallery -versions are created directly from that VHD, without an intermediate managed image, -in the source storage account's region. Replication includes East US, West Europe, -and the source region when different. The managed image used as Image Builder's -input for TL/CVM builds is unchanged. - -`copy_optimized_vhd` and `copy_optimized_vhd_to_packer_gallery` receive the same -Image Builder artifact URI and run in parallel. The prefetch task waits for both -storage delivery and gallery replication, even if either fails. Storage delivery -failure is fatal; gallery publication failure emits a pipeline warning but does -not fail the task. Cleanup can run once both operations finish and storage delivery -has succeeded. -The storage function deletes its own staging copy after immutable delivery; -Image Builder's original VHD remains available until the later cleanup task. - -The existing retry behavior is unchanged: an already-completed immutable VHD -causes an early exit without checking gallery publication. - -The publisher uses Azure CLI's built-in wait for provisioning and replication. -It does not inspect, delete, or recover failed image versions before publication; -Azure CLI failures propagate to the prefetch task. \ No newline at end of file From bc0d6165f9afce9e645cb2ed13e85f68805d003a Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 14:30:49 -0400 Subject: [PATCH 21/37] fix --- vhdbuilder/prefetch/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 vhdbuilder/prefetch/README.md diff --git a/vhdbuilder/prefetch/README.md b/vhdbuilder/prefetch/README.md new file mode 100644 index 00000000000..6080ef1cf7e --- /dev/null +++ b/vhdbuilder/prefetch/README.md @@ -0,0 +1,3 @@ +# Prefetch + +Provides functionality to generate scripts used for prefetch optimization. \ No newline at end of file From e5e40fc1ec7b94d0ac896709512bb8721cc04ace Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 14:45:27 -0400 Subject: [PATCH 22/37] fix --- spec/vhdbuilder/prefetch/publication_spec.sh | 35 +++---------------- .../publish-generalized-cvm-if-required.sh | 18 ++-------- 2 files changed, 7 insertions(+), 46 deletions(-) diff --git a/spec/vhdbuilder/prefetch/publication_spec.sh b/spec/vhdbuilder/prefetch/publication_spec.sh index e6726b6bb2c..c3e5399e28c 100644 --- a/spec/vhdbuilder/prefetch/publication_spec.sh +++ b/spec/vhdbuilder/prefetch/publication_spec.sh @@ -184,18 +184,19 @@ Describe 'Post-prefetch CVM publication' The output should include "--os-vhd-uri ${SOURCE_URI}" The output should include '--os-vhd-storage-account /subscriptions/test-subscription/resourceGroups/IT_staging/providers/Microsoft.Storage/storageAccounts/sourceaccount' The output should include '--location northeurope' - The output should include '--target-regions eastus westeurope northeurope' + The output should include '--target-regions westus --tags' The output should include 'buildId=123' The output should not include '--managed-image' The output should not include 'image create' The output should not include 'image delete' End - It 'does not duplicate the source region when it is already a replication target' - MOCK_SOURCE_LOCATION=eastus + It 'replicates only to West US' When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" The status should be success - The output should include '--target-regions eastus westeurope --tags' + The output should include '--target-regions westus --tags' + The output should not include 'eastus' + The output should not include 'westeurope' End It 'creates a generalized Gen2 confidential-VM-supported definition when needed' @@ -225,25 +226,6 @@ Describe 'Post-prefetch CVM publication' The output should not include 'image-version create' End - It 'checks completion without mutating Azure resources' - MOCK_VERSION_STATE=missing - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh --check-only - The status should be failure - The output should include 'ResourceNotFound' - The output should not include 'image-definition list' - The output should not include 'image-version create' - The output should not include 'image-version delete' - End - - It 'surfaces authorization failures when checking completion' - MOCK_VERSION_READ_FAILURE=true - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh --check-only - The status should be failure - The output should include 'AuthorizationFailed' - The output should not include 'image-version create' - The output should not include 'image-version delete' - End - It 'does not inspect or delete failed versions before publishing' MOCK_VERSION_STATE=failed When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" @@ -262,13 +244,6 @@ Describe 'Post-prefetch CVM publication' The output should not include 'provisioned successfully' End - It 'does not treat incomplete replication as completed delivery' - MOCK_VERSION_STATE=succeeded MOCK_REPLICATION_PENDING_ONCE=true - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh --check-only - The status should be failure - The output should include 'publication is not complete' - End - It 'skips unsupported OS flavors without storage lookups' OS_SKU=AzureContainerLinux When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" diff --git a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh index b4a0325d017..9359100bac5 100755 --- a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh +++ b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh @@ -2,7 +2,7 @@ set -euo pipefail -SOURCE_VHD_URI="${1:?source VHD URI or --check-only is required}" +SOURCE_VHD_URI="${1:?source VHD URI is required}" if [[ "${FEATURE_FLAGS,,}" != *cvm* || ! "${OS_SKU,,}" =~ ^(ubuntu|azurelinux)$ ]]; then echo "Generalized CVM publication is not required for OS_SKU=${OS_SKU:-unset}, FEATURE_FLAGS=${FEATURE_FLAGS:-unset}" @@ -17,15 +17,6 @@ gallery_args=( definition_args=("${gallery_args[@]}" --gallery-image-definition "${SKU_NAME:?}") version_args=("${definition_args[@]}" --gallery-image-version "${CAPTURED_SIG_VERSION:?}") -if [[ "${SOURCE_VHD_URI}" = "--check-only" ]]; then - version_info="$(az sig image-version show "${version_args[@]}" --expand ReplicationStatus --output json)" - if ! jq -e '.provisioningState == "Succeeded" and .replicationStatus.aggregatedState == "Completed"' <<<"${version_info}" >/dev/null; then - echo "${SKU_NAME}/${CAPTURED_SIG_VERSION} publication is not complete" >&2 - exit 1 - fi - exit 0 -fi - if [[ ! "${SOURCE_VHD_URI}" =~ ^https://([a-z0-9]{3,24})\.blob\.[^/]+/.+ ]]; then echo "Source VHD must be an Azure Storage blob URL" >&2 exit 1 @@ -57,16 +48,11 @@ if [[ -z "${definition_id}" || "${definition_id}" = "None" ]]; then az resource update --ids "${definition_id}" --set properties.allowUpdateImage=true --only-show-errors >/dev/null fi -target_regions=(eastus westeurope) -if [[ "${source_location}" != "eastus" && "${source_location}" != "westeurope" ]]; then - target_regions+=("${source_location}") -fi - az sig image-version create "${version_args[@]}" \ --location "${source_location}" \ --os-vhd-uri "${SOURCE_VHD_URI}" \ --os-vhd-storage-account "${source_storage_id}" \ - --target-regions "${target_regions[@]}" \ + --target-regions westus \ --tags "buildId=${BUILD_BUILDID:?}" "createdBy=aks-vhd-pipeline" >/dev/null az sig image-version wait "${version_args[@]}" --expand ReplicationStatus \ From 6495b276cd7ad6ec862693cdb2d0ce7259f5ddab Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 14:57:24 -0400 Subject: [PATCH 23/37] fix --- spec/vhdbuilder/prefetch/publication_spec.sh | 352 ------------------- 1 file changed, 352 deletions(-) delete mode 100644 spec/vhdbuilder/prefetch/publication_spec.sh diff --git a/spec/vhdbuilder/prefetch/publication_spec.sh b/spec/vhdbuilder/prefetch/publication_spec.sh deleted file mode 100644 index c3e5399e28c..00000000000 --- a/spec/vhdbuilder/prefetch/publication_spec.sh +++ /dev/null @@ -1,352 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2329 - -Describe 'Post-prefetch CVM publication' - setup_environment() { - TEST_ROOT="$(mktemp -d "${SHELLSPEC_TMPBASE}/publication.XXXXXX")" - export TEST_ROOT - export AZ_MOCK_LOG="${TEST_ROOT}/az.log" - : > "${AZ_MOCK_LOG}" - export SUBSCRIPTION_ID=test-subscription LOCATION=westus PACKER_BUILD_LOCATION=westus - export SIG_GALLERY_RESOURCE_GROUP_NAME=gallery-rg AZURE_RESOURCE_GROUP_NAME=gallery-rg - export SIG_GALLERY_NAME=gallery SIG_IMAGE_NAME=specialized SKU_NAME=generalized - export STORAGE_ACCOUNT_BLOB_URL=https://destination.blob.core.windows.net/vhd - export STORAGE_ACCOUNT_BLOB_URL_STAGING=https://destination.blob.core.windows.net/vhdstaging - export VHD_NAME=test.vhd IMAGE_BUILDER_IDENTITY_ID=identity BUILD_RUN_NUMBER=run - export CAPTURED_SIG_VERSION=1.2.3 HYPERV_GENERATION=V2 FEATURE_FLAGS=cvm - export ENABLE_TRUSTED_LAUNCH=False WORKLOAD_OPTIMIZATION_SCRIPT_URL=https://example.test/optimize.sh - export OS_SKU=Ubuntu BUILD_BUILDID=123 - export SOURCE_URI=https://sourceaccount.blob.core.windows.net/vhds/original.vhd - export MOCK_SOURCE_LOCATION=westus MOCK_STORAGE_ACCOUNT_MISSING=false - export MOCK_FINAL_EXISTS=false MOCK_STAGING_COMPLETE=false MOCK_VERSION_STATE=missing - export MOCK_STORAGE_FAILURE=false MOCK_GALLERY_FAILURE=false MOCK_REQUIRE_OVERLAP=false - export MOCK_GALLERY_WAIT_FOR_STORAGE=false MOCK_REPLICATION_PENDING_ONCE=false - export MOCK_ARTIFACT_MISSING=false - export MOCK_DEFINITION_EXISTS=true MOCK_VERSION_READ_FAILURE=false - export MOCK_REPLICATION_FAILURE=false - export MOCK_COPY_PENDING_ONCE=false MOCK_COPY_STATUS=success - - wait_for_marker() { - local attempt - for ((attempt=0; attempt<200; attempt++)); do - [ -f "${TEST_ROOT}/$1" ] && return 0 - command sleep 0.01 - done - echo "Timed out waiting for mocked operation: $1" >&2 - return 90 - } - - az() { - printf '%s\n' "$*" >> "${AZ_MOCK_LOG}" - case "$1 $2 $3" in - 'sig image-definition list') - if [ "${MOCK_DEFINITION_EXISTS}" = false ] && [ ! -f "${TEST_ROOT}/definition-created" ]; then - return 0 - fi - echo definition - ;; - 'sig image-definition create') - touch "${TEST_ROOT}/definition-created" - echo definition - ;; - 'resource update --ids') - ;; - 'sig image-version show') - if [ "${MOCK_VERSION_READ_FAILURE}" = true ]; then - echo '(AuthorizationFailed) Access denied' >&2 - return 4 - fi - if [ "${MOCK_VERSION_STATE}" = succeeded ] || [ -f "${TEST_ROOT}/version-created" ]; then - if [ "${MOCK_REPLICATION_FAILURE}" = true ]; then - echo '{"provisioningState":"Succeeded","replicationStatus":{"aggregatedState":"Failed"}}' - elif [ "${MOCK_REPLICATION_PENDING_ONCE}" = true ] && [ ! -f "${TEST_ROOT}/replication-ready" ]; then - echo '{"provisioningState":"Succeeded","replicationStatus":{"aggregatedState":"InProgress"}}' - else - echo '{"provisioningState":"Succeeded","replicationStatus":{"aggregatedState":"Completed"}}' - fi - elif [ "${MOCK_VERSION_STATE}" = failed ]; then - echo '{"provisioningState":"Failed","replicationStatus":{"aggregatedState":"Failed"}}' - else - echo '(ResourceNotFound) Image version not found' >&2 - return 3 - fi - ;; - 'sig image-version create') - touch "${TEST_ROOT}/gallery-started" - if [ "${MOCK_REQUIRE_OVERLAP}" = true ]; then - wait_for_marker storage-started || return $? - fi - if [ "${MOCK_GALLERY_WAIT_FOR_STORAGE}" = true ]; then - wait_for_marker storage-finished || return $? - fi - touch "${TEST_ROOT}/gallery-finished" - [ "${MOCK_GALLERY_FAILURE}" = true ] && return 7 - touch "${TEST_ROOT}/version-created" - ;; - 'sig image-version wait') - if [ "${MOCK_REPLICATION_PENDING_ONCE}" = true ]; then - sleep 60 - fi - if [ "${MOCK_REPLICATION_FAILURE}" = true ]; then - echo 'Gallery replication did not complete' >&2 - return 8 - fi - touch "${TEST_ROOT}/replication-ready" - ;; - 'storage account list') - if [ "${MOCK_STORAGE_ACCOUNT_MISSING}" = true ]; then - echo null - else - printf '{"id":"/subscriptions/test-subscription/resourceGroups/IT_staging/providers/Microsoft.Storage/storageAccounts/sourceaccount","location":"%s"}\n' "${MOCK_SOURCE_LOCATION}" - fi - ;; - 'storage blob exists') - printf '{"exists":%s}\n' "${MOCK_FINAL_EXISTS}" - ;; - 'storage blob show') - if [[ "$*" = *'/vhdstaging/'* ]]; then - if [ "${MOCK_STAGING_COMPLETE}" = true ]; then - echo '{"name":"test.vhd","metadata":{"prefetched":"true"}}' - else - echo '{}' - fi - else - if [ "${MOCK_COPY_PENDING_ONCE}" = true ] && [ ! -f "${TEST_ROOT}/copy-pending-checked" ]; then - touch "${TEST_ROOT}/copy-pending-checked" - echo '{"properties":{"copy":{"status":"pending"}}}' - else - touch "${TEST_ROOT}/storage-finished" - printf '{"properties":{"copy":{"status":"%s"}}}\n' "${MOCK_COPY_STATUS}" - fi - fi - ;; - 'storage blob metadata'|'storage blob copy') - ;; - 'storage blob delete') - ;; - 'resource show --ids') - if [ "${MOCK_ARTIFACT_MISSING}" = true ]; then - echo null - else - echo "${SOURCE_URI}" - fi - ;; - 'group exists -g') - echo true - ;; - 'image builder show') - echo '{"provisioningState":"Succeeded","lastRunStatus":{"runState":"Succeeded"}}' - ;; - 'image builder wait') - ;; - 'group delete -g') - ;; - *) - echo "Unexpected Azure call: $*" >&2 - return 99 - ;; - esac - } - - azcopy() { - printf 'azcopy %s\n' "$*" >> "${AZ_MOCK_LOG}" - touch "${TEST_ROOT}/storage-started" - if [ "${MOCK_REQUIRE_OVERLAP}" = true ]; then - wait_for_marker gallery-started || return $? - fi - if [ "${MOCK_STORAGE_FAILURE}" = true ]; then - touch "${TEST_ROOT}/storage-finished" - return 6 - fi - } - - sleep() { - printf 'sleep %s\n' "$*" >> "${AZ_MOCK_LOG}" - touch "${TEST_ROOT}/replication-ready" - } - - export -f az azcopy sleep wait_for_marker - } - BeforeEach 'setup_environment' - - run_script() { - local status=0 - bash "$@" 2>&1 || status=$? - cat "${AZ_MOCK_LOG}" - return "${status}" - } - - Describe 'Direct VHD gallery versions' - It 'uses the original VHD, account ID and actual account region without a managed image' - MOCK_SOURCE_LOCATION=northeurope - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" - The status should be success - The output should include "--os-vhd-uri ${SOURCE_URI}" - The output should include '--os-vhd-storage-account /subscriptions/test-subscription/resourceGroups/IT_staging/providers/Microsoft.Storage/storageAccounts/sourceaccount' - The output should include '--location northeurope' - The output should include '--target-regions westus --tags' - The output should include 'buildId=123' - The output should not include '--managed-image' - The output should not include 'image create' - The output should not include 'image delete' - End - - It 'replicates only to West US' - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" - The status should be success - The output should include '--target-regions westus --tags' - The output should not include 'eastus' - The output should not include 'westeurope' - End - - It 'creates a generalized Gen2 confidential-VM-supported definition when needed' - MOCK_DEFINITION_EXISTS=false - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" - The status should be success - The output should include '--os-state Generalized' - The output should include '--hyper-v-generation V2' - The output should include 'SecurityType=ConfidentialVMSupported' - End - - It 'waits for replication even when provisioning has succeeded' - MOCK_REPLICATION_PENDING_ONCE=true - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" - The status should be success - The output should include 'sig image-version wait' - The output should include '--expand ReplicationStatus --custom' - The output should include "provisioningState=='Succeeded' && replicationStatus.aggregatedState=='Completed'" - The output should include 'sleep 60' - End - - It 'fails explicitly if the source storage metadata is unavailable' - MOCK_STORAGE_ACCOUNT_MISSING=true - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" - The status should be failure - The output should include 'Unable to resolve source storage account' - The output should not include 'image-version create' - End - - It 'does not inspect or delete failed versions before publishing' - MOCK_VERSION_STATE=failed - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" - The status should be success - The output should include 'sig image-version create' - The output should not include 'sig image-version show' - The output should not include 'sig image-version delete' - The output should not include '--deleted' - End - - It 'does not succeed when gallery replication fails' - MOCK_REPLICATION_FAILURE=true - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" - The status should be failure - The output should include 'Gallery replication did not complete' - The output should not include 'provisioned successfully' - End - - It 'skips unsupported OS flavors without storage lookups' - OS_SKU=AzureContainerLinux - When call run_script vhdbuilder/packer/publish-generalized-cvm-if-required.sh "${SOURCE_URI}" - The status should be success - The output should include 'publication is not required' - The output should not include 'storage account list' - End - End - - Describe 'Parallel delivery and retries' - It 'reads the original blob concurrently and joins both operations before returning' - MOCK_REQUIRE_OVERLAP=true MOCK_GALLERY_WAIT_FOR_STORAGE=true MOCK_COPY_PENDING_ONCE=true - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be success - The output should include "azcopy copy ${SOURCE_URI} ${STORAGE_ACCOUNT_BLOB_URL_STAGING}/${VHD_NAME}" - The output should include "--os-vhd-uri ${SOURCE_URI}" - The output should include "storage blob copy start" - The output should include 'sleep 30s' - The output should include 'provisioned successfully' - The output should include 'storage blob delete' - End - - It 'waits for gallery publication even if the storage copy fails' - MOCK_STORAGE_FAILURE=true MOCK_GALLERY_WAIT_FOR_STORAGE=true - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should equal 6 - The output should include 'failed to copy optimized VHD to staging location' - The output should include 'provisioned successfully' - The output should not include 'storage blob delete' - End - - It 'warns without failing and waits for immutable storage delivery if gallery publication fails' - MOCK_GALLERY_FAILURE=true MOCK_REQUIRE_OVERLAP=true - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be success - The output should include 'server-side copy to https://destination.blob.core.windows.net/vhd/test.vhd completed successfully' - The output should include '##vso[task.logissue type=warning]Generalized CVM gallery publication failed (exit 7)' - The output should include 'storage blob delete' - End - - It 'blocks cleanup if the server-side immutable copy fails' - MOCK_COPY_STATUS=failed - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be failure - The output should include 'finished with unexpected status: failed' - The output should include 'provisioned successfully' - The output should not include 'storage blob delete' - End - - It 'preserves the existing wait for a pending immutable copy on retry' - MOCK_FINAL_EXISTS=true MOCK_VERSION_STATE=succeeded MOCK_COPY_PENDING_ONCE=true - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be success - The output should include 'is currently being copied' - The output should include 'completed successfully' - The output should not include 'storage blob copy start' - The output should not include 'resource show' - End - - It 'preserves the existing early exit when an immutable VHD already exists' - MOCK_FINAL_EXISTS=true - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be success - The output should include 'nothing to do' - The output should not include 'image-version create' - The output should not include 'azcopy copy' - The output should not include 'storage blob copy start' - The output should not include 'image builder show' - End - - It 'resolves the original blob even when the destination staging copy is reusable' - MOCK_STAGING_COMPLETE=true - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be success - The output should include "--os-vhd-uri ${SOURCE_URI}" - The output should include 'storage blob copy start' - The output should not include 'azcopy copy' - End - - It 'allows an already-completed retry after Image Builder resources have been removed' - MOCK_FINAL_EXISTS=true MOCK_VERSION_STATE=succeeded MOCK_ARTIFACT_MISSING=true - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be success - The output should include 'nothing to do' - The output should not include 'resource show' - End - - It 'fails before starting either delivery if the original source URI is missing' - MOCK_ARTIFACT_MISSING=true - When call run_script vhdbuilder/prefetch/scripts/optimize.sh - The status should be failure - The output should include 'unable to determine artifactUri' - The output should not include 'image-version create' - The output should not include 'azcopy copy' - End - End - - Describe 'Image Builder cleanup' - It 'retains resource group cleanup without additional prerequisites' - When call run_script vhdbuilder/prefetch/scripts/cleanup.sh - The status should be success - The output should include 'group delete -g image-builder-1.2.3-run --yes --no-wait' - The output should not include 'image builder delete' - End - End -End From 1a3b41ee1bf4a2fe33a64984fc5eb9e198be908a Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 15:13:44 -0400 Subject: [PATCH 24/37] fix --- .pipelines/templates/.builder-release-template.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 5e27f39dbd5..3f8ab1ca2fc 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -475,8 +475,6 @@ steps: export AZCOPY_CONCURRENCY_VALUE="AUTO" az storage blob copy start --account-name "$STORAGE_ACCOUNT_NAME" --destination-blob "${CAPTURED_SIG_VERSION}.vhd" --destination-container "$VHD_CONTAINER_NAME" --source-uri "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --auth-mode login || exit 1 echo "Successfully copied to immutable container" - ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ - "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" || exit 1 # Remove old VHD from staging container after copy to immutable container is complete azcopy remove "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true condition: | @@ -490,15 +488,6 @@ steps: ) ) displayName: Copy to Immutable Storage Container - env: - OS_SKU: $(OS_SKU) - FEATURE_FLAGS: $(FEATURE_FLAGS) - AZURE_RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) - SIG_GALLERY_NAME: $(SIG_GALLERY_NAME) - SKU_NAME: $(SKU_NAME) - CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) - PACKER_BUILD_LOCATION: $(PACKER_BUILD_LOCATION) - BUILD_BUILDID: $(Build.BuildId) - task: DownloadPipelineArtifact@2 displayName: Download Build Performance Program From 23af06ba21346daf357f527e71bca3105de5668a Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Fri, 16 Oct 2026 15:25:15 -0400 Subject: [PATCH 25/37] fix --- e2e/scenario/scenario_cvm.go | 2 +- e2e/scenario/scenario_cvm_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/scenario/scenario_cvm.go b/e2e/scenario/scenario_cvm.go index f281f098279..72cc3d7a238 100644 --- a/e2e/scenario/scenario_cvm.go +++ b/e2e/scenario/scenario_cvm.go @@ -10,7 +10,7 @@ import ( ) const ( - cvmLocation = "westeurope" + cvmLocation = "westus" cvmSNPSKU = "Standard_DC2ads_v5" cvmTDXSKU = "Standard_DC2eds_v6" ) diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go index dafdc7bcdec..5e3cd9997aa 100644 --- a/e2e/scenario/scenario_cvm_test.go +++ b/e2e/scenario/scenario_cvm_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestNewCVMScenarioUsesWestEuropeAndDefaultValidation(t *testing.T) { +func TestNewCVMScenarioUsesWestUSAndDefaultValidation(t *testing.T) { s := newCVMScenario( "test", "description", @@ -21,7 +21,7 @@ func TestNewCVMScenarioUsesWestEuropeAndDefaultValidation(t *testing.T) { armcompute.SecurityEncryptionTypesVMGuestStateOnly, ) - assert.Equal(t, cvmLocation, s.Location) + assert.Equal(t, "westus", s.Location) assert.True(t, s.Tags.CVM) assert.Equal(t, config.VHDUbuntu2404CVMGen2Containerd, s.VHD) assert.False(t, s.SkipDefaultValidation) @@ -74,7 +74,7 @@ func TestRegisteredCVMScenarios(t *testing.T) { continue } found[s.Name] = true - assert.Equal(t, cvmLocation, s.Location) + assert.Equal(t, "westus", s.Location) assert.Equal(t, want.vhd, s.VHD) assert.Equal(t, want.useNVMe, s.UseNVMe) assert.False(t, s.SkipDefaultValidation) From e5ea5167d96f53af321e9853b678b5fbfc29916a Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Mon, 19 Oct 2026 11:44:46 -0400 Subject: [PATCH 26/37] fix --- .../publish-generalized-cvm-if-required.sh | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh index 9359100bac5..05fdab07e1b 100755 --- a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh +++ b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh @@ -22,6 +22,16 @@ if [[ ! "${SOURCE_VHD_URI}" =~ ^https://([a-z0-9]{3,24})\.blob\.[^/]+/.+ ]]; the exit 1 fi source_storage_account="${BASH_REMATCH[1]}" +publication_stage="resolving source storage account (az storage account list)" +report_publication_failure() { + local exit_code=$? + echo "ERROR: Generalized CVM publication failed during ${publication_stage} (exit ${exit_code})" >&2 + exit "${exit_code}" +} +trap report_publication_failure ERR + +echo "Publishing generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} in resource group ${AZURE_RESOURCE_GROUP_NAME}" +echo "Resolving source storage account ${source_storage_account}" source_storage_info="$( az storage account list \ --query "[?name=='${source_storage_account}'].{id:id,location:location} | [0]" \ @@ -34,9 +44,14 @@ if [[ -z "${source_storage_id}" || -z "${source_location}" ]]; then echo "Unable to resolve source storage account ${source_storage_account} and its location" >&2 exit 1 fi +echo "Resolved source storage account ${source_storage_account} in ${source_location}" +publication_stage="looking up image definition ${SIG_GALLERY_NAME}/${SKU_NAME} (az sig image-definition list)" +echo "Looking up image definition ${SIG_GALLERY_NAME}/${SKU_NAME}" definition_id="$(az sig image-definition list "${gallery_args[@]}" --query "[?name=='${SKU_NAME}'].id | [0]" --output tsv)" if [[ -z "${definition_id}" || "${definition_id}" = "None" ]]; then + publication_stage="creating image definition ${SIG_GALLERY_NAME}/${SKU_NAME} (az sig image-definition create)" + echo "Image definition not found; creating generalized image definition ${SIG_GALLERY_NAME}/${SKU_NAME}" definition_id="$( az sig image-definition create "${definition_args[@]}" \ --publisher microsoft-aks --offer "${SIG_GALLERY_NAME}" --sku "${SKU_NAME}" \ @@ -45,16 +60,28 @@ if [[ -z "${definition_id}" || "${definition_id}" = "None" ]]; then --features "DiskControllerTypes=SCSI,NVMe" "SecurityType=ConfidentialVMSupported" \ --query id --output tsv )" + echo "Created image definition ${definition_id}" + publication_stage="enabling image definition updates (az resource update)" + echo "Enabling allowUpdateImage on ${definition_id}" az resource update --ids "${definition_id}" --set properties.allowUpdateImage=true --only-show-errors >/dev/null + echo "Enabled allowUpdateImage on ${definition_id}" +else + echo "Using existing image definition ${definition_id}" fi +publication_stage="creating image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} (az sig image-version create)" +CREATE_TIME="$(date +%s)" +echo "Creating image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} with buildId=${BUILD_BUILDID:?}, now=${CREATE_TIME}, source location ${source_location}, target region westus" az sig image-version create "${version_args[@]}" \ --location "${source_location}" \ --os-vhd-uri "${SOURCE_VHD_URI}" \ --os-vhd-storage-account "${source_storage_id}" \ --target-regions westus \ - --tags "buildId=${BUILD_BUILDID:?}" "createdBy=aks-vhd-pipeline" >/dev/null + --tags "buildId=${BUILD_BUILDID:?}" "createdBy=aks-vhd-pipeline" "now=${CREATE_TIME}" >/dev/null +echo "Image version create succeeded for ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}" +publication_stage="waiting for image version provisioning and replication (az sig image-version wait)" +echo "Waiting for image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}: provisioningState=Succeeded and replicationStatus.aggregatedState=Completed" az sig image-version wait "${version_args[@]}" --expand ReplicationStatus \ --custom "provisioningState=='Succeeded' && replicationStatus.aggregatedState=='Completed'" \ --interval 60 From c7b573a588a769d9d774d05c80a2e29423b92e5f Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Mon, 19 Oct 2026 12:47:38 -0400 Subject: [PATCH 27/37] fix --- .../templates/.builder-release-template.yaml | 4 + .../packer/publish_generalized_cvm_spec.sh | 152 ++++++++++++++++++ .../publish-generalized-cvm-if-required.sh | 42 ++++- 3 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 3f8ab1ca2fc..b1538cb4b10 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -366,6 +366,10 @@ steps: IMAGE_BUILDER_IDENTITY_ID: $(AZURE_MSI_RESOURCE_STRING) BUILD_RUN_NUMBER: $(Build.BuildNumber) BUILD_BUILDID: $(Build.BuildId) + BUILD_DEFINITION_NAME: $(Build.DefinitionName) + BUILD_NUMBER: $(Build.BuildNumber) + BRANCH: $(Build.SourceBranch) + IMG_SKU: $(IMG_SKU) CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) HYPERV_GENERATION: $(HYPERV_GENERATION) FEATURE_FLAGS: $(FEATURE_FLAGS) diff --git a/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh b/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh new file mode 100644 index 00000000000..105612c4525 --- /dev/null +++ b/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env shellspec +# shellcheck shell=bash disable=SC2329 + +Describe 'publish-generalized-cvm-if-required.sh' + setup() { + export FEATURE_FLAGS=cvm OS_SKU=Ubuntu + export AZURE_RESOURCE_GROUP_NAME=test-rg SIG_GALLERY_NAME=test-gallery + export SKU_NAME=2404gen2CVMcontainerd CAPTURED_SIG_VERSION=1.2.3 + export PACKER_BUILD_LOCATION=eastus BUILD_BUILDID=123 + export BUILD_DEFINITION_NAME='AKS Linux VHD Build - TME Tenant' + export BUILD_NUMBER=20261019.1 BRANCH=refs/heads/zb/CVMABE2E IMG_SKU=server + export TAGS_FILE="${SHELLSPEC_WORKDIR}/cvm-version-tags" + export MOCK_DEFINITION_ID=/test/image-definition + export MOCK_FAIL_COMMAND='' MOCK_STATE=Succeeded + export MOCK_REPLICATION=Completed MOCK_REGION=westus MOCK_REGION_STATE=Completed + export -f az + } + BeforeEach 'setup' + + az() { + local command="${1} ${2} ${3}" + if [[ "${command}" = "${MOCK_FAIL_COMMAND}" ]]; then + echo "Mock Azure operation failed: ${command}" >&2 + return 37 + fi + case "${command}" in + 'storage account list') + echo '{"id":"/test/storage-account","location":"eastus"}' + ;; + 'sig image-definition list') + echo "${MOCK_DEFINITION_ID}" + ;; + 'sig image-definition create') + echo /test/new-image-definition + ;; + 'resource update --ids') + ;; + 'sig image-version create') + while [[ "$#" -gt 0 ]]; do + if [[ "$1" = --tags ]]; then + shift + printf '%s\n' "$@" >"${TAGS_FILE}" + break + fi + shift + done + ;; + 'sig image-version wait') + echo '{}' + ;; + 'sig image-version show') + jq -nc --arg state "${MOCK_STATE}" --arg replication "${MOCK_REPLICATION}" \ + --arg region "${MOCK_REGION}" --arg regionState "${MOCK_REGION_STATE}" \ + '{state:$state,replication:{aggregatedState:$replication,summary:[{region:$region,state:$regionState}]}}' + ;; + *) + echo "Unexpected Azure command: ${command}" >&2 + return 99 + ;; + esac + } + + publish_and_check_tags() { + bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ + https://teststorage.blob.core.windows.net/vhds/test.vhd && + jq -Rsc ' + split("\n") | map(select(length > 0) | capture("^(?[^=]+)=(?.*)$")) | from_entries | + .now as $now | + . == { + buildDefinitionName: env.BUILD_DEFINITION_NAME, + buildNumber: env.BUILD_NUMBER, + buildId: env.BUILD_BUILDID, + SkipLinuxAzSecPack: "true", + os: "Linux", + now: $now, + createdBy: "aks-vhd-pipeline", + image_sku: env.IMG_SKU, + branch: env.BRANCH + } and ($now | test("^[0-9]+$")) and + (($now | tonumber) <= now) and (($now | tonumber) > now - 60) + ' "${TAGS_FILE}" | grep -qx true + } + + Describe 'ready images' + Parameters + Ubuntu /test/image-definition server + AzureLinux None azure-linux-3-cvm + End + + It "publishes $1 with all standard version tags and verified readiness" + OS_SKU="$1" MOCK_DEFINITION_ID="$2" IMG_SKU="$3" + When call publish_and_check_tags + The status should be success + The output should include 'Image version creation completed' + The output should include 'replicated successfully to westus' + The output should include 'image ready in westus' + The output should not include 'provisioned' + End + End + + Describe 'unready images after a zero-exit wait' + Parameters + Updating Completed westus Completed + Succeeded InProgress westus InProgress + Succeeded Completed eastus Completed + Succeeded Completed westus InProgress + Failed Failed westus Failed + End + + It "rejects state=$1, replication=$2, region=$3, regional state=$4" + MOCK_STATE="$1" MOCK_REPLICATION="$2" MOCK_REGION="$3" MOCK_REGION_STATE="$4" + When run bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ + https://teststorage.blob.core.windows.net/vhds/test.vhd + The status should be failure + The output should include 'Verifying image version' + The output should not include 'image ready in westus' + The output should not include 'replicated successfully' + The stderr should include 'is not ready in westus after waiting for replication' + End + End + + Describe 'Azure operation failures' + Parameters + 'storage account list' 'resolving source storage account' + 'sig image-definition list' 'looking up image definition' + 'sig image-definition create' 'creating image definition' + 'resource update --ids' 'enabling image definition updates' + 'sig image-version create' 'creating image version' + 'sig image-version wait' 'waiting for image version replication' + 'sig image-version show' 'verifying image version readiness' + End + + It "reports failure during $2 without declaring readiness" + MOCK_DEFINITION_ID=None MOCK_FAIL_COMMAND="$1" + When run bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ + https://teststorage.blob.core.windows.net/vhds/test.vhd + The status should equal 37 + The output should include 'Publishing generalized CVM image' + The output should not include 'image ready in westus' + The output should not include 'replicated successfully' + The stderr should include "failed during $2" + End + End + + It 'skips images that do not need generalized CVM publication' + OS_SKU=AzureContainerLinux + When run bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ + https://teststorage.blob.core.windows.net/vhds/test.vhd + The status should be success + The output should include 'Generalized CVM publication is not required' + End +End diff --git a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh index 05fdab07e1b..14cc5dcd47e 100755 --- a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh +++ b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh @@ -71,19 +71,49 @@ fi publication_stage="creating image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} (az sig image-version create)" CREATE_TIME="$(date +%s)" +version_tags=( + "buildDefinitionName=${BUILD_DEFINITION_NAME:?}" + "buildNumber=${BUILD_NUMBER:?}" + "buildId=${BUILD_BUILDID:?}" + "SkipLinuxAzSecPack=true" + "os=Linux" + "now=${CREATE_TIME}" + "createdBy=aks-vhd-pipeline" + "image_sku=${IMG_SKU:?}" + "branch=${BRANCH:?}" +) echo "Creating image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} with buildId=${BUILD_BUILDID:?}, now=${CREATE_TIME}, source location ${source_location}, target region westus" +echo "Image version tags: ${version_tags[*]}" az sig image-version create "${version_args[@]}" \ --location "${source_location}" \ --os-vhd-uri "${SOURCE_VHD_URI}" \ --os-vhd-storage-account "${source_storage_id}" \ --target-regions westus \ - --tags "buildId=${BUILD_BUILDID:?}" "createdBy=aks-vhd-pipeline" "now=${CREATE_TIME}" >/dev/null -echo "Image version create succeeded for ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}" + --tags "${version_tags[@]}" >/dev/null +echo "Image version creation completed for ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}; checking replication" -publication_stage="waiting for image version provisioning and replication (az sig image-version wait)" -echo "Waiting for image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}: provisioningState=Succeeded and replicationStatus.aggregatedState=Completed" +publication_stage="waiting for image version replication (az sig image-version wait)" +echo "Waiting for image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replication to westus to complete" az sig image-version wait "${version_args[@]}" --expand ReplicationStatus \ --custom "provisioningState=='Succeeded' && replicationStatus.aggregatedState=='Completed'" \ - --interval 60 + --interval 60 --timeout 3600 >/dev/null + +publication_stage="verifying image version readiness (az sig image-version show)" +echo "Verifying image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} is ready in westus" +image_status="$( + az sig image-version show "${version_args[@]}" --expand ReplicationStatus \ + --query '{state:provisioningState,replication:replicationStatus}' --output json +)" +echo "Image version status: ${image_status}" +# The CLI wait command can exit zero on timeout; verify the actual resource state. +if ! jq -e ' + .state == "Succeeded" and + .replication.aggregatedState == "Completed" and + any(.replication.summary[]?; .region == "westus" and .state == "Completed") +' <<<"${image_status}" >/dev/null; then + echo "ERROR: Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} is not ready in westus after waiting for replication" >&2 + exit 1 +fi -echo "Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} provisioned successfully" +echo "Image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replicated successfully to westus" +echo "Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}: image ready in westus" From 125f5d631dfa2554c3f9c6f5a3a7a40059d278cd Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Mon, 19 Oct 2026 15:55:28 -0400 Subject: [PATCH 28/37] fix --- .pipelines/scripts/create-e2e-vhd-metadata.sh | 64 +++++++ .../templates/.builder-release-template.yaml | 68 +++---- e2e/README.md | 11 ++ spec/e2e/create_vhd_metadata_spec.sh | 175 ++++++++++++++++++ 4 files changed, 278 insertions(+), 40 deletions(-) create mode 100644 .pipelines/scripts/create-e2e-vhd-metadata.sh create mode 100644 spec/e2e/create_vhd_metadata_spec.sh diff --git a/.pipelines/scripts/create-e2e-vhd-metadata.sh b/.pipelines/scripts/create-e2e-vhd-metadata.sh new file mode 100644 index 00000000000..45aa80cecb9 --- /dev/null +++ b/.pipelines/scripts/create-e2e-vhd-metadata.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -euo pipefail + +metadata_file="${1:?metadata output path is required}" + +if [ -z "${MANAGED_SIG_ID:-}" ]; then + echo "##vso[task.logissue type=error]Packer did not return a managed SIG image version ID" >&2 + exit 1 +fi + +regions="$(jq -nc \ + --arg buildLocation "${PACKER_BUILD_LOCATION:?}" \ + --arg replications "${REPLICATIONS:-}" \ + '([$buildLocation] + ($replications | split(",") | map(split("=")[0] | gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)))) | unique')" + +metadata="$(jq -nc \ + --arg imageName "${SIG_IMAGE_NAME:?}" \ + --arg resourceId "${MANAGED_SIG_ID}" \ + --arg version "${CAPTURED_SIG_VERSION:?}" \ + --argjson regions "${regions}" \ + '{($imageName): {resourceId: $resourceId, version: $version, regions: $regions}}')" + +case "${OS_SKU,,}:${FEATURE_FLAGS,,}" in + ubuntu:*cvm*|azurelinux:*cvm*) + # Preserve the capture entry; query the generalized version in the same gallery. + gallery_id="${MANAGED_SIG_ID%/images/*}" + generalized_id="${gallery_id}/images/${SKU_NAME:?}/versions/${CAPTURED_SIG_VERSION}" + echo "Reading generalized CVM image ${SKU_NAME}/${CAPTURED_SIG_VERSION} for E2E metadata" + generalized_version="$( + az sig image-version show --ids "${generalized_id}" --expand ReplicationStatus \ + --output json --only-show-errors + )" || { + exit_code=$? + echo "##vso[task.logissue type=error]Unable to read generalized CVM image ${generalized_id} for E2E metadata" >&2 + exit "${exit_code}" + } + + if ! generalized_entry="$(jq -ce --arg expectedId "${generalized_id}" --arg version "${CAPTURED_SIG_VERSION}" ' + select((.id | type) == "string") | + select((.id | ascii_downcase) == ($expectedId | ascii_downcase)) | + select(.provisioningState == "Succeeded" and .replicationStatus.aggregatedState == "Completed") | + { + resourceId: .id, + version: $version, + regions: ([.replicationStatus.summary[]? | + select(.state == "Completed") | .region | + select(type == "string" and length > 0)] | unique) + } | + select(.regions | length > 0) + ' <<<"${generalized_version}")"; then + echo "##vso[task.logissue type=error]Generalized CVM image ${generalized_id} is not ready with confirmed replica regions; E2E metadata will not be published" >&2 + exit 1 + fi + + metadata="$(jq -c --arg imageName "${SKU_NAME}" --argjson entry "${generalized_entry}" \ + '. + {($imageName): $entry}' <<<"${metadata}")" + echo "Added image ready for E2Es: ${SKU_NAME}/${CAPTURED_SIG_VERSION}, replicated regions: $(jq -r '.regions | join(", ")' <<<"${generalized_entry}")" + ;; +esac + +mkdir -p "$(dirname "${metadata_file}")" +printf '%s\n' "${metadata}" >"${metadata_file}" +echo "Created E2E VHD metadata: ${metadata_file}" diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index b1538cb4b10..72dbdbe13ed 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -294,46 +294,6 @@ steps: RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) DRY_RUN: $(DRY_RUN) - - bash: | - set -euo pipefail - - metadata_dir="$(Build.ArtifactStagingDirectory)/e2e-vhd-metadata" - metadata_file="${metadata_dir}/${VHD_ARTIFACT_NAME}.json" - mkdir -p "$metadata_dir" - - if [ -z "${MANAGED_SIG_ID}" ]; then - echo "##vso[task.logissue type=error]Packer did not return a managed SIG image version ID" >&2 - exit 1 - fi - - regions="$(jq -nc \ - --arg buildLocation "${PACKER_BUILD_LOCATION}" \ - --arg replications "${REPLICATIONS:-}" \ - '([$buildLocation] + ($replications | split(",") | map(split("=")[0] | gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)))) | unique')" - - jq -n \ - --arg imageName "${SIG_IMAGE_NAME}" \ - --arg resourceId "${MANAGED_SIG_ID}" \ - --arg version "${CAPTURED_SIG_VERSION}" \ - --argjson regions "$regions" \ - '{($imageName): {resourceId: $resourceId, version: $version, regions: $regions}}' > "$metadata_file" - condition: succeeded() - displayName: Create E2E VHD metadata - env: - VHD_ARTIFACT_NAME: ${{ parameters.artifactName }} - SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) - CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) - MANAGED_SIG_ID: $(MANAGED_SIG_ID) - PACKER_BUILD_LOCATION: $(PACKER_BUILD_LOCATION) - REPLICATIONS: $(REPLICATIONS) - - - task: PublishPipelineArtifact@0 - condition: succeeded() - displayName: Publish E2E VHD metadata - inputs: - artifactName: 'e2e-vhd-metadata-${{ parameters.artifactName }}-stage$(System.StageAttempt)-job$(System.JobAttempt)' - targetPath: '$(Build.ArtifactStagingDirectory)/e2e-vhd-metadata/${{ parameters.artifactName }}.json' - - task: AzureCLI@2 inputs: azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) @@ -493,6 +453,34 @@ steps: ) displayName: Copy to Immutable Storage Container + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + bash .pipelines/scripts/create-e2e-vhd-metadata.sh \ + "$(Build.ArtifactStagingDirectory)/e2e-vhd-metadata/${VHD_ARTIFACT_NAME}.json" + condition: succeeded() + displayName: Create E2E VHD metadata + env: + VHD_ARTIFACT_NAME: ${{ parameters.artifactName }} + SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) + SKU_NAME: $(SKU_NAME) + OS_SKU: $(OS_SKU) + FEATURE_FLAGS: $(FEATURE_FLAGS) + CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) + MANAGED_SIG_ID: $(MANAGED_SIG_ID) + PACKER_BUILD_LOCATION: $(PACKER_BUILD_LOCATION) + REPLICATIONS: $(REPLICATIONS) + + - task: PublishPipelineArtifact@0 + condition: succeeded() + displayName: Publish E2E VHD metadata + inputs: + artifactName: 'e2e-vhd-metadata-${{ parameters.artifactName }}-stage$(System.StageAttempt)-job$(System.JobAttempt)' + targetPath: '$(Build.ArtifactStagingDirectory)/e2e-vhd-metadata/${{ parameters.artifactName }}.json' + - task: DownloadPipelineArtifact@2 displayName: Download Build Performance Program condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme')) diff --git a/e2e/README.md b/e2e/README.md index afdcac7fa26..9027ba7336e 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -16,6 +16,17 @@ From a high-level, for each scenario, 3. Liveness and health checks and then run to make sure the new VM's kubelet is posting NodeReady, and that workload pods can successfully be scheduled and run on the new node. +### VHD image metadata + +The Linux VHD release pipeline publishes E2E image metadata after the image-publication +steps. Ubuntu and Azure Linux CVM builds retain their original captured-image entry +and add a generalized entry keyed by `SKU_NAME`. The generalized resource ID and +completed replica regions are read from Azure; missing or unready generalized versions +fail metadata creation instead of silently skipping their scenarios. ACL and non-CVM +builds retain their existing captured-image metadata. +The generalized publisher currently runs in the prefetch path; a non-prefetch CVM +build must already have a ready generalized version to pass metadata creation. + ## Writing and extending scenarios Extend an existing scenario's `Validator` when its node has the required settings. diff --git a/spec/e2e/create_vhd_metadata_spec.sh b/spec/e2e/create_vhd_metadata_spec.sh new file mode 100644 index 00000000000..dc594298bbe --- /dev/null +++ b/spec/e2e/create_vhd_metadata_spec.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env shellspec +# shellcheck shell=bash disable=SC2329 + +Describe 'create-e2e-vhd-metadata.sh' + setup() { + TEST_ROOT="$(mktemp -d "${SHELLSPEC_WORKDIR}/vhd-metadata.XXXXXX")" + METADATA_FILE="${TEST_ROOT}/output/image.json" + export QUERY_LOG="${TEST_ROOT}/query.log" + : >"${QUERY_LOG}" + export OS_SKU=Ubuntu FEATURE_FLAGS=cvm PACKER_BUILD_LOCATION=eastus + export REPLICATIONS='westus=1, eastus2=2' + export SKU_NAME=2404gen2CVMcontainerd SIG_IMAGE_NAME=2404gen2CVMcontainerdSpecialized + export CAPTURED_SIG_VERSION=1.2.3 + export GALLERY_ID=/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Compute/galleries/test-gallery + export MANAGED_SIG_ID="${GALLERY_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" + export MOCK_AZ_STATUS=0 MOCK_VERSION + set_mock_version + export -f az + } + BeforeEach 'setup' + + set_mock_version() { + MOCK_VERSION="$(jq -nc \ + --arg id "${GALLERY_ID}/images/${SKU_NAME}/versions/${CAPTURED_SIG_VERSION}" ' + { + id: $id, + provisioningState: "Succeeded", + replicationStatus: { + aggregatedState: "Completed", + summary: [ + {region: "westus", state: "Completed"}, + {region: "eastus", state: "Completed"}, + {region: "westus", state: "Completed"} + ] + } + }')" + } + + az() { + printf '%s\n' "$@" >"${QUERY_LOG}" + if [ "${MOCK_AZ_STATUS}" -ne 0 ]; then + echo 'ResourceNotFound: generalized image version is missing' >&2 + return "${MOCK_AZ_STATUS}" + fi + printf '%s\n' "${MOCK_VERSION}" + } + + create_and_check_metadata() { + bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" && + jq -e --arg generalizedId "${GALLERY_ID}/images/${SKU_NAME}/versions/${CAPTURED_SIG_VERSION}" ' + length == 2 and + .[env.SIG_IMAGE_NAME] == { + resourceId: env.MANAGED_SIG_ID, + version: env.CAPTURED_SIG_VERSION, + regions: ["eastus", "eastus2", "westus"] + } and + .[env.SKU_NAME] == { + resourceId: $generalizedId, + version: env.CAPTURED_SIG_VERSION, + regions: ["eastus", "westus"] + } + ' "${METADATA_FILE}" >/dev/null && + grep -Fxq "${GALLERY_ID}/images/${SKU_NAME}/versions/${CAPTURED_SIG_VERSION}" "${QUERY_LOG}" && + grep -Fxq ReplicationStatus "${QUERY_LOG}" + } + + Describe 'generalized CVM metadata' + Parameters + Ubuntu True 2404gen2CVMcontainerd 2404gen2CVMcontainerdSpecialized + Ubuntu False 2604minimalgen2CVMcontainerd 2604minimalgen2CVMcontainerdSpecialized + AzureLinux True V3gen2CVM AzureLinuxV3gen2CVM + AzureLinux False V3gen2CVM AzureLinuxV3gen2CVM + End + + It "adds the actual generalized ID for $1 with prefetch=$2 and preserves the captured entry" + OS_SKU="$1" SKU_NAME="$3" SIG_IMAGE_NAME="$4" + export RUN_PREFETCH_OPTIMIZATION="$2" + MANAGED_SIG_ID="${GALLERY_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" + set_mock_version + When call create_and_check_metadata + The status should be success + The output should include "Added image ready for E2Es: ${SKU_NAME}/${CAPTURED_SIG_VERSION}" + The output should include 'Created E2E VHD metadata:' + End + End + + Describe 'captured-image metadata' + Parameters + Ubuntu None 2404gen2containerd + AzureLinux None AzureLinuxV3gen2 + AzureContainerLinux cvm aclgen2CVM + End + + It "preserves existing behavior for $1/$2 without an Azure query" + OS_SKU="$1" FEATURE_FLAGS="$2" + SIG_IMAGE_NAME="$3" SKU_NAME="$3" + MANAGED_SIG_ID="${GALLERY_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" + When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + The status should be success + The output should include 'Created E2E VHD metadata:' + The contents of file "${QUERY_LOG}" should equal '' + The value "$(jq -c --arg id "${MANAGED_SIG_ID}" 'length == 1 and .[env.SIG_IMAGE_NAME].resourceId == $id and .[env.SIG_IMAGE_NAME].regions == ["eastus", "eastus2", "westus"]' "${METADATA_FILE}")" should equal true + End + End + + It 'advertises only confirmed replica regions' + MOCK_VERSION="$(jq -c '.replicationStatus.summary += [{region:"westus2",state:"InProgress"}]' <<<"${MOCK_VERSION}")" + When call create_and_check_metadata + The status should be success + The output should include 'replicated regions: eastus, westus' + The output should include 'Created E2E VHD metadata:' + End + + It 'matches the publisher gate case-insensitively' + OS_SKU=ubuntu FEATURE_FLAGS=CVM + When call create_and_check_metadata + The status should be success + The output should include 'Added image ready for E2Es:' + The output should include 'Created E2E VHD metadata:' + End + + Describe 'unready or incorrect generalized versions' + Parameters + missing-id 'del(.id)' + captured-id '.id = env.MANAGED_SIG_ID' + wrong-version '.id |= sub("1.2.3$"; "1.2.4")' + updating '.provisioningState = "Updating"' + replicating '.replicationStatus.aggregatedState = "InProgress"' + no-regions '.replicationStatus.summary = []' + incomplete-region '.replicationStatus.summary = [{region:"westus",state:"InProgress"}]' + invalid-region '.replicationStatus.summary = [{region:null,state:"Completed"}]' + End + + It "rejects $1 without writing metadata" + MOCK_VERSION="$(jq -c "$2" <<<"${MOCK_VERSION}")" + When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + The status should be failure + The output should include 'Reading generalized CVM image' + The stderr should include 'is not ready with confirmed replica regions' + The path "${METADATA_FILE}" should not be exist + End + End + + It 'fails explicitly when the generalized image lookup fails' + MOCK_AZ_STATUS=37 + When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + The status should equal 37 + The output should include 'Reading generalized CVM image' + The stderr should include 'Unable to read generalized CVM image' + The path "${METADATA_FILE}" should not be exist + End + + It 'rejects a missing captured image ID' + MANAGED_SIG_ID='' + When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + The status should be failure + The stderr should include 'Packer did not return a managed SIG image version ID' + The path "${METADATA_FILE}" should not be exist + End + + check_pipeline_order() { + awk ' + /displayName: Run Prefetch Optimization and Convert to VHD blob/ {prefetch = NR} + /displayName: Copy to Immutable Storage Container/ {copy = NR} + /displayName: Create E2E VHD metadata/ {create = NR} + /displayName: Publish E2E VHD metadata/ {publish = NR} + END {exit !(prefetch > 0 && copy > prefetch && create > copy && publish > create)} + ' .pipelines/templates/.builder-release-template.yaml + } + + It 'creates and publishes metadata after both VHD publication paths' + When call check_pipeline_order + The status should be success + End +End From 04aed81f35e378b0243c6f2602794a8ea95782a9 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Mon, 19 Oct 2026 16:44:53 -0400 Subject: [PATCH 29/37] fix --- .pipelines/scripts/create-e2e-vhd-metadata.sh | 28 ++++----- .../templates/.builder-release-template.yaml | 3 + e2e/README.md | 17 +++--- spec/e2e/create_vhd_metadata_spec.sh | 60 ++++++++++++++----- 4 files changed, 71 insertions(+), 37 deletions(-) diff --git a/.pipelines/scripts/create-e2e-vhd-metadata.sh b/.pipelines/scripts/create-e2e-vhd-metadata.sh index 45aa80cecb9..3d248d9a206 100644 --- a/.pipelines/scripts/create-e2e-vhd-metadata.sh +++ b/.pipelines/scripts/create-e2e-vhd-metadata.sh @@ -21,22 +21,19 @@ metadata="$(jq -nc \ --argjson regions "${regions}" \ '{($imageName): {resourceId: $resourceId, version: $version, regions: $regions}}')" -case "${OS_SKU,,}:${FEATURE_FLAGS,,}" in - ubuntu:*cvm*|azurelinux:*cvm*) +prefetch_settings="${GENERATE_PUBLISHING_INFO:-False}:${RUN_PREFETCH_OPTIMIZATION:-False}:${PREFETCH_COMPATIBLE:-False}" +case "${prefetch_settings,,}:${OS_SKU,,}:${FEATURE_FLAGS,,}" in + true:true:true:ubuntu:*cvm*|true:true:true:azurelinux:*cvm*) # Preserve the capture entry; query the generalized version in the same gallery. gallery_id="${MANAGED_SIG_ID%/images/*}" generalized_id="${gallery_id}/images/${SKU_NAME:?}/versions/${CAPTURED_SIG_VERSION}" echo "Reading generalized CVM image ${SKU_NAME}/${CAPTURED_SIG_VERSION} for E2E metadata" - generalized_version="$( + if ! generalized_version="$( az sig image-version show --ids "${generalized_id}" --expand ReplicationStatus \ --output json --only-show-errors - )" || { - exit_code=$? - echo "##vso[task.logissue type=error]Unable to read generalized CVM image ${generalized_id} for E2E metadata" >&2 - exit "${exit_code}" - } - - if ! generalized_entry="$(jq -ce --arg expectedId "${generalized_id}" --arg version "${CAPTURED_SIG_VERSION}" ' + )"; then + echo "##vso[task.logissue type=warning]Unable to read generalized CVM image ${generalized_id}; omitting it from E2E metadata" >&2 + elif ! generalized_entry="$(jq -ce --arg expectedId "${generalized_id}" --arg version "${CAPTURED_SIG_VERSION}" ' select((.id | type) == "string") | select((.id | ascii_downcase) == ($expectedId | ascii_downcase)) | select(.provisioningState == "Succeeded" and .replicationStatus.aggregatedState == "Completed") | @@ -49,13 +46,12 @@ case "${OS_SKU,,}:${FEATURE_FLAGS,,}" in } | select(.regions | length > 0) ' <<<"${generalized_version}")"; then - echo "##vso[task.logissue type=error]Generalized CVM image ${generalized_id} is not ready with confirmed replica regions; E2E metadata will not be published" >&2 - exit 1 + echo "##vso[task.logissue type=warning]Generalized CVM image ${generalized_id} is not ready with confirmed replica regions; omitting it from E2E metadata" >&2 + else + metadata="$(jq -c --arg imageName "${SKU_NAME}" --argjson entry "${generalized_entry}" \ + '. + {($imageName): $entry}' <<<"${metadata}")" + echo "Added image ready for E2Es: ${SKU_NAME}/${CAPTURED_SIG_VERSION}, replicated regions: $(jq -r '.regions | join(", ")' <<<"${generalized_entry}")" fi - - metadata="$(jq -c --arg imageName "${SKU_NAME}" --argjson entry "${generalized_entry}" \ - '. + {($imageName): $entry}' <<<"${metadata}")" - echo "Added image ready for E2Es: ${SKU_NAME}/${CAPTURED_SIG_VERSION}, replicated regions: $(jq -r '.regions | join(", ")' <<<"${generalized_entry}")" ;; esac diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 72dbdbe13ed..8c36913e677 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -464,6 +464,9 @@ steps: condition: succeeded() displayName: Create E2E VHD metadata env: + GENERATE_PUBLISHING_INFO: $(GENERATE_PUBLISHING_INFO) + RUN_PREFETCH_OPTIMIZATION: $(RUN_PREFETCH_OPTIMIZATION) + PREFETCH_COMPATIBLE: $(PREFETCH_COMPATIBLE) VHD_ARTIFACT_NAME: ${{ parameters.artifactName }} SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) SKU_NAME: $(SKU_NAME) diff --git a/e2e/README.md b/e2e/README.md index 9027ba7336e..5a6259a9d39 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -19,13 +19,16 @@ From a high-level, for each scenario, ### VHD image metadata The Linux VHD release pipeline publishes E2E image metadata after the image-publication -steps. Ubuntu and Azure Linux CVM builds retain their original captured-image entry -and add a generalized entry keyed by `SKU_NAME`. The generalized resource ID and -completed replica regions are read from Azure; missing or unready generalized versions -fail metadata creation instead of silently skipping their scenarios. ACL and non-CVM -builds retain their existing captured-image metadata. -The generalized publisher currently runs in the prefetch path; a non-prefetch CVM -build must already have a ready generalized version to pass metadata creation. +steps. All builds retain their original captured-image entry. Ubuntu and Azure Linux +CVM builds also attempt generalized gallery publication for AgentBaker E2Es when +prefetch runs (`GENERATE_PUBLISHING_INFO`, `RUN_PREFETCH_OPTIMIZATION`, and +`PREFETCH_COMPATIBLE` are all true). Metadata then adds a generalized entry keyed by +`SKU_NAME`, using the actual resource ID and completed replica regions read from Azure. +Publication is best-effort: missing or unready generalized versions produce warnings +and are omitted without failing the VHD build. Non-prefetch builds do not query or +advertise generalized versions, even if an older version exists. Metadata-backed E2Es +skip missing CVM images when `IgnoreScenariosWithMissingVhd` is enabled; otherwise +image selection fails. Production VHD output and ACL/non-CVM metadata are unchanged. ## Writing and extending scenarios diff --git a/spec/e2e/create_vhd_metadata_spec.sh b/spec/e2e/create_vhd_metadata_spec.sh index dc594298bbe..9faf6ac0eef 100644 --- a/spec/e2e/create_vhd_metadata_spec.sh +++ b/spec/e2e/create_vhd_metadata_spec.sh @@ -8,6 +8,7 @@ Describe 'create-e2e-vhd-metadata.sh' export QUERY_LOG="${TEST_ROOT}/query.log" : >"${QUERY_LOG}" export OS_SKU=Ubuntu FEATURE_FLAGS=cvm PACKER_BUILD_LOCATION=eastus + export GENERATE_PUBLISHING_INFO=True RUN_PREFETCH_OPTIMIZATION=True PREFETCH_COMPATIBLE=True export REPLICATIONS='westus=1, eastus2=2' export SKU_NAME=2404gen2CVMcontainerd SIG_IMAGE_NAME=2404gen2CVMcontainerdSpecialized export CAPTURED_SIG_VERSION=1.2.3 @@ -64,17 +65,25 @@ Describe 'create-e2e-vhd-metadata.sh' grep -Fxq ReplicationStatus "${QUERY_LOG}" } + check_captured_metadata() { + jq -e ' + length == 1 and .[env.SIG_IMAGE_NAME] == { + resourceId: env.MANAGED_SIG_ID, + version: env.CAPTURED_SIG_VERSION, + regions: ["eastus", "eastus2", "westus"] + } + ' "${METADATA_FILE}" >/dev/null + } + Describe 'generalized CVM metadata' Parameters - Ubuntu True 2404gen2CVMcontainerd 2404gen2CVMcontainerdSpecialized - Ubuntu False 2604minimalgen2CVMcontainerd 2604minimalgen2CVMcontainerdSpecialized - AzureLinux True V3gen2CVM AzureLinuxV3gen2CVM - AzureLinux False V3gen2CVM AzureLinuxV3gen2CVM + Ubuntu 2404gen2CVMcontainerd 2404gen2CVMcontainerdSpecialized + Ubuntu 2604minimalgen2CVMcontainerd 2604minimalgen2CVMcontainerdSpecialized + AzureLinux V3gen2CVM AzureLinuxV3gen2CVM End - It "adds the actual generalized ID for $1 with prefetch=$2 and preserves the captured entry" - OS_SKU="$1" SKU_NAME="$3" SIG_IMAGE_NAME="$4" - export RUN_PREFETCH_OPTIMIZATION="$2" + It "adds the actual generalized ID for prefetched $1/$2 and preserves the captured entry" + OS_SKU="$1" SKU_NAME="$2" SIG_IMAGE_NAME="$3" MANAGED_SIG_ID="${GALLERY_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" set_mock_version When call create_and_check_metadata @@ -84,6 +93,24 @@ Describe 'create-e2e-vhd-metadata.sh' End End + Describe 'CVM builds without prefetch' + Parameters + Ubuntu False True True + AzureLinux True False True + Ubuntu True True False + AzureLinux '' '' '' + End + + It "omits generalized metadata for $1 with publishing=$2, prefetch=$3, compatible=$4 without querying Azure" + OS_SKU="$1" GENERATE_PUBLISHING_INFO="$2" RUN_PREFETCH_OPTIMIZATION="$3" PREFETCH_COMPATIBLE="$4" + When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + The status should be success + The output should include 'Created E2E VHD metadata:' + The contents of file "${QUERY_LOG}" should equal '' + The result of function check_captured_metadata should be successful + End + End + Describe 'captured-image metadata' Parameters Ubuntu None 2404gen2containerd @@ -113,6 +140,7 @@ Describe 'create-e2e-vhd-metadata.sh' It 'matches the publisher gate case-insensitively' OS_SKU=ubuntu FEATURE_FLAGS=CVM + GENERATE_PUBLISHING_INFO=true RUN_PREFETCH_OPTIMIZATION=TRUE PREFETCH_COMPATIBLE=true When call create_and_check_metadata The status should be success The output should include 'Added image ready for E2Es:' @@ -131,23 +159,27 @@ Describe 'create-e2e-vhd-metadata.sh' invalid-region '.replicationStatus.summary = [{region:null,state:"Completed"}]' End - It "rejects $1 without writing metadata" + It "omits $1 with a warning and preserves captured metadata" MOCK_VERSION="$(jq -c "$2" <<<"${MOCK_VERSION}")" When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" - The status should be failure + The status should be success The output should include 'Reading generalized CVM image' - The stderr should include 'is not ready with confirmed replica regions' - The path "${METADATA_FILE}" should not be exist + The output should include 'Created E2E VHD metadata:' + The stderr should include 'task.logissue type=warning' + The stderr should include 'is not ready with confirmed replica regions; omitting it' + The result of function check_captured_metadata should be successful End End - It 'fails explicitly when the generalized image lookup fails' + It 'warns and preserves captured metadata when the generalized image lookup fails' MOCK_AZ_STATUS=37 When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" - The status should equal 37 + The status should be success The output should include 'Reading generalized CVM image' + The output should include 'Created E2E VHD metadata:' + The stderr should include 'task.logissue type=warning' The stderr should include 'Unable to read generalized CVM image' - The path "${METADATA_FILE}" should not be exist + The result of function check_captured_metadata should be successful End It 'rejects a missing captured image ID' From a11a137ee2bd9016e4fca2868d82a5a76a73c454 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 20 Oct 2026 12:08:55 -0400 Subject: [PATCH 30/37] fix --- .../templates/.builder-release-template.yaml | 1 + e2e/README.md | 5 +- spec/vhdbuilder/prefetch/optimize_spec.sh | 202 ++++++++++++++++++ vhdbuilder/prefetch/scripts/optimize.sh | 95 +++++--- 4 files changed, 276 insertions(+), 27 deletions(-) create mode 100644 spec/vhdbuilder/prefetch/optimize_spec.sh diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 8c36913e677..05f9694ad76 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -346,6 +346,7 @@ steps: condition: | and( succeeded(), + eq(variables.PREFETCH_READY_FOR_CLEANUP, 'true'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables.RUN_PREFETCH_OPTIMIZATION, 'True'), eq(variables.PREFETCH_COMPATIBLE, 'True') diff --git a/e2e/README.md b/e2e/README.md index 5a6259a9d39..2fa8d78450f 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -25,7 +25,10 @@ prefetch runs (`GENERATE_PUBLISHING_INFO`, `RUN_PREFETCH_OPTIMIZATION`, and `PREFETCH_COMPATIBLE` are all true). Metadata then adds a generalized entry keyed by `SKU_NAME`, using the actual resource ID and completed replica regions read from Azure. Publication is best-effort: missing or unready generalized versions produce warnings -and are omitted without failing the VHD build. Non-prefetch builds do not query or +and are omitted without failing the VHD build. Prefetch resources are cleaned up only +after the immutable VHD copy and any required gallery publication succeed. On gallery failure, +the Image Builder source is retained; rerunning prefetch retries gallery publication +without recopying an already-published VHD. Non-prefetch builds do not query or advertise generalized versions, even if an older version exists. Metadata-backed E2Es skip missing CVM images when `IgnoreScenariosWithMissingVhd` is enabled; otherwise image selection fails. Production VHD output and ACL/non-CVM metadata are unchanged. diff --git a/spec/vhdbuilder/prefetch/optimize_spec.sh b/spec/vhdbuilder/prefetch/optimize_spec.sh new file mode 100644 index 00000000000..59b03d9802a --- /dev/null +++ b/spec/vhdbuilder/prefetch/optimize_spec.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env shellspec +# shellcheck shell=bash disable=SC2329 + +Describe 'prefetch publication and cleanup' + setup() { + export SUBSCRIPTION_ID=test LOCATION=eastus SIG_GALLERY_RESOURCE_GROUP_NAME=test-rg + export SIG_GALLERY_NAME=test-gallery SIG_IMAGE_NAME=specialized SKU_NAME=generalized + export STORAGE_ACCOUNT_BLOB_URL=https://test.blob.core.windows.net/vhd + export STORAGE_ACCOUNT_BLOB_URL_STAGING=https://test.blob.core.windows.net/staging + export VHD_NAME=test.vhd IMAGE_BUILDER_IDENTITY_ID=test-identity BUILD_RUN_NUMBER=test + export CAPTURED_SIG_VERSION=1.2.3 HYPERV_GENERATION=V2 FEATURE_FLAGS=cvm OS_SKU=Ubuntu + export ENABLE_TRUSTED_LAUNCH=false WORKLOAD_OPTIMIZATION_SCRIPT_URL=https://example.test/optimize.sh + export MOCK_EXISTS=false MOCK_COPY_STATUS=success MOCK_GALLERY_READY=false + export MOCK_GALLERY_REGION=westus MOCK_GALLERY_REGION_STATE=Completed + export MOCK_STORAGE_STATUS=0 MOCK_GALLERY_STATUS=0 MOCK_ARTIFACT_STATUS=0 + export MOCK_ARTIFACT_URI=https://source.blob.core.windows.net/vhds/test.vhd + export MOCK_WAIT_STATUS=0 + export CALL_LOG="${SHELLSPEC_WORKDIR}/prefetch-calls" + : >"${CALL_LOG}" + } + BeforeEach 'setup' + + az() { + case "${1} ${2} ${3}" in + 'storage blob exists') + printf '{"exists":%s}\n' "${MOCK_EXISTS}" + ;; + 'storage blob show') + printf '{"properties":{"copy":{"status":"%s"}}}\n' "${MOCK_COPY_STATUS}" + ;; + 'sig image-version show') + echo gallery-check >>"${CALL_LOG}" + if [ "${MOCK_GALLERY_READY}" = true ]; then + jq -nc --arg region "${MOCK_GALLERY_REGION}" --arg state "${MOCK_GALLERY_REGION_STATE}" \ + '{provisioningState:"Succeeded",replicationStatus:{aggregatedState:"Completed",summary:[{region:$region,state:$state}]}}' + else + echo '{"provisioningState":"Updating","replicationStatus":{"aggregatedState":"InProgress"}}' + fi + ;; + 'resource show --ids') + echo artifact-read >>"${CALL_LOG}" + printf '%s\n' "${MOCK_ARTIFACT_URI}" + return "${MOCK_ARTIFACT_STATUS}" + ;; + *) + echo "Unexpected Azure command: $*" >&2 + return 99 + ;; + esac + } + + run_optimizer() ( + # Load the real functions without executing main, then replace resource operations. + # shellcheck disable=SC1090 + source <(sed '$d' vhdbuilder/prefetch/scripts/optimize.sh) 2>/dev/null + { set +x; } 2>/dev/null + ensure_image_builder_rg() { echo ensure-builder >>"${CALL_LOG}"; } + run_image_builder_template() { echo run-builder >>"${CALL_LOG}"; } + copy_optimized_vhd() { + echo storage-copy >>"${CALL_LOG}" + return "${MOCK_STORAGE_STATUS}" + } + copy_optimized_vhd_to_packer_gallery() { + printf 'gallery-publish %s\n' "$1" >>"${CALL_LOG}" + return "${MOCK_GALLERY_STATUS}" + } + wait_for_vhd_copy() { + echo storage-wait >>"${CALL_LOG}" + return "${MOCK_WAIT_STATUS}" + } + main + ) + + It 'allows cleanup only after both publications succeed' + When run run_optimizer + The status should be success + The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]false' + The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + The contents of file "${CALL_LOG}" should include 'storage-copy' + The contents of file "${CALL_LOG}" should include "gallery-publish ${MOCK_ARTIFACT_URI}" + End + + It 'keeps gallery failure warning-only while blocking cleanup' + MOCK_GALLERY_STATUS=37 + When run run_optimizer + The status should be success + The output should include 'Generalized CVM gallery publication failed (exit 37)' + The output should include 'preserving prefetch resources for retry' + The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]false' + The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + End + + It 'still fails storage delivery and blocks cleanup even if gallery publication succeeds' + MOCK_STORAGE_STATUS=23 + When run run_optimizer + The status should equal 23 + The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + The contents of file "${CALL_LOG}" should include 'gallery-publish' + End + + Describe 'gallery recovery with an existing immutable VHD' + Parameters + success + pending + End + + It "retries gallery publication after storage state $1 without rebuilding or recopying" + MOCK_EXISTS=true MOCK_COPY_STATUS="$1" + When run run_optimizer + The status should be success + The output should include 'Retrying generalized CVM publication' + The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + The contents of file "${CALL_LOG}" should include "gallery-publish ${MOCK_ARTIFACT_URI}" + The contents of file "${CALL_LOG}" should not include 'storage-copy' + The contents of file "${CALL_LOG}" should not include 'run-builder' + The contents of file "${CALL_LOG}" should not include 'ensure-builder' + End + End + + It 'allows an already-complete CVM rerun without needing the deleted source' + MOCK_EXISTS=true MOCK_GALLERY_READY=true MOCK_ARTIFACT_STATUS=99 + When run run_optimizer + The status should be success + The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + The contents of file "${CALL_LOG}" should equal 'gallery-check' + End + + Describe 'publication is not required' + Parameters + None Ubuntu + cvm ACL + End + + It "preserves the existing early return for feature flags $1 and OS $2" + MOCK_EXISTS=true FEATURE_FLAGS="$1" OS_SKU="$2" + When run run_optimizer + The status should be success + The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + The contents of file "${CALL_LOG}" should equal '' + End + End + + Describe 'gallery replication must be complete in westus' + Parameters + westus InProgress + eastus Completed + End + + It "retries publication when region $1 has state $2" + MOCK_EXISTS=true MOCK_GALLERY_READY=true + MOCK_GALLERY_REGION="$1" MOCK_GALLERY_REGION_STATE="$2" + When run run_optimizer + The status should be success + The output should include 'Retrying generalized CVM publication' + The contents of file "${CALL_LOG}" should include "gallery-publish ${MOCK_ARTIFACT_URI}" + End + End + + Describe 'unavailable retained output' + Parameters + 37 https://source.blob.core.windows.net/vhds/test.vhd + 0 null + End + + It "keeps an already-delivered VHD successful with source status $1 and URI $2 but blocks cleanup" + MOCK_EXISTS=true MOCK_ARTIFACT_STATUS="$1" MOCK_ARTIFACT_URI="$2" + When run run_optimizer + The status should be success + The output should include 'Unable to read retained Image Builder output' + The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + The contents of file "${CALL_LOG}" should not include 'gallery-publish' + End + End + + It 'still fails a fresh build if Image Builder returns an empty URI' + MOCK_ARTIFACT_URI='' + When run run_optimizer + The status should equal 1 + The stderr should include 'unable to determine artifactUri' + The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + End + + It 'does not treat a failed immutable copy as recoverable' + MOCK_EXISTS=true MOCK_COPY_STATUS=failed + When run run_optimizer + The status should equal 1 + The output should include 'unrecoverable copy state' + The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' + The contents of file "${CALL_LOG}" should equal '' + End + + cleanup_requires_both_publications() { + sed -n '/make -f packer.mk cleanup-prefetch/,/displayName: Cleanup Prefetch Optimization Resources/p' \ + .pipelines/templates/.builder-release-template.yaml | + grep -Fq "eq(variables.PREFETCH_READY_FOR_CLEANUP, 'true')" + } + + It 'gates the pipeline cleanup step on successful publication' + When call cleanup_requires_both_publications + The status should be success + End +End diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index 939209ec644..6fc40fa130a 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -56,38 +56,79 @@ DESTINATION_STORAGE_ACCOUNT_NAME="${STORAGE_ACCOUNT_BLOB_URL#*://}" DESTINATION_STORAGE_ACCOUNT_NAME="${DESTINATION_STORAGE_ACCOUNT_NAME%%.*}" DESTINATION_CONTAINER_NAME="${STORAGE_ACCOUNT_BLOB_URL##*/}" STAGING_VHD_URI="${STORAGE_ACCOUNT_BLOB_URL_STAGING}/${VHD_NAME}" +VHD_ALREADY_PUBLISHED=false main() { - # for idempotency, check whether the target VHD already exists. If a previous run already produced it - # successfully, exit early; if a server-side copy is still in progress, wait for it to finish. Because - # the destination container may be immutable, a VHD left in a bad state cannot be deleted and recreated - # and is treated as fatal. - check_for_existing_vhd - - # attempt to perform prefetch optimization, distributing the optimized image as a VHD into the - # image builder staging storage account, then copy that VHD into the target storage account - ensure_image_builder_rg || exit $? - run_image_builder_template || exit $? - - local artifact_uri storage_pid gallery_pid storage_status=0 - artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv)" || return $? - if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then - echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" - return 1 + echo "##vso[task.setvariable variable=PREFETCH_READY_FOR_CLEANUP]false" + check_for_existing_vhd || return $? + + if [ "${VHD_ALREADY_PUBLISHED}" = "true" ]; then + if ! requires_generalized_cvm_publication || generalized_cvm_gallery_is_ready; then + echo "##vso[task.setvariable variable=PREFETCH_READY_FOR_CLEANUP]true" + return 0 + fi + echo "Retrying generalized CVM publication using the retained Image Builder output" + else + ensure_image_builder_rg || return $? + run_image_builder_template || return $? fi - copy_optimized_vhd "${artifact_uri}" & - storage_pid=$! + local artifact_uri artifact_status=0 storage_pid="" gallery_pid storage_status=0 gallery_status=0 + artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv)" || artifact_status=$? + if [ "${artifact_status}" -ne 0 ] || [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then + if [ "${VHD_ALREADY_PUBLISHED}" = "true" ]; then + echo "##vso[task.logissue type=warning]Unable to read retained Image Builder output for generalized CVM publication; preserving prefetch resources" + return 0 + fi + echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" >&2 + [ "${artifact_status}" -ne 0 ] || artifact_status=1 + return "${artifact_status}" + fi + + if [ "${VHD_ALREADY_PUBLISHED}" != "true" ]; then + copy_optimized_vhd "${artifact_uri}" & + storage_pid=$! + fi copy_optimized_vhd_to_packer_gallery "${artifact_uri}" & gallery_pid=$! - wait "${storage_pid}" || storage_status=$? - wait "${gallery_pid}" || echo "##vso[task.logissue type=warning]Generalized CVM gallery publication failed (exit $?)" + if [ -n "${storage_pid}" ]; then + wait "${storage_pid}" || storage_status=$? + fi + wait "${gallery_pid}" || gallery_status=$? + if [ "${gallery_status}" -ne 0 ]; then + echo "##vso[task.logissue type=warning]Generalized CVM gallery publication failed (exit ${gallery_status}); preserving prefetch resources for retry" + elif [ "${storage_status}" -eq 0 ]; then + echo "##vso[task.setvariable variable=PREFETCH_READY_FOR_CLEANUP]true" + fi return "${storage_status}" } +requires_generalized_cvm_publication() { + [[ "${FEATURE_FLAGS,,}" == *cvm* && "${OS_SKU,,}" =~ ^(ubuntu|azurelinux)$ ]] +} + +generalized_cvm_gallery_is_ready() { + local image_status + if ! image_status="$(az sig image-version show \ + --resource-group "${SIG_GALLERY_RESOURCE_GROUP_NAME}" --gallery-name "${SIG_GALLERY_NAME}" \ + --gallery-image-definition "${SKU_NAME}" --gallery-image-version "${CAPTURED_SIG_VERSION}" \ + --expand ReplicationStatus --output json --only-show-errors)"; then + echo "Unable to confirm generalized CVM gallery readiness; will retry publication" >&2 + return 1 + fi + jq -e ' + .provisioningState == "Succeeded" and + .replicationStatus.aggregatedState == "Completed" and + any(.replicationStatus.summary[]?; .region == "westus" and .state == "Completed") + ' <<<"${image_status}" >/dev/null +} + check_for_existing_vhd() { - if [ "$(az storage blob exists --blob-url "${VHD_URI}" --auth-mode login | jq -r '.exists')" = "false" ]; then + local exists copy_status + exists="$(az storage blob exists --blob-url "${VHD_URI}" --auth-mode login | jq -r '.exists')" || return $? + VHD_ALREADY_PUBLISHED=false + if [ "${exists}" = "false" ]; then echo "no existing VHD was found at: ${VHD_URI}, will proceed with optimization and VHD creation" return 0 fi @@ -96,21 +137,23 @@ check_for_existing_vhd() { # the VHD in a potentially immutable container is produced by a server-side blob copy, so its copy status # tells us whether a previous run already completed. Because the container may have an immutability policy, # a VHD in a bad state cannot be deleted and recreated, so any non-success terminal state is fatal. - copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" + copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" || return $? if [ "${copy_status,,}" = "success" ]; then - echo "VHD ${VHD_URI} was already produced by a previous prefetch optimization run, nothing to do" - exit 0 + echo "VHD ${VHD_URI} was already produced by a previous prefetch optimization run" + VHD_ALREADY_PUBLISHED=true + return 0 fi if [ "${copy_status,,}" = "pending" ]; then echo "VHD ${VHD_URI} is currently being copied by a previous prefetch optimization run, will wait for completion" if wait_for_vhd_copy; then - exit 0 + VHD_ALREADY_PUBLISHED=true + return 0 fi fi # the VHD exists in a non-success state within the immutable container. It cannot be deleted due to # the immutability policy, so there is nothing we can do to recover - fail the script. echo "VHD ${VHD_URI} exists in an unrecoverable copy state: '${copy_status}'" - exit 1 + return 1 } ensure_image_builder_rg() { From 2e6f36bb1e8fbf5fb51e2d481c0da9cef3f48fa8 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 20 Oct 2026 16:00:36 -0400 Subject: [PATCH 31/37] fix --- .pipelines/scripts/create-e2e-vhd-metadata.sh | 5 +- spec/e2e/create_vhd_metadata_spec.sh | 5 +- .../packer/publish_generalized_cvm_spec.sh | 105 +++++++++++++++--- spec/vhdbuilder/prefetch/optimize_spec.sh | 9 +- .../publish-generalized-cvm-if-required.sh | 53 ++++----- vhdbuilder/prefetch/scripts/optimize.sh | 5 +- 6 files changed, 128 insertions(+), 54 deletions(-) diff --git a/.pipelines/scripts/create-e2e-vhd-metadata.sh b/.pipelines/scripts/create-e2e-vhd-metadata.sh index 3d248d9a206..546fd999aec 100644 --- a/.pipelines/scripts/create-e2e-vhd-metadata.sh +++ b/.pipelines/scripts/create-e2e-vhd-metadata.sh @@ -33,10 +33,11 @@ case "${prefetch_settings,,}:${OS_SKU,,}:${FEATURE_FLAGS,,}" in --output json --only-show-errors )"; then echo "##vso[task.logissue type=warning]Unable to read generalized CVM image ${generalized_id}; omitting it from E2E metadata" >&2 - elif ! generalized_entry="$(jq -ce --arg expectedId "${generalized_id}" --arg version "${CAPTURED_SIG_VERSION}" ' + elif ! generalized_entry="$(jq -ce --arg expectedId "${generalized_id}" --arg version "${CAPTURED_SIG_VERSION}" --arg buildLocation "${PACKER_BUILD_LOCATION}" ' select((.id | type) == "string") | select((.id | ascii_downcase) == ($expectedId | ascii_downcase)) | - select(.provisioningState == "Succeeded" and .replicationStatus.aggregatedState == "Completed") | + select(.provisioningState == "Succeeded" and + any(.replicationStatus.summary[]?; .region == $buildLocation and .state == "Completed")) | { resourceId: .id, version: $version, diff --git a/spec/e2e/create_vhd_metadata_spec.sh b/spec/e2e/create_vhd_metadata_spec.sh index 9faf6ac0eef..8a67c8837bc 100644 --- a/spec/e2e/create_vhd_metadata_spec.sh +++ b/spec/e2e/create_vhd_metadata_spec.sh @@ -131,7 +131,7 @@ Describe 'create-e2e-vhd-metadata.sh' End It 'advertises only confirmed replica regions' - MOCK_VERSION="$(jq -c '.replicationStatus.summary += [{region:"westus2",state:"InProgress"}]' <<<"${MOCK_VERSION}")" + MOCK_VERSION="$(jq -c '.replicationStatus.aggregatedState = "InProgress" | .replicationStatus.summary += [{region:"westus2",state:"InProgress"}]' <<<"${MOCK_VERSION}")" When call create_and_check_metadata The status should be success The output should include 'replicated regions: eastus, westus' @@ -153,7 +153,8 @@ Describe 'create-e2e-vhd-metadata.sh' captured-id '.id = env.MANAGED_SIG_ID' wrong-version '.id |= sub("1.2.3$"; "1.2.4")' updating '.provisioningState = "Updating"' - replicating '.replicationStatus.aggregatedState = "InProgress"' + replicating '.replicationStatus.summary |= map(if .region == "eastus" then .state = "InProgress" else . end)' + wrong-region '.replicationStatus.summary = [{region:"westus",state:"Completed"}]' no-regions '.replicationStatus.summary = []' incomplete-region '.replicationStatus.summary = [{region:"westus",state:"InProgress"}]' invalid-region '.replicationStatus.summary = [{region:null,state:"Completed"}]' diff --git a/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh b/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh index 105612c4525..7636b631a66 100644 --- a/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh +++ b/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh @@ -12,8 +12,11 @@ Describe 'publish-generalized-cvm-if-required.sh' export TAGS_FILE="${SHELLSPEC_WORKDIR}/cvm-version-tags" export MOCK_DEFINITION_ID=/test/image-definition export MOCK_FAIL_COMMAND='' MOCK_STATE=Succeeded - export MOCK_REPLICATION=Completed MOCK_REGION=westus MOCK_REGION_STATE=Completed - export -f az + export MOCK_REPLICATION=Completed MOCK_REGION=eastus MOCK_REGION_STATE=Completed + export MOCK_PENDING_POLLS=0 MOCK_INITIAL_RESPONSE=pending + export POLL_TRACE_FILE="${SHELLSPEC_WORKDIR}/cvm-replication-polls" + : >"${POLL_TRACE_FILE}" + export -f az sleep } BeforeEach 'setup' @@ -37,6 +40,9 @@ Describe 'publish-generalized-cvm-if-required.sh' ;; 'sig image-version create') while [[ "$#" -gt 0 ]]; do + if [[ "$1" = --target-regions ]]; then + echo "target-region $2" >>"${POLL_TRACE_FILE}" + fi if [[ "$1" = --tags ]]; then shift printf '%s\n' "$@" >"${TAGS_FILE}" @@ -45,10 +51,18 @@ Describe 'publish-generalized-cvm-if-required.sh' shift done ;; - 'sig image-version wait') - echo '{}' - ;; 'sig image-version show') + echo show >>"${POLL_TRACE_FILE}" + local show_count + show_count="$(grep -c '^show$' "${POLL_TRACE_FILE}")" + if ((show_count <= MOCK_PENDING_POLLS)); then + case "${MOCK_INITIAL_RESPONSE}" in + empty) return 0 ;; + null) echo null; return 0 ;; + missing) return 3 ;; + pending) echo '{"state":"Updating","replication":{"aggregatedState":"InProgress"}}'; return 0 ;; + esac + fi jq -nc --arg state "${MOCK_STATE}" --arg replication "${MOCK_REPLICATION}" \ --arg region "${MOCK_REGION}" --arg regionState "${MOCK_REGION_STATE}" \ '{state:$state,replication:{aggregatedState:$replication,summary:[{region:$region,state:$regionState}]}}' @@ -60,6 +74,18 @@ Describe 'publish-generalized-cvm-if-required.sh' esac } + sleep() { + echo "sleep $*" >>"${POLL_TRACE_FILE}" + SECONDS=$((SECONDS + $1)) + } + + publish_and_check_polling() { + bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ + https://teststorage.blob.core.windows.net/vhds/test.vhd && + [[ "$(grep -c '^show$' "${POLL_TRACE_FILE}")" -eq 3 ]] && + [[ "$(grep -c '^sleep 60$' "${POLL_TRACE_FILE}")" -eq 2 ]] + } + publish_and_check_tags() { bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ https://teststorage.blob.core.windows.net/vhds/test.vhd && @@ -92,19 +118,37 @@ Describe 'publish-generalized-cvm-if-required.sh' When call publish_and_check_tags The status should be success The output should include 'Image version creation completed' - The output should include 'replicated successfully to westus' - The output should include 'image ready in westus' + The output should include "replicated successfully to ${PACKER_BUILD_LOCATION}" + The output should include "image ready in ${PACKER_BUILD_LOCATION}" The output should not include 'provisioned' + The contents of file "${POLL_TRACE_FILE}" should not include 'sleep' + The contents of file "${POLL_TRACE_FILE}" should include "target-region ${PACKER_BUILD_LOCATION}" End End - Describe 'unready images after a zero-exit wait' + Describe 'replication polling' Parameters - Updating Completed westus Completed - Succeeded InProgress westus InProgress - Succeeded Completed eastus Completed - Succeeded Completed westus InProgress - Failed Failed westus Failed + pending + empty + null + missing + End + + It "polls every 60 seconds until a $1 image response becomes ready" + MOCK_PENDING_POLLS=2 MOCK_INITIAL_RESPONSE="$1" + When call publish_and_check_polling + The status should be success + The output should include 'retrying in 60 seconds' + The output should include "image ready in ${PACKER_BUILD_LOCATION}" + End + End + + Describe 'unready images' + Parameters + Updating Completed eastus Completed + Succeeded InProgress eastus InProgress + Succeeded Completed westus Completed + Succeeded Completed eastus InProgress End It "rejects state=$1, replication=$2, region=$3, regional state=$4" @@ -113,12 +157,40 @@ Describe 'publish-generalized-cvm-if-required.sh' https://teststorage.blob.core.windows.net/vhds/test.vhd The status should be failure The output should include 'Verifying image version' - The output should not include 'image ready in westus' + The output should not include 'image ready in' The output should not include 'replicated successfully' - The stderr should include 'is not ready in westus after waiting for replication' + The stderr should include "is not ready in ${PACKER_BUILD_LOCATION} after waiting 900 seconds for replication" End End + Describe 'terminal provisioning or replication failures' + Parameters + Failed Completed Completed + Canceled Completed Completed + Succeeded Failed Failed + Succeeded InProgress Failed + End + + It "stops immediately for state=$1, replication=$2, regional state=$3" + MOCK_STATE="$1" MOCK_REPLICATION="$2" MOCK_REGION_STATE="$3" + When run bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ + https://teststorage.blob.core.windows.net/vhds/test.vhd + The status should be failure + The output should not include 'image ready in' + The stderr should include 'provisioning or replication failed' + The contents of file "${POLL_TRACE_FILE}" should not include 'sleep' + End + End + + It 'does not wait for aggregate replication when the build region is ready' + PACKER_BUILD_LOCATION=northeurope MOCK_REGION=northeurope MOCK_REPLICATION=InProgress + When call publish_and_check_tags + The status should be success + The output should include 'image ready in northeurope' + The contents of file "${POLL_TRACE_FILE}" should include 'target-region northeurope' + The contents of file "${POLL_TRACE_FILE}" should not include 'sleep' + End + Describe 'Azure operation failures' Parameters 'storage account list' 'resolving source storage account' @@ -126,7 +198,6 @@ Describe 'publish-generalized-cvm-if-required.sh' 'sig image-definition create' 'creating image definition' 'resource update --ids' 'enabling image definition updates' 'sig image-version create' 'creating image version' - 'sig image-version wait' 'waiting for image version replication' 'sig image-version show' 'verifying image version readiness' End @@ -136,7 +207,7 @@ Describe 'publish-generalized-cvm-if-required.sh' https://teststorage.blob.core.windows.net/vhds/test.vhd The status should equal 37 The output should include 'Publishing generalized CVM image' - The output should not include 'image ready in westus' + The output should not include 'image ready in' The output should not include 'replicated successfully' The stderr should include "failed during $2" End diff --git a/spec/vhdbuilder/prefetch/optimize_spec.sh b/spec/vhdbuilder/prefetch/optimize_spec.sh index 59b03d9802a..3e82ade639a 100644 --- a/spec/vhdbuilder/prefetch/optimize_spec.sh +++ b/spec/vhdbuilder/prefetch/optimize_spec.sh @@ -11,7 +11,7 @@ Describe 'prefetch publication and cleanup' export CAPTURED_SIG_VERSION=1.2.3 HYPERV_GENERATION=V2 FEATURE_FLAGS=cvm OS_SKU=Ubuntu export ENABLE_TRUSTED_LAUNCH=false WORKLOAD_OPTIMIZATION_SCRIPT_URL=https://example.test/optimize.sh export MOCK_EXISTS=false MOCK_COPY_STATUS=success MOCK_GALLERY_READY=false - export MOCK_GALLERY_REGION=westus MOCK_GALLERY_REGION_STATE=Completed + export MOCK_GALLERY_REGION=eastus MOCK_GALLERY_REGION_STATE=Completed export MOCK_STORAGE_STATUS=0 MOCK_GALLERY_STATUS=0 MOCK_ARTIFACT_STATUS=0 export MOCK_ARTIFACT_URI=https://source.blob.core.windows.net/vhds/test.vhd export MOCK_WAIT_STATUS=0 @@ -118,6 +118,7 @@ Describe 'prefetch publication and cleanup' End It 'allows an already-complete CVM rerun without needing the deleted source' + LOCATION=northeurope MOCK_GALLERY_REGION=northeurope MOCK_EXISTS=true MOCK_GALLERY_READY=true MOCK_ARTIFACT_STATUS=99 When run run_optimizer The status should be success @@ -140,10 +141,10 @@ Describe 'prefetch publication and cleanup' End End - Describe 'gallery replication must be complete in westus' + Describe 'gallery replication must be complete in the build location' Parameters - westus InProgress - eastus Completed + eastus InProgress + westus Completed End It "retries publication when region $1 has state $2" diff --git a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh index 14cc5dcd47e..ccc641f127d 100755 --- a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh +++ b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh @@ -24,7 +24,7 @@ fi source_storage_account="${BASH_REMATCH[1]}" publication_stage="resolving source storage account (az storage account list)" report_publication_failure() { - local exit_code=$? + local exit_code="${1:-$?}" echo "ERROR: Generalized CVM publication failed during ${publication_stage} (exit ${exit_code})" >&2 exit "${exit_code}" } @@ -82,38 +82,39 @@ version_tags=( "image_sku=${IMG_SKU:?}" "branch=${BRANCH:?}" ) -echo "Creating image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} with buildId=${BUILD_BUILDID:?}, now=${CREATE_TIME}, source location ${source_location}, target region westus" +echo "Creating image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} with buildId=${BUILD_BUILDID:?}, now=${CREATE_TIME}, source location ${source_location}, target region ${PACKER_BUILD_LOCATION:?}" echo "Image version tags: ${version_tags[*]}" az sig image-version create "${version_args[@]}" \ --location "${source_location}" \ --os-vhd-uri "${SOURCE_VHD_URI}" \ --os-vhd-storage-account "${source_storage_id}" \ - --target-regions westus \ + --target-regions "${PACKER_BUILD_LOCATION}" \ --tags "${version_tags[@]}" >/dev/null echo "Image version creation completed for ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}; checking replication" -publication_stage="waiting for image version replication (az sig image-version wait)" -echo "Waiting for image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replication to westus to complete" -az sig image-version wait "${version_args[@]}" --expand ReplicationStatus \ - --custom "provisioningState=='Succeeded' && replicationStatus.aggregatedState=='Completed'" \ - --interval 60 --timeout 3600 >/dev/null - +echo "Waiting for image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replication to ${PACKER_BUILD_LOCATION} to complete" publication_stage="verifying image version readiness (az sig image-version show)" -echo "Verifying image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} is ready in westus" -image_status="$( - az sig image-version show "${version_args[@]}" --expand ReplicationStatus \ - --query '{state:provisioningState,replication:replicationStatus}' --output json -)" -echo "Image version status: ${image_status}" -# The CLI wait command can exit zero on timeout; verify the actual resource state. -if ! jq -e ' - .state == "Succeeded" and - .replication.aggregatedState == "Completed" and - any(.replication.summary[]?; .region == "westus" and .state == "Completed") -' <<<"${image_status}" >/dev/null; then - echo "ERROR: Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} is not ready in westus after waiting for replication" >&2 - exit 1 -fi +echo "Verifying image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} is ready in ${PACKER_BUILD_LOCATION}" +replication_deadline=$((SECONDS + 900)) +while true; do + image_status="$( + az sig image-version show "${version_args[@]}" --expand ReplicationStatus \ + --query '{state:provisioningState,replication:replicationStatus}' --output json + )" || { show_status=$?; [[ "${show_status}" -eq 3 ]] || report_publication_failure "${show_status}"; } + echo "Image version status: ${image_status:-not found}" + readiness="$(jq -r --arg region "${PACKER_BUILD_LOCATION}" ' + if .state == "Succeeded" and + any(.replication.summary[]?; .region == $region and .state == "Completed") then "ready" + elif .state == "Failed" or .state == "Canceled" or + any(.replication.summary[]?; .region == $region and .state == "Failed") then "failed" + else "pending" end + ' <<<"${image_status:-null}")" + [[ "${readiness}" != "ready" ]] || break + [[ "${readiness}" != "failed" ]] || { echo "ERROR: Image provisioning or replication failed" >&2; exit 1; } + ((SECONDS < replication_deadline)) || { echo "ERROR: Image is not ready in ${PACKER_BUILD_LOCATION} after waiting 900 seconds for replication" >&2; exit 1; } + echo "Image version is not ready in ${PACKER_BUILD_LOCATION}; retrying in 60 seconds" + sleep 60 +done -echo "Image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replicated successfully to westus" -echo "Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}: image ready in westus" +echo "Image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replicated successfully to ${PACKER_BUILD_LOCATION}" +echo "Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}: image ready in ${PACKER_BUILD_LOCATION}" diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index 6fc40fa130a..1dacd1ced8e 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -117,10 +117,9 @@ generalized_cvm_gallery_is_ready() { echo "Unable to confirm generalized CVM gallery readiness; will retry publication" >&2 return 1 fi - jq -e ' + jq -e --arg region "${LOCATION}" ' .provisioningState == "Succeeded" and - .replicationStatus.aggregatedState == "Completed" and - any(.replicationStatus.summary[]?; .region == "westus" and .state == "Completed") + any(.replicationStatus.summary[]?; .region == $region and .state == "Completed") ' <<<"${image_status}" >/dev/null } From c85aeba44f87911a393ba95d56ffc9f27eed8a00 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 20 Oct 2026 16:10:56 -0400 Subject: [PATCH 32/37] fix --- .pipelines/scripts/create-e2e-vhd-metadata.sh | 61 ------------------- .../templates/.builder-release-template.yaml | 61 ++++++++++++++++++- spec/e2e/create_vhd_metadata_spec.sh | 26 +++++--- 3 files changed, 77 insertions(+), 71 deletions(-) delete mode 100644 .pipelines/scripts/create-e2e-vhd-metadata.sh diff --git a/.pipelines/scripts/create-e2e-vhd-metadata.sh b/.pipelines/scripts/create-e2e-vhd-metadata.sh deleted file mode 100644 index 546fd999aec..00000000000 --- a/.pipelines/scripts/create-e2e-vhd-metadata.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -metadata_file="${1:?metadata output path is required}" - -if [ -z "${MANAGED_SIG_ID:-}" ]; then - echo "##vso[task.logissue type=error]Packer did not return a managed SIG image version ID" >&2 - exit 1 -fi - -regions="$(jq -nc \ - --arg buildLocation "${PACKER_BUILD_LOCATION:?}" \ - --arg replications "${REPLICATIONS:-}" \ - '([$buildLocation] + ($replications | split(",") | map(split("=")[0] | gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)))) | unique')" - -metadata="$(jq -nc \ - --arg imageName "${SIG_IMAGE_NAME:?}" \ - --arg resourceId "${MANAGED_SIG_ID}" \ - --arg version "${CAPTURED_SIG_VERSION:?}" \ - --argjson regions "${regions}" \ - '{($imageName): {resourceId: $resourceId, version: $version, regions: $regions}}')" - -prefetch_settings="${GENERATE_PUBLISHING_INFO:-False}:${RUN_PREFETCH_OPTIMIZATION:-False}:${PREFETCH_COMPATIBLE:-False}" -case "${prefetch_settings,,}:${OS_SKU,,}:${FEATURE_FLAGS,,}" in - true:true:true:ubuntu:*cvm*|true:true:true:azurelinux:*cvm*) - # Preserve the capture entry; query the generalized version in the same gallery. - gallery_id="${MANAGED_SIG_ID%/images/*}" - generalized_id="${gallery_id}/images/${SKU_NAME:?}/versions/${CAPTURED_SIG_VERSION}" - echo "Reading generalized CVM image ${SKU_NAME}/${CAPTURED_SIG_VERSION} for E2E metadata" - if ! generalized_version="$( - az sig image-version show --ids "${generalized_id}" --expand ReplicationStatus \ - --output json --only-show-errors - )"; then - echo "##vso[task.logissue type=warning]Unable to read generalized CVM image ${generalized_id}; omitting it from E2E metadata" >&2 - elif ! generalized_entry="$(jq -ce --arg expectedId "${generalized_id}" --arg version "${CAPTURED_SIG_VERSION}" --arg buildLocation "${PACKER_BUILD_LOCATION}" ' - select((.id | type) == "string") | - select((.id | ascii_downcase) == ($expectedId | ascii_downcase)) | - select(.provisioningState == "Succeeded" and - any(.replicationStatus.summary[]?; .region == $buildLocation and .state == "Completed")) | - { - resourceId: .id, - version: $version, - regions: ([.replicationStatus.summary[]? | - select(.state == "Completed") | .region | - select(type == "string" and length > 0)] | unique) - } | - select(.regions | length > 0) - ' <<<"${generalized_version}")"; then - echo "##vso[task.logissue type=warning]Generalized CVM image ${generalized_id} is not ready with confirmed replica regions; omitting it from E2E metadata" >&2 - else - metadata="$(jq -c --arg imageName "${SKU_NAME}" --argjson entry "${generalized_entry}" \ - '. + {($imageName): $entry}' <<<"${metadata}")" - echo "Added image ready for E2Es: ${SKU_NAME}/${CAPTURED_SIG_VERSION}, replicated regions: $(jq -r '.regions | join(", ")' <<<"${generalized_entry}")" - fi - ;; -esac - -mkdir -p "$(dirname "${metadata_file}")" -printf '%s\n' "${metadata}" >"${metadata_file}" -echo "Created E2E VHD metadata: ${metadata_file}" diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 20ffd2a4c74..50d7ed3c07b 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -484,8 +484,65 @@ steps: scriptType: bash scriptLocation: inlineScript inlineScript: | - bash .pipelines/scripts/create-e2e-vhd-metadata.sh \ - "$(Build.ArtifactStagingDirectory)/e2e-vhd-metadata/${VHD_ARTIFACT_NAME}.json" + set -euo pipefail + + metadata_file="${BUILD_ARTIFACTSTAGINGDIRECTORY:?}/e2e-vhd-metadata/${VHD_ARTIFACT_NAME}.json" + + if [ -z "${MANAGED_SIG_ID:-}" ]; then + echo "##vso[task.logissue type=error]Packer did not return a managed SIG image version ID" >&2 + exit 1 + fi + + regions="$(jq -nc \ + --arg buildLocation "${PACKER_BUILD_LOCATION:?}" \ + --arg replications "${REPLICATIONS:-}" \ + '([$buildLocation] + ($replications | split(",") | map(split("=")[0] | gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)))) | unique')" + + metadata="$(jq -nc \ + --arg imageName "${SIG_IMAGE_NAME:?}" \ + --arg resourceId "${MANAGED_SIG_ID}" \ + --arg version "${CAPTURED_SIG_VERSION:?}" \ + --argjson regions "${regions}" \ + '{($imageName): {resourceId: $resourceId, version: $version, regions: $regions}}')" + + prefetch_settings="${GENERATE_PUBLISHING_INFO:-False}:${RUN_PREFETCH_OPTIMIZATION:-False}:${PREFETCH_COMPATIBLE:-False}" + case "${prefetch_settings,,}:${OS_SKU,,}:${FEATURE_FLAGS,,}" in + true:true:true:ubuntu:*cvm*|true:true:true:azurelinux:*cvm*) + # Preserve the capture entry; query the generalized version in the same gallery. + gallery_id="${MANAGED_SIG_ID%/images/*}" + generalized_id="${gallery_id}/images/${SKU_NAME:?}/versions/${CAPTURED_SIG_VERSION}" + echo "Reading generalized CVM image ${SKU_NAME}/${CAPTURED_SIG_VERSION} for E2E metadata" + if ! generalized_version="$( + az sig image-version show --ids "${generalized_id}" --expand ReplicationStatus \ + --output json --only-show-errors + )"; then + echo "##vso[task.logissue type=warning]Unable to read generalized CVM image ${generalized_id}; omitting it from E2E metadata" >&2 + elif ! generalized_entry="$(jq -ce --arg expectedId "${generalized_id}" --arg version "${CAPTURED_SIG_VERSION}" --arg buildLocation "${PACKER_BUILD_LOCATION}" ' + select((.id | type) == "string") | + select((.id | ascii_downcase) == ($expectedId | ascii_downcase)) | + select(.provisioningState == "Succeeded" and + any(.replicationStatus.summary[]?; .region == $buildLocation and .state == "Completed")) | + { + resourceId: .id, + version: $version, + regions: ([.replicationStatus.summary[]? | + select(.state == "Completed") | .region | + select(type == "string" and length > 0)] | unique) + } | + select(.regions | length > 0) + ' <<<"${generalized_version}")"; then + echo "##vso[task.logissue type=warning]Generalized CVM image ${generalized_id} is not ready with confirmed replica regions; omitting it from E2E metadata" >&2 + else + metadata="$(jq -c --arg imageName "${SKU_NAME}" --argjson entry "${generalized_entry}" \ + '. + {($imageName): $entry}' <<<"${metadata}")" + echo "Added image ready for E2Es: ${SKU_NAME}/${CAPTURED_SIG_VERSION}, replicated regions: $(jq -r '.regions | join(", ")' <<<"${generalized_entry}")" + fi + ;; + esac + + mkdir -p "$(dirname "${metadata_file}")" + printf '%s\n' "${metadata}" >"${metadata_file}" + echo "Created E2E VHD metadata: ${metadata_file}" condition: succeeded() displayName: Create E2E VHD metadata env: diff --git a/spec/e2e/create_vhd_metadata_spec.sh b/spec/e2e/create_vhd_metadata_spec.sh index 8a67c8837bc..d552c395b29 100644 --- a/spec/e2e/create_vhd_metadata_spec.sh +++ b/spec/e2e/create_vhd_metadata_spec.sh @@ -1,10 +1,20 @@ #!/usr/bin/env shellspec # shellcheck shell=bash disable=SC2329 -Describe 'create-e2e-vhd-metadata.sh' +Describe 'Create E2E VHD metadata pipeline step' setup() { TEST_ROOT="$(mktemp -d "${SHELLSPEC_WORKDIR}/vhd-metadata.XXXXXX")" - METADATA_FILE="${TEST_ROOT}/output/image.json" + export BUILD_ARTIFACTSTAGINGDIRECTORY="${TEST_ROOT}/output" VHD_ARTIFACT_NAME=image + METADATA_FILE="${BUILD_ARTIFACTSTAGINGDIRECTORY}/e2e-vhd-metadata/${VHD_ARTIFACT_NAME}.json" + INLINE_METADATA_SCRIPT="${TEST_ROOT}/metadata.sh" + awk ' + /^ inlineScript: \|$/ {script = ""; collecting = 1; next} + collecting && /^ / {script = script substr($0, 9) "\n"; next} + collecting && /^$/ {script = script "\n"; next} + {collecting = 0} + /^ displayName: Create E2E VHD metadata$/ {printf "%s", script; found = 1; exit} + END {if (!found) exit 1} + ' .pipelines/templates/.builder-release-template.yaml >"${INLINE_METADATA_SCRIPT}" || return $? export QUERY_LOG="${TEST_ROOT}/query.log" : >"${QUERY_LOG}" export OS_SKU=Ubuntu FEATURE_FLAGS=cvm PACKER_BUILD_LOCATION=eastus @@ -47,7 +57,7 @@ Describe 'create-e2e-vhd-metadata.sh' } create_and_check_metadata() { - bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" && + bash "${INLINE_METADATA_SCRIPT}" && jq -e --arg generalizedId "${GALLERY_ID}/images/${SKU_NAME}/versions/${CAPTURED_SIG_VERSION}" ' length == 2 and .[env.SIG_IMAGE_NAME] == { @@ -103,7 +113,7 @@ Describe 'create-e2e-vhd-metadata.sh' It "omits generalized metadata for $1 with publishing=$2, prefetch=$3, compatible=$4 without querying Azure" OS_SKU="$1" GENERATE_PUBLISHING_INFO="$2" RUN_PREFETCH_OPTIMIZATION="$3" PREFETCH_COMPATIBLE="$4" - When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + When run bash "${INLINE_METADATA_SCRIPT}" The status should be success The output should include 'Created E2E VHD metadata:' The contents of file "${QUERY_LOG}" should equal '' @@ -122,7 +132,7 @@ Describe 'create-e2e-vhd-metadata.sh' OS_SKU="$1" FEATURE_FLAGS="$2" SIG_IMAGE_NAME="$3" SKU_NAME="$3" MANAGED_SIG_ID="${GALLERY_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" - When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + When run bash "${INLINE_METADATA_SCRIPT}" The status should be success The output should include 'Created E2E VHD metadata:' The contents of file "${QUERY_LOG}" should equal '' @@ -162,7 +172,7 @@ Describe 'create-e2e-vhd-metadata.sh' It "omits $1 with a warning and preserves captured metadata" MOCK_VERSION="$(jq -c "$2" <<<"${MOCK_VERSION}")" - When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + When run bash "${INLINE_METADATA_SCRIPT}" The status should be success The output should include 'Reading generalized CVM image' The output should include 'Created E2E VHD metadata:' @@ -174,7 +184,7 @@ Describe 'create-e2e-vhd-metadata.sh' It 'warns and preserves captured metadata when the generalized image lookup fails' MOCK_AZ_STATUS=37 - When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + When run bash "${INLINE_METADATA_SCRIPT}" The status should be success The output should include 'Reading generalized CVM image' The output should include 'Created E2E VHD metadata:' @@ -185,7 +195,7 @@ Describe 'create-e2e-vhd-metadata.sh' It 'rejects a missing captured image ID' MANAGED_SIG_ID='' - When run bash .pipelines/scripts/create-e2e-vhd-metadata.sh "${METADATA_FILE}" + When run bash "${INLINE_METADATA_SCRIPT}" The status should be failure The stderr should include 'Packer did not return a managed SIG image version ID' The path "${METADATA_FILE}" should not be exist From d579e3f9aa6c74a0810a4b693eb38072e3f07b96 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 20 Oct 2026 16:23:46 -0400 Subject: [PATCH 33/37] fix --- e2e/README.md | 17 -- .../packer/publish_generalized_cvm_spec.sh | 223 ------------------ spec/vhdbuilder/prefetch/optimize_spec.sh | 203 ---------------- 3 files changed, 443 deletions(-) delete mode 100644 spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh delete mode 100644 spec/vhdbuilder/prefetch/optimize_spec.sh diff --git a/e2e/README.md b/e2e/README.md index 2fa8d78450f..afdcac7fa26 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -16,23 +16,6 @@ From a high-level, for each scenario, 3. Liveness and health checks and then run to make sure the new VM's kubelet is posting NodeReady, and that workload pods can successfully be scheduled and run on the new node. -### VHD image metadata - -The Linux VHD release pipeline publishes E2E image metadata after the image-publication -steps. All builds retain their original captured-image entry. Ubuntu and Azure Linux -CVM builds also attempt generalized gallery publication for AgentBaker E2Es when -prefetch runs (`GENERATE_PUBLISHING_INFO`, `RUN_PREFETCH_OPTIMIZATION`, and -`PREFETCH_COMPATIBLE` are all true). Metadata then adds a generalized entry keyed by -`SKU_NAME`, using the actual resource ID and completed replica regions read from Azure. -Publication is best-effort: missing or unready generalized versions produce warnings -and are omitted without failing the VHD build. Prefetch resources are cleaned up only -after the immutable VHD copy and any required gallery publication succeed. On gallery failure, -the Image Builder source is retained; rerunning prefetch retries gallery publication -without recopying an already-published VHD. Non-prefetch builds do not query or -advertise generalized versions, even if an older version exists. Metadata-backed E2Es -skip missing CVM images when `IgnoreScenariosWithMissingVhd` is enabled; otherwise -image selection fails. Production VHD output and ACL/non-CVM metadata are unchanged. - ## Writing and extending scenarios Extend an existing scenario's `Validator` when its node has the required settings. diff --git a/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh b/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh deleted file mode 100644 index 7636b631a66..00000000000 --- a/spec/vhdbuilder/packer/publish_generalized_cvm_spec.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env shellspec -# shellcheck shell=bash disable=SC2329 - -Describe 'publish-generalized-cvm-if-required.sh' - setup() { - export FEATURE_FLAGS=cvm OS_SKU=Ubuntu - export AZURE_RESOURCE_GROUP_NAME=test-rg SIG_GALLERY_NAME=test-gallery - export SKU_NAME=2404gen2CVMcontainerd CAPTURED_SIG_VERSION=1.2.3 - export PACKER_BUILD_LOCATION=eastus BUILD_BUILDID=123 - export BUILD_DEFINITION_NAME='AKS Linux VHD Build - TME Tenant' - export BUILD_NUMBER=20261019.1 BRANCH=refs/heads/zb/CVMABE2E IMG_SKU=server - export TAGS_FILE="${SHELLSPEC_WORKDIR}/cvm-version-tags" - export MOCK_DEFINITION_ID=/test/image-definition - export MOCK_FAIL_COMMAND='' MOCK_STATE=Succeeded - export MOCK_REPLICATION=Completed MOCK_REGION=eastus MOCK_REGION_STATE=Completed - export MOCK_PENDING_POLLS=0 MOCK_INITIAL_RESPONSE=pending - export POLL_TRACE_FILE="${SHELLSPEC_WORKDIR}/cvm-replication-polls" - : >"${POLL_TRACE_FILE}" - export -f az sleep - } - BeforeEach 'setup' - - az() { - local command="${1} ${2} ${3}" - if [[ "${command}" = "${MOCK_FAIL_COMMAND}" ]]; then - echo "Mock Azure operation failed: ${command}" >&2 - return 37 - fi - case "${command}" in - 'storage account list') - echo '{"id":"/test/storage-account","location":"eastus"}' - ;; - 'sig image-definition list') - echo "${MOCK_DEFINITION_ID}" - ;; - 'sig image-definition create') - echo /test/new-image-definition - ;; - 'resource update --ids') - ;; - 'sig image-version create') - while [[ "$#" -gt 0 ]]; do - if [[ "$1" = --target-regions ]]; then - echo "target-region $2" >>"${POLL_TRACE_FILE}" - fi - if [[ "$1" = --tags ]]; then - shift - printf '%s\n' "$@" >"${TAGS_FILE}" - break - fi - shift - done - ;; - 'sig image-version show') - echo show >>"${POLL_TRACE_FILE}" - local show_count - show_count="$(grep -c '^show$' "${POLL_TRACE_FILE}")" - if ((show_count <= MOCK_PENDING_POLLS)); then - case "${MOCK_INITIAL_RESPONSE}" in - empty) return 0 ;; - null) echo null; return 0 ;; - missing) return 3 ;; - pending) echo '{"state":"Updating","replication":{"aggregatedState":"InProgress"}}'; return 0 ;; - esac - fi - jq -nc --arg state "${MOCK_STATE}" --arg replication "${MOCK_REPLICATION}" \ - --arg region "${MOCK_REGION}" --arg regionState "${MOCK_REGION_STATE}" \ - '{state:$state,replication:{aggregatedState:$replication,summary:[{region:$region,state:$regionState}]}}' - ;; - *) - echo "Unexpected Azure command: ${command}" >&2 - return 99 - ;; - esac - } - - sleep() { - echo "sleep $*" >>"${POLL_TRACE_FILE}" - SECONDS=$((SECONDS + $1)) - } - - publish_and_check_polling() { - bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ - https://teststorage.blob.core.windows.net/vhds/test.vhd && - [[ "$(grep -c '^show$' "${POLL_TRACE_FILE}")" -eq 3 ]] && - [[ "$(grep -c '^sleep 60$' "${POLL_TRACE_FILE}")" -eq 2 ]] - } - - publish_and_check_tags() { - bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ - https://teststorage.blob.core.windows.net/vhds/test.vhd && - jq -Rsc ' - split("\n") | map(select(length > 0) | capture("^(?[^=]+)=(?.*)$")) | from_entries | - .now as $now | - . == { - buildDefinitionName: env.BUILD_DEFINITION_NAME, - buildNumber: env.BUILD_NUMBER, - buildId: env.BUILD_BUILDID, - SkipLinuxAzSecPack: "true", - os: "Linux", - now: $now, - createdBy: "aks-vhd-pipeline", - image_sku: env.IMG_SKU, - branch: env.BRANCH - } and ($now | test("^[0-9]+$")) and - (($now | tonumber) <= now) and (($now | tonumber) > now - 60) - ' "${TAGS_FILE}" | grep -qx true - } - - Describe 'ready images' - Parameters - Ubuntu /test/image-definition server - AzureLinux None azure-linux-3-cvm - End - - It "publishes $1 with all standard version tags and verified readiness" - OS_SKU="$1" MOCK_DEFINITION_ID="$2" IMG_SKU="$3" - When call publish_and_check_tags - The status should be success - The output should include 'Image version creation completed' - The output should include "replicated successfully to ${PACKER_BUILD_LOCATION}" - The output should include "image ready in ${PACKER_BUILD_LOCATION}" - The output should not include 'provisioned' - The contents of file "${POLL_TRACE_FILE}" should not include 'sleep' - The contents of file "${POLL_TRACE_FILE}" should include "target-region ${PACKER_BUILD_LOCATION}" - End - End - - Describe 'replication polling' - Parameters - pending - empty - null - missing - End - - It "polls every 60 seconds until a $1 image response becomes ready" - MOCK_PENDING_POLLS=2 MOCK_INITIAL_RESPONSE="$1" - When call publish_and_check_polling - The status should be success - The output should include 'retrying in 60 seconds' - The output should include "image ready in ${PACKER_BUILD_LOCATION}" - End - End - - Describe 'unready images' - Parameters - Updating Completed eastus Completed - Succeeded InProgress eastus InProgress - Succeeded Completed westus Completed - Succeeded Completed eastus InProgress - End - - It "rejects state=$1, replication=$2, region=$3, regional state=$4" - MOCK_STATE="$1" MOCK_REPLICATION="$2" MOCK_REGION="$3" MOCK_REGION_STATE="$4" - When run bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ - https://teststorage.blob.core.windows.net/vhds/test.vhd - The status should be failure - The output should include 'Verifying image version' - The output should not include 'image ready in' - The output should not include 'replicated successfully' - The stderr should include "is not ready in ${PACKER_BUILD_LOCATION} after waiting 900 seconds for replication" - End - End - - Describe 'terminal provisioning or replication failures' - Parameters - Failed Completed Completed - Canceled Completed Completed - Succeeded Failed Failed - Succeeded InProgress Failed - End - - It "stops immediately for state=$1, replication=$2, regional state=$3" - MOCK_STATE="$1" MOCK_REPLICATION="$2" MOCK_REGION_STATE="$3" - When run bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ - https://teststorage.blob.core.windows.net/vhds/test.vhd - The status should be failure - The output should not include 'image ready in' - The stderr should include 'provisioning or replication failed' - The contents of file "${POLL_TRACE_FILE}" should not include 'sleep' - End - End - - It 'does not wait for aggregate replication when the build region is ready' - PACKER_BUILD_LOCATION=northeurope MOCK_REGION=northeurope MOCK_REPLICATION=InProgress - When call publish_and_check_tags - The status should be success - The output should include 'image ready in northeurope' - The contents of file "${POLL_TRACE_FILE}" should include 'target-region northeurope' - The contents of file "${POLL_TRACE_FILE}" should not include 'sleep' - End - - Describe 'Azure operation failures' - Parameters - 'storage account list' 'resolving source storage account' - 'sig image-definition list' 'looking up image definition' - 'sig image-definition create' 'creating image definition' - 'resource update --ids' 'enabling image definition updates' - 'sig image-version create' 'creating image version' - 'sig image-version show' 'verifying image version readiness' - End - - It "reports failure during $2 without declaring readiness" - MOCK_DEFINITION_ID=None MOCK_FAIL_COMMAND="$1" - When run bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ - https://teststorage.blob.core.windows.net/vhds/test.vhd - The status should equal 37 - The output should include 'Publishing generalized CVM image' - The output should not include 'image ready in' - The output should not include 'replicated successfully' - The stderr should include "failed during $2" - End - End - - It 'skips images that do not need generalized CVM publication' - OS_SKU=AzureContainerLinux - When run bash ./vhdbuilder/packer/publish-generalized-cvm-if-required.sh \ - https://teststorage.blob.core.windows.net/vhds/test.vhd - The status should be success - The output should include 'Generalized CVM publication is not required' - End -End diff --git a/spec/vhdbuilder/prefetch/optimize_spec.sh b/spec/vhdbuilder/prefetch/optimize_spec.sh deleted file mode 100644 index 3e82ade639a..00000000000 --- a/spec/vhdbuilder/prefetch/optimize_spec.sh +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env shellspec -# shellcheck shell=bash disable=SC2329 - -Describe 'prefetch publication and cleanup' - setup() { - export SUBSCRIPTION_ID=test LOCATION=eastus SIG_GALLERY_RESOURCE_GROUP_NAME=test-rg - export SIG_GALLERY_NAME=test-gallery SIG_IMAGE_NAME=specialized SKU_NAME=generalized - export STORAGE_ACCOUNT_BLOB_URL=https://test.blob.core.windows.net/vhd - export STORAGE_ACCOUNT_BLOB_URL_STAGING=https://test.blob.core.windows.net/staging - export VHD_NAME=test.vhd IMAGE_BUILDER_IDENTITY_ID=test-identity BUILD_RUN_NUMBER=test - export CAPTURED_SIG_VERSION=1.2.3 HYPERV_GENERATION=V2 FEATURE_FLAGS=cvm OS_SKU=Ubuntu - export ENABLE_TRUSTED_LAUNCH=false WORKLOAD_OPTIMIZATION_SCRIPT_URL=https://example.test/optimize.sh - export MOCK_EXISTS=false MOCK_COPY_STATUS=success MOCK_GALLERY_READY=false - export MOCK_GALLERY_REGION=eastus MOCK_GALLERY_REGION_STATE=Completed - export MOCK_STORAGE_STATUS=0 MOCK_GALLERY_STATUS=0 MOCK_ARTIFACT_STATUS=0 - export MOCK_ARTIFACT_URI=https://source.blob.core.windows.net/vhds/test.vhd - export MOCK_WAIT_STATUS=0 - export CALL_LOG="${SHELLSPEC_WORKDIR}/prefetch-calls" - : >"${CALL_LOG}" - } - BeforeEach 'setup' - - az() { - case "${1} ${2} ${3}" in - 'storage blob exists') - printf '{"exists":%s}\n' "${MOCK_EXISTS}" - ;; - 'storage blob show') - printf '{"properties":{"copy":{"status":"%s"}}}\n' "${MOCK_COPY_STATUS}" - ;; - 'sig image-version show') - echo gallery-check >>"${CALL_LOG}" - if [ "${MOCK_GALLERY_READY}" = true ]; then - jq -nc --arg region "${MOCK_GALLERY_REGION}" --arg state "${MOCK_GALLERY_REGION_STATE}" \ - '{provisioningState:"Succeeded",replicationStatus:{aggregatedState:"Completed",summary:[{region:$region,state:$state}]}}' - else - echo '{"provisioningState":"Updating","replicationStatus":{"aggregatedState":"InProgress"}}' - fi - ;; - 'resource show --ids') - echo artifact-read >>"${CALL_LOG}" - printf '%s\n' "${MOCK_ARTIFACT_URI}" - return "${MOCK_ARTIFACT_STATUS}" - ;; - *) - echo "Unexpected Azure command: $*" >&2 - return 99 - ;; - esac - } - - run_optimizer() ( - # Load the real functions without executing main, then replace resource operations. - # shellcheck disable=SC1090 - source <(sed '$d' vhdbuilder/prefetch/scripts/optimize.sh) 2>/dev/null - { set +x; } 2>/dev/null - ensure_image_builder_rg() { echo ensure-builder >>"${CALL_LOG}"; } - run_image_builder_template() { echo run-builder >>"${CALL_LOG}"; } - copy_optimized_vhd() { - echo storage-copy >>"${CALL_LOG}" - return "${MOCK_STORAGE_STATUS}" - } - copy_optimized_vhd_to_packer_gallery() { - printf 'gallery-publish %s\n' "$1" >>"${CALL_LOG}" - return "${MOCK_GALLERY_STATUS}" - } - wait_for_vhd_copy() { - echo storage-wait >>"${CALL_LOG}" - return "${MOCK_WAIT_STATUS}" - } - main - ) - - It 'allows cleanup only after both publications succeed' - When run run_optimizer - The status should be success - The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]false' - The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - The contents of file "${CALL_LOG}" should include 'storage-copy' - The contents of file "${CALL_LOG}" should include "gallery-publish ${MOCK_ARTIFACT_URI}" - End - - It 'keeps gallery failure warning-only while blocking cleanup' - MOCK_GALLERY_STATUS=37 - When run run_optimizer - The status should be success - The output should include 'Generalized CVM gallery publication failed (exit 37)' - The output should include 'preserving prefetch resources for retry' - The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]false' - The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - End - - It 'still fails storage delivery and blocks cleanup even if gallery publication succeeds' - MOCK_STORAGE_STATUS=23 - When run run_optimizer - The status should equal 23 - The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - The contents of file "${CALL_LOG}" should include 'gallery-publish' - End - - Describe 'gallery recovery with an existing immutable VHD' - Parameters - success - pending - End - - It "retries gallery publication after storage state $1 without rebuilding or recopying" - MOCK_EXISTS=true MOCK_COPY_STATUS="$1" - When run run_optimizer - The status should be success - The output should include 'Retrying generalized CVM publication' - The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - The contents of file "${CALL_LOG}" should include "gallery-publish ${MOCK_ARTIFACT_URI}" - The contents of file "${CALL_LOG}" should not include 'storage-copy' - The contents of file "${CALL_LOG}" should not include 'run-builder' - The contents of file "${CALL_LOG}" should not include 'ensure-builder' - End - End - - It 'allows an already-complete CVM rerun without needing the deleted source' - LOCATION=northeurope MOCK_GALLERY_REGION=northeurope - MOCK_EXISTS=true MOCK_GALLERY_READY=true MOCK_ARTIFACT_STATUS=99 - When run run_optimizer - The status should be success - The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - The contents of file "${CALL_LOG}" should equal 'gallery-check' - End - - Describe 'publication is not required' - Parameters - None Ubuntu - cvm ACL - End - - It "preserves the existing early return for feature flags $1 and OS $2" - MOCK_EXISTS=true FEATURE_FLAGS="$1" OS_SKU="$2" - When run run_optimizer - The status should be success - The output should include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - The contents of file "${CALL_LOG}" should equal '' - End - End - - Describe 'gallery replication must be complete in the build location' - Parameters - eastus InProgress - westus Completed - End - - It "retries publication when region $1 has state $2" - MOCK_EXISTS=true MOCK_GALLERY_READY=true - MOCK_GALLERY_REGION="$1" MOCK_GALLERY_REGION_STATE="$2" - When run run_optimizer - The status should be success - The output should include 'Retrying generalized CVM publication' - The contents of file "${CALL_LOG}" should include "gallery-publish ${MOCK_ARTIFACT_URI}" - End - End - - Describe 'unavailable retained output' - Parameters - 37 https://source.blob.core.windows.net/vhds/test.vhd - 0 null - End - - It "keeps an already-delivered VHD successful with source status $1 and URI $2 but blocks cleanup" - MOCK_EXISTS=true MOCK_ARTIFACT_STATUS="$1" MOCK_ARTIFACT_URI="$2" - When run run_optimizer - The status should be success - The output should include 'Unable to read retained Image Builder output' - The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - The contents of file "${CALL_LOG}" should not include 'gallery-publish' - End - End - - It 'still fails a fresh build if Image Builder returns an empty URI' - MOCK_ARTIFACT_URI='' - When run run_optimizer - The status should equal 1 - The stderr should include 'unable to determine artifactUri' - The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - End - - It 'does not treat a failed immutable copy as recoverable' - MOCK_EXISTS=true MOCK_COPY_STATUS=failed - When run run_optimizer - The status should equal 1 - The output should include 'unrecoverable copy state' - The output should not include 'variable=PREFETCH_READY_FOR_CLEANUP]true' - The contents of file "${CALL_LOG}" should equal '' - End - - cleanup_requires_both_publications() { - sed -n '/make -f packer.mk cleanup-prefetch/,/displayName: Cleanup Prefetch Optimization Resources/p' \ - .pipelines/templates/.builder-release-template.yaml | - grep -Fq "eq(variables.PREFETCH_READY_FOR_CLEANUP, 'true')" - } - - It 'gates the pipeline cleanup step on successful publication' - When call cleanup_requires_both_publications - The status should be success - End -End From e2a9e8a882301711baee0d7ec5a0fc4856084922 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 20 Oct 2026 16:42:15 -0400 Subject: [PATCH 34/37] fix --- .../templates/.builder-release-template.yaml | 1 - spec/e2e/create_vhd_metadata_spec.sh | 218 ------------------ vhdbuilder/prefetch/scripts/optimize.sh | 94 +++----- 3 files changed, 26 insertions(+), 287 deletions(-) delete mode 100644 spec/e2e/create_vhd_metadata_spec.sh diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 50d7ed3c07b..9c72fc48166 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -370,7 +370,6 @@ steps: condition: | and( succeeded(), - eq(variables.PREFETCH_READY_FOR_CLEANUP, 'true'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables.RUN_PREFETCH_OPTIMIZATION, 'True'), eq(variables.PREFETCH_COMPATIBLE, 'True') diff --git a/spec/e2e/create_vhd_metadata_spec.sh b/spec/e2e/create_vhd_metadata_spec.sh deleted file mode 100644 index d552c395b29..00000000000 --- a/spec/e2e/create_vhd_metadata_spec.sh +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env shellspec -# shellcheck shell=bash disable=SC2329 - -Describe 'Create E2E VHD metadata pipeline step' - setup() { - TEST_ROOT="$(mktemp -d "${SHELLSPEC_WORKDIR}/vhd-metadata.XXXXXX")" - export BUILD_ARTIFACTSTAGINGDIRECTORY="${TEST_ROOT}/output" VHD_ARTIFACT_NAME=image - METADATA_FILE="${BUILD_ARTIFACTSTAGINGDIRECTORY}/e2e-vhd-metadata/${VHD_ARTIFACT_NAME}.json" - INLINE_METADATA_SCRIPT="${TEST_ROOT}/metadata.sh" - awk ' - /^ inlineScript: \|$/ {script = ""; collecting = 1; next} - collecting && /^ / {script = script substr($0, 9) "\n"; next} - collecting && /^$/ {script = script "\n"; next} - {collecting = 0} - /^ displayName: Create E2E VHD metadata$/ {printf "%s", script; found = 1; exit} - END {if (!found) exit 1} - ' .pipelines/templates/.builder-release-template.yaml >"${INLINE_METADATA_SCRIPT}" || return $? - export QUERY_LOG="${TEST_ROOT}/query.log" - : >"${QUERY_LOG}" - export OS_SKU=Ubuntu FEATURE_FLAGS=cvm PACKER_BUILD_LOCATION=eastus - export GENERATE_PUBLISHING_INFO=True RUN_PREFETCH_OPTIMIZATION=True PREFETCH_COMPATIBLE=True - export REPLICATIONS='westus=1, eastus2=2' - export SKU_NAME=2404gen2CVMcontainerd SIG_IMAGE_NAME=2404gen2CVMcontainerdSpecialized - export CAPTURED_SIG_VERSION=1.2.3 - export GALLERY_ID=/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Compute/galleries/test-gallery - export MANAGED_SIG_ID="${GALLERY_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" - export MOCK_AZ_STATUS=0 MOCK_VERSION - set_mock_version - export -f az - } - BeforeEach 'setup' - - set_mock_version() { - MOCK_VERSION="$(jq -nc \ - --arg id "${GALLERY_ID}/images/${SKU_NAME}/versions/${CAPTURED_SIG_VERSION}" ' - { - id: $id, - provisioningState: "Succeeded", - replicationStatus: { - aggregatedState: "Completed", - summary: [ - {region: "westus", state: "Completed"}, - {region: "eastus", state: "Completed"}, - {region: "westus", state: "Completed"} - ] - } - }')" - } - - az() { - printf '%s\n' "$@" >"${QUERY_LOG}" - if [ "${MOCK_AZ_STATUS}" -ne 0 ]; then - echo 'ResourceNotFound: generalized image version is missing' >&2 - return "${MOCK_AZ_STATUS}" - fi - printf '%s\n' "${MOCK_VERSION}" - } - - create_and_check_metadata() { - bash "${INLINE_METADATA_SCRIPT}" && - jq -e --arg generalizedId "${GALLERY_ID}/images/${SKU_NAME}/versions/${CAPTURED_SIG_VERSION}" ' - length == 2 and - .[env.SIG_IMAGE_NAME] == { - resourceId: env.MANAGED_SIG_ID, - version: env.CAPTURED_SIG_VERSION, - regions: ["eastus", "eastus2", "westus"] - } and - .[env.SKU_NAME] == { - resourceId: $generalizedId, - version: env.CAPTURED_SIG_VERSION, - regions: ["eastus", "westus"] - } - ' "${METADATA_FILE}" >/dev/null && - grep -Fxq "${GALLERY_ID}/images/${SKU_NAME}/versions/${CAPTURED_SIG_VERSION}" "${QUERY_LOG}" && - grep -Fxq ReplicationStatus "${QUERY_LOG}" - } - - check_captured_metadata() { - jq -e ' - length == 1 and .[env.SIG_IMAGE_NAME] == { - resourceId: env.MANAGED_SIG_ID, - version: env.CAPTURED_SIG_VERSION, - regions: ["eastus", "eastus2", "westus"] - } - ' "${METADATA_FILE}" >/dev/null - } - - Describe 'generalized CVM metadata' - Parameters - Ubuntu 2404gen2CVMcontainerd 2404gen2CVMcontainerdSpecialized - Ubuntu 2604minimalgen2CVMcontainerd 2604minimalgen2CVMcontainerdSpecialized - AzureLinux V3gen2CVM AzureLinuxV3gen2CVM - End - - It "adds the actual generalized ID for prefetched $1/$2 and preserves the captured entry" - OS_SKU="$1" SKU_NAME="$2" SIG_IMAGE_NAME="$3" - MANAGED_SIG_ID="${GALLERY_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" - set_mock_version - When call create_and_check_metadata - The status should be success - The output should include "Added image ready for E2Es: ${SKU_NAME}/${CAPTURED_SIG_VERSION}" - The output should include 'Created E2E VHD metadata:' - End - End - - Describe 'CVM builds without prefetch' - Parameters - Ubuntu False True True - AzureLinux True False True - Ubuntu True True False - AzureLinux '' '' '' - End - - It "omits generalized metadata for $1 with publishing=$2, prefetch=$3, compatible=$4 without querying Azure" - OS_SKU="$1" GENERATE_PUBLISHING_INFO="$2" RUN_PREFETCH_OPTIMIZATION="$3" PREFETCH_COMPATIBLE="$4" - When run bash "${INLINE_METADATA_SCRIPT}" - The status should be success - The output should include 'Created E2E VHD metadata:' - The contents of file "${QUERY_LOG}" should equal '' - The result of function check_captured_metadata should be successful - End - End - - Describe 'captured-image metadata' - Parameters - Ubuntu None 2404gen2containerd - AzureLinux None AzureLinuxV3gen2 - AzureContainerLinux cvm aclgen2CVM - End - - It "preserves existing behavior for $1/$2 without an Azure query" - OS_SKU="$1" FEATURE_FLAGS="$2" - SIG_IMAGE_NAME="$3" SKU_NAME="$3" - MANAGED_SIG_ID="${GALLERY_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" - When run bash "${INLINE_METADATA_SCRIPT}" - The status should be success - The output should include 'Created E2E VHD metadata:' - The contents of file "${QUERY_LOG}" should equal '' - The value "$(jq -c --arg id "${MANAGED_SIG_ID}" 'length == 1 and .[env.SIG_IMAGE_NAME].resourceId == $id and .[env.SIG_IMAGE_NAME].regions == ["eastus", "eastus2", "westus"]' "${METADATA_FILE}")" should equal true - End - End - - It 'advertises only confirmed replica regions' - MOCK_VERSION="$(jq -c '.replicationStatus.aggregatedState = "InProgress" | .replicationStatus.summary += [{region:"westus2",state:"InProgress"}]' <<<"${MOCK_VERSION}")" - When call create_and_check_metadata - The status should be success - The output should include 'replicated regions: eastus, westus' - The output should include 'Created E2E VHD metadata:' - End - - It 'matches the publisher gate case-insensitively' - OS_SKU=ubuntu FEATURE_FLAGS=CVM - GENERATE_PUBLISHING_INFO=true RUN_PREFETCH_OPTIMIZATION=TRUE PREFETCH_COMPATIBLE=true - When call create_and_check_metadata - The status should be success - The output should include 'Added image ready for E2Es:' - The output should include 'Created E2E VHD metadata:' - End - - Describe 'unready or incorrect generalized versions' - Parameters - missing-id 'del(.id)' - captured-id '.id = env.MANAGED_SIG_ID' - wrong-version '.id |= sub("1.2.3$"; "1.2.4")' - updating '.provisioningState = "Updating"' - replicating '.replicationStatus.summary |= map(if .region == "eastus" then .state = "InProgress" else . end)' - wrong-region '.replicationStatus.summary = [{region:"westus",state:"Completed"}]' - no-regions '.replicationStatus.summary = []' - incomplete-region '.replicationStatus.summary = [{region:"westus",state:"InProgress"}]' - invalid-region '.replicationStatus.summary = [{region:null,state:"Completed"}]' - End - - It "omits $1 with a warning and preserves captured metadata" - MOCK_VERSION="$(jq -c "$2" <<<"${MOCK_VERSION}")" - When run bash "${INLINE_METADATA_SCRIPT}" - The status should be success - The output should include 'Reading generalized CVM image' - The output should include 'Created E2E VHD metadata:' - The stderr should include 'task.logissue type=warning' - The stderr should include 'is not ready with confirmed replica regions; omitting it' - The result of function check_captured_metadata should be successful - End - End - - It 'warns and preserves captured metadata when the generalized image lookup fails' - MOCK_AZ_STATUS=37 - When run bash "${INLINE_METADATA_SCRIPT}" - The status should be success - The output should include 'Reading generalized CVM image' - The output should include 'Created E2E VHD metadata:' - The stderr should include 'task.logissue type=warning' - The stderr should include 'Unable to read generalized CVM image' - The result of function check_captured_metadata should be successful - End - - It 'rejects a missing captured image ID' - MANAGED_SIG_ID='' - When run bash "${INLINE_METADATA_SCRIPT}" - The status should be failure - The stderr should include 'Packer did not return a managed SIG image version ID' - The path "${METADATA_FILE}" should not be exist - End - - check_pipeline_order() { - awk ' - /displayName: Run Prefetch Optimization and Convert to VHD blob/ {prefetch = NR} - /displayName: Copy to Immutable Storage Container/ {copy = NR} - /displayName: Create E2E VHD metadata/ {create = NR} - /displayName: Publish E2E VHD metadata/ {publish = NR} - END {exit !(prefetch > 0 && copy > prefetch && create > copy && publish > create)} - ' .pipelines/templates/.builder-release-template.yaml - } - - It 'creates and publishes metadata after both VHD publication paths' - When call check_pipeline_order - The status should be success - End -End diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index 1dacd1ced8e..939209ec644 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -56,78 +56,38 @@ DESTINATION_STORAGE_ACCOUNT_NAME="${STORAGE_ACCOUNT_BLOB_URL#*://}" DESTINATION_STORAGE_ACCOUNT_NAME="${DESTINATION_STORAGE_ACCOUNT_NAME%%.*}" DESTINATION_CONTAINER_NAME="${STORAGE_ACCOUNT_BLOB_URL##*/}" STAGING_VHD_URI="${STORAGE_ACCOUNT_BLOB_URL_STAGING}/${VHD_NAME}" -VHD_ALREADY_PUBLISHED=false main() { - echo "##vso[task.setvariable variable=PREFETCH_READY_FOR_CLEANUP]false" - check_for_existing_vhd || return $? - - if [ "${VHD_ALREADY_PUBLISHED}" = "true" ]; then - if ! requires_generalized_cvm_publication || generalized_cvm_gallery_is_ready; then - echo "##vso[task.setvariable variable=PREFETCH_READY_FOR_CLEANUP]true" - return 0 - fi - echo "Retrying generalized CVM publication using the retained Image Builder output" - else - ensure_image_builder_rg || return $? - run_image_builder_template || return $? - fi - - local artifact_uri artifact_status=0 storage_pid="" gallery_pid storage_status=0 gallery_status=0 - artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv)" || artifact_status=$? - if [ "${artifact_status}" -ne 0 ] || [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then - if [ "${VHD_ALREADY_PUBLISHED}" = "true" ]; then - echo "##vso[task.logissue type=warning]Unable to read retained Image Builder output for generalized CVM publication; preserving prefetch resources" - return 0 - fi - echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" >&2 - [ "${artifact_status}" -ne 0 ] || artifact_status=1 - return "${artifact_status}" + # for idempotency, check whether the target VHD already exists. If a previous run already produced it + # successfully, exit early; if a server-side copy is still in progress, wait for it to finish. Because + # the destination container may be immutable, a VHD left in a bad state cannot be deleted and recreated + # and is treated as fatal. + check_for_existing_vhd + + # attempt to perform prefetch optimization, distributing the optimized image as a VHD into the + # image builder staging storage account, then copy that VHD into the target storage account + ensure_image_builder_rg || exit $? + run_image_builder_template || exit $? + + local artifact_uri storage_pid gallery_pid storage_status=0 + artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv)" || return $? + if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then + echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" + return 1 fi - if [ "${VHD_ALREADY_PUBLISHED}" != "true" ]; then - copy_optimized_vhd "${artifact_uri}" & - storage_pid=$! - fi + copy_optimized_vhd "${artifact_uri}" & + storage_pid=$! copy_optimized_vhd_to_packer_gallery "${artifact_uri}" & gallery_pid=$! - if [ -n "${storage_pid}" ]; then - wait "${storage_pid}" || storage_status=$? - fi - wait "${gallery_pid}" || gallery_status=$? - if [ "${gallery_status}" -ne 0 ]; then - echo "##vso[task.logissue type=warning]Generalized CVM gallery publication failed (exit ${gallery_status}); preserving prefetch resources for retry" - elif [ "${storage_status}" -eq 0 ]; then - echo "##vso[task.setvariable variable=PREFETCH_READY_FOR_CLEANUP]true" - fi + wait "${storage_pid}" || storage_status=$? + wait "${gallery_pid}" || echo "##vso[task.logissue type=warning]Generalized CVM gallery publication failed (exit $?)" return "${storage_status}" } -requires_generalized_cvm_publication() { - [[ "${FEATURE_FLAGS,,}" == *cvm* && "${OS_SKU,,}" =~ ^(ubuntu|azurelinux)$ ]] -} - -generalized_cvm_gallery_is_ready() { - local image_status - if ! image_status="$(az sig image-version show \ - --resource-group "${SIG_GALLERY_RESOURCE_GROUP_NAME}" --gallery-name "${SIG_GALLERY_NAME}" \ - --gallery-image-definition "${SKU_NAME}" --gallery-image-version "${CAPTURED_SIG_VERSION}" \ - --expand ReplicationStatus --output json --only-show-errors)"; then - echo "Unable to confirm generalized CVM gallery readiness; will retry publication" >&2 - return 1 - fi - jq -e --arg region "${LOCATION}" ' - .provisioningState == "Succeeded" and - any(.replicationStatus.summary[]?; .region == $region and .state == "Completed") - ' <<<"${image_status}" >/dev/null -} - check_for_existing_vhd() { - local exists copy_status - exists="$(az storage blob exists --blob-url "${VHD_URI}" --auth-mode login | jq -r '.exists')" || return $? - VHD_ALREADY_PUBLISHED=false - if [ "${exists}" = "false" ]; then + if [ "$(az storage blob exists --blob-url "${VHD_URI}" --auth-mode login | jq -r '.exists')" = "false" ]; then echo "no existing VHD was found at: ${VHD_URI}, will proceed with optimization and VHD creation" return 0 fi @@ -136,23 +96,21 @@ check_for_existing_vhd() { # the VHD in a potentially immutable container is produced by a server-side blob copy, so its copy status # tells us whether a previous run already completed. Because the container may have an immutability policy, # a VHD in a bad state cannot be deleted and recreated, so any non-success terminal state is fatal. - copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" || return $? + copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" if [ "${copy_status,,}" = "success" ]; then - echo "VHD ${VHD_URI} was already produced by a previous prefetch optimization run" - VHD_ALREADY_PUBLISHED=true - return 0 + echo "VHD ${VHD_URI} was already produced by a previous prefetch optimization run, nothing to do" + exit 0 fi if [ "${copy_status,,}" = "pending" ]; then echo "VHD ${VHD_URI} is currently being copied by a previous prefetch optimization run, will wait for completion" if wait_for_vhd_copy; then - VHD_ALREADY_PUBLISHED=true - return 0 + exit 0 fi fi # the VHD exists in a non-success state within the immutable container. It cannot be deleted due to # the immutability policy, so there is nothing we can do to recover - fail the script. echo "VHD ${VHD_URI} exists in an unrecoverable copy state: '${copy_status}'" - return 1 + exit 1 } ensure_image_builder_rg() { From e0fd2a28e3e0c97932355748adb8af99805abdf7 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Tue, 20 Oct 2026 16:53:09 -0400 Subject: [PATCH 35/37] fix --- vhdbuilder/packer/publish-generalized-cvm-if-required.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh index ccc641f127d..3646f2ab4a1 100755 --- a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh +++ b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh @@ -22,10 +22,9 @@ if [[ ! "${SOURCE_VHD_URI}" =~ ^https://([a-z0-9]{3,24})\.blob\.[^/]+/.+ ]]; the exit 1 fi source_storage_account="${BASH_REMATCH[1]}" -publication_stage="resolving source storage account (az storage account list)" report_publication_failure() { local exit_code="${1:-$?}" - echo "ERROR: Generalized CVM publication failed during ${publication_stage} (exit ${exit_code})" >&2 + echo "ERROR: Generalized CVM publication failed (exit ${exit_code})" >&2 exit "${exit_code}" } trap report_publication_failure ERR @@ -46,11 +45,9 @@ if [[ -z "${source_storage_id}" || -z "${source_location}" ]]; then fi echo "Resolved source storage account ${source_storage_account} in ${source_location}" -publication_stage="looking up image definition ${SIG_GALLERY_NAME}/${SKU_NAME} (az sig image-definition list)" echo "Looking up image definition ${SIG_GALLERY_NAME}/${SKU_NAME}" definition_id="$(az sig image-definition list "${gallery_args[@]}" --query "[?name=='${SKU_NAME}'].id | [0]" --output tsv)" if [[ -z "${definition_id}" || "${definition_id}" = "None" ]]; then - publication_stage="creating image definition ${SIG_GALLERY_NAME}/${SKU_NAME} (az sig image-definition create)" echo "Image definition not found; creating generalized image definition ${SIG_GALLERY_NAME}/${SKU_NAME}" definition_id="$( az sig image-definition create "${definition_args[@]}" \ @@ -61,7 +58,6 @@ if [[ -z "${definition_id}" || "${definition_id}" = "None" ]]; then --query id --output tsv )" echo "Created image definition ${definition_id}" - publication_stage="enabling image definition updates (az resource update)" echo "Enabling allowUpdateImage on ${definition_id}" az resource update --ids "${definition_id}" --set properties.allowUpdateImage=true --only-show-errors >/dev/null echo "Enabled allowUpdateImage on ${definition_id}" @@ -69,7 +65,6 @@ else echo "Using existing image definition ${definition_id}" fi -publication_stage="creating image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} (az sig image-version create)" CREATE_TIME="$(date +%s)" version_tags=( "buildDefinitionName=${BUILD_DEFINITION_NAME:?}" @@ -93,7 +88,6 @@ az sig image-version create "${version_args[@]}" \ echo "Image version creation completed for ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}; checking replication" echo "Waiting for image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replication to ${PACKER_BUILD_LOCATION} to complete" -publication_stage="verifying image version readiness (az sig image-version show)" echo "Verifying image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} is ready in ${PACKER_BUILD_LOCATION}" replication_deadline=$((SECONDS + 900)) while true; do From 71869d0f6fff7bfb46c68a80bf79c0c778371e57 Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 21 Oct 2026 15:29:21 -0400 Subject: [PATCH 36/37] fix --- e2e/config/vhd.go | 11 ++++++----- e2e/scenario/scenario_cvm.go | 12 +++++++++++- e2e/scenario/scenario_cvm_test.go | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 43c3bd311bd..7d6f7eb8b24 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -36,11 +36,12 @@ var ( var ( VHDUbuntu2004CVMGen2Containerd = &Image{ - Name: "2004gen2CVMcontainerd", - OS: OSUbuntu, - Arch: "amd64", - Distro: datamodel.AKSUbuntuContainerd2004CVMGen2, - Gallery: &Config.GalleryLinux, + Name: "2004gen2CVMcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2004CVMGen2, + Gallery: &Config.GalleryLinux, + UnsupportedLocalDns: true, } VHDUbuntu2204Gen2Arm64Containerd = &Image{ diff --git a/e2e/scenario/scenario_cvm.go b/e2e/scenario/scenario_cvm.go index 72cc3d7a238..4c18faa330b 100644 --- a/e2e/scenario/scenario_cvm.go +++ b/e2e/scenario/scenario_cvm.go @@ -3,6 +3,7 @@ package scenario import ( "context" + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -81,7 +82,7 @@ func newCVMScenario( useNVMe bool, securityEncryptionType armcompute.SecurityEncryptionTypes, ) *Scenario { - return &Scenario{ + s := &Scenario{ Name: name, Description: description, Location: cvmLocation, @@ -94,12 +95,21 @@ func newCVMScenario( UseNVMe: useNVMe, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.AgentPoolProfile.VMSize = vmSize + if vhd.Distro == datamodel.AKSUbuntuContainerd2004CVMGen2 { + nbc.AgentPoolProfile.LocalDNSProfile = nil + } }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { configureConfidentialVM(vmss, vmSize, securityEncryptionType) }, }, } + if vhd.Distro == datamodel.AKSUbuntuContainerd2004CVMGen2 { + s.AKSNodeConfigMutator = func(_ *Cluster, cfg *aksnodeconfigv1.Configuration) { + cfg.LocalDnsProfile = nil + } + } + return s } func configureConfidentialVM( diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go index 5e3cd9997aa..14a10680180 100644 --- a/e2e/scenario/scenario_cvm_test.go +++ b/e2e/scenario/scenario_cvm_test.go @@ -38,10 +38,12 @@ func TestRegisteredCVMScenarios(t *testing.T) { expected := map[string]struct { vhd *config.Image useNVMe bool + disableLocalDNS bool securityEncryptionType armcompute.SecurityEncryptionTypes }{ "Ubuntu2004_CVM_SNP": { vhd: config.VHDUbuntu2004CVMGen2Containerd, + disableLocalDNS: true, securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, }, "Ubuntu2404_CVM_SNP": { @@ -80,6 +82,25 @@ func TestRegisteredCVMScenarios(t *testing.T) { assert.False(t, s.SkipDefaultValidation) assert.Nil(t, s.Validator) + nbc, err := baseTemplateLinux("westus", "1.36.5", "amd64") + require.NoError(t, err) + s.BootstrapConfigMutator(nil, nbc) + nodeConfig, err := nbcToAKSNodeConfigV1(nbc) + require.NoError(t, err) + if s.AKSNodeConfigMutator != nil { + s.AKSNodeConfigMutator(nil, nodeConfig) + } + if want.disableLocalDNS { + assert.Nil(t, nbc.AgentPoolProfile.LocalDNSProfile) + assert.Nil(t, nodeConfig.LocalDnsProfile) + assert.True(t, s.VHD.UnsupportedLocalDns) + } else { + require.NotNil(t, nbc.AgentPoolProfile.LocalDNSProfile) + assert.True(t, nbc.AgentPoolProfile.LocalDNSProfile.EnableLocalDNS) + require.NotNil(t, nodeConfig.LocalDnsProfile) + assert.True(t, nodeConfig.LocalDnsProfile.EnableLocalDns) + } + vmss := &armcompute.VirtualMachineScaleSet{} s.VMConfigMutator(vmss) assert.Equal( From d8a9d3724f3ef63ebad9b0c918001315cfcaf5ab Mon Sep 17 00:00:00 2001 From: Zachary Bailey Date: Wed, 21 Oct 2026 17:58:19 -0400 Subject: [PATCH 37/37] fix --- .pipelines/scripts/verify_shell.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/scripts/verify_shell.sh b/.pipelines/scripts/verify_shell.sh index 9d5a441d633..3b2ec3253d5 100755 --- a/.pipelines/scripts/verify_shell.sh +++ b/.pipelines/scripts/verify_shell.sh @@ -31,6 +31,7 @@ filesToCheck=$(find . -type f -name "*.sh" -not -path './pkg/agent/testdata/*' - BASH_ONLY_LIST=$(cat <<'EOF' ./e2e/scenario/validate_anc_hotfix.sh ./vhdbuilder/packer/install-ig.sh +./vhdbuilder/packer/publish-generalized-cvm-if-required.sh ./vhdbuilder/packer/ubuntu-2604-cvm/trim-2604-cvm-packages.sh ./parts/linux/cloud-init/artifacts/npd-update.sh ./parts/linux/cloud-init/artifacts/aks-localdns-hosts-setup.sh