diff --git a/.pipelines/e2e.yaml b/.pipelines/e2e.yaml index c621ed1596c..d8244fbf39b 100644 --- a/.pipelines/e2e.yaml +++ b/.pipelines/e2e.yaml @@ -1,6 +1,6 @@ name: $(Date:yyyyMMdd)$(Rev:.r) variables: - TAGS_TO_SKIP: "os=windows,gpu=true" + TAGS_TO_SKIP: "os=windows,gpu=true,cvm=true" SKIP_E2E_TESTS: false E2E_FAILED_TESTS_RETRY_COUNT: 0 schedules: diff --git a/.pipelines/scripts/verify_shell.sh b/.pipelines/scripts/verify_shell.sh index 9d5a441d633..3b2ec3253d5 100755 --- a/.pipelines/scripts/verify_shell.sh +++ b/.pipelines/scripts/verify_shell.sh @@ -31,6 +31,7 @@ filesToCheck=$(find . -type f -name "*.sh" -not -path './pkg/agent/testdata/*' - BASH_ONLY_LIST=$(cat <<'EOF' ./e2e/scenario/validate_anc_hotfix.sh ./vhdbuilder/packer/install-ig.sh +./vhdbuilder/packer/publish-generalized-cvm-if-required.sh ./vhdbuilder/packer/ubuntu-2604-cvm/trim-2604-cvm-packages.sh ./parts/linux/cloud-init/artifacts/npd-update.sh ./parts/linux/cloud-init/artifacts/aks-localdns-hosts-setup.sh diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 6437015ab83..9c72fc48166 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -318,46 +318,6 @@ steps: RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) DRY_RUN: $(DRY_RUN) - - bash: | - set -euo pipefail - - metadata_dir="$(Build.ArtifactStagingDirectory)/e2e-vhd-metadata" - metadata_file="${metadata_dir}/${VHD_ARTIFACT_NAME}.json" - mkdir -p "$metadata_dir" - - if [ -z "${MANAGED_SIG_ID}" ]; then - echo "##vso[task.logissue type=error]Packer did not return a managed SIG image version ID" >&2 - exit 1 - fi - - regions="$(jq -nc \ - --arg buildLocation "${PACKER_BUILD_LOCATION}" \ - --arg replications "${REPLICATIONS:-}" \ - '([$buildLocation] + ($replications | split(",") | map(split("=")[0] | gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)))) | unique')" - - jq -n \ - --arg imageName "${SIG_IMAGE_NAME}" \ - --arg resourceId "${MANAGED_SIG_ID}" \ - --arg version "${CAPTURED_SIG_VERSION}" \ - --argjson regions "$regions" \ - '{($imageName): {resourceId: $resourceId, version: $version, regions: $regions}}' > "$metadata_file" - condition: succeeded() - displayName: Create E2E VHD metadata - env: - VHD_ARTIFACT_NAME: ${{ parameters.artifactName }} - SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) - CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) - MANAGED_SIG_ID: $(MANAGED_SIG_ID) - PACKER_BUILD_LOCATION: $(PACKER_BUILD_LOCATION) - REPLICATIONS: $(REPLICATIONS) - - - task: PublishPipelineArtifact@0 - condition: succeeded() - displayName: Publish E2E VHD metadata - inputs: - artifactName: 'e2e-vhd-metadata-${{ parameters.artifactName }}-stage$(System.StageAttempt)-job$(System.JobAttempt)' - targetPath: '$(Build.ArtifactStagingDirectory)/e2e-vhd-metadata/${{ parameters.artifactName }}.json' - - task: AzureCLI@2 inputs: azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) @@ -377,15 +337,23 @@ steps: env: SUBSCRIPTION_ID: $(SUBSCRIPTION_ID) LOCATION: $(PACKER_BUILD_LOCATION) + PACKER_BUILD_LOCATION: $(PACKER_BUILD_LOCATION) SIG_GALLERY_RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) + AZURE_RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) SIG_GALLERY_NAME: $(SIG_GALLERY_NAME) SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) SKU_NAME: $(SKU_NAME) + OS_SKU: $(OS_SKU) STORAGE_ACCOUNT_BLOB_URL: $(CLASSIC_BLOB) STORAGE_ACCOUNT_BLOB_URL_STAGING: $(CLASSIC_BLOB_STAGING) VHD_NAME: $(VHD_NAME) IMAGE_BUILDER_IDENTITY_ID: $(AZURE_MSI_RESOURCE_STRING) BUILD_RUN_NUMBER: $(Build.BuildNumber) + BUILD_BUILDID: $(Build.BuildId) + BUILD_DEFINITION_NAME: $(Build.DefinitionName) + BUILD_NUMBER: $(Build.BuildNumber) + BRANCH: $(Build.SourceBranch) + IMG_SKU: $(IMG_SKU) CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) HYPERV_GENERATION: $(HYPERV_GENERATION) FEATURE_FLAGS: $(FEATURE_FLAGS) @@ -509,6 +477,94 @@ steps: ) displayName: Copy to Immutable Storage Container + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -euo pipefail + + metadata_file="${BUILD_ARTIFACTSTAGINGDIRECTORY:?}/e2e-vhd-metadata/${VHD_ARTIFACT_NAME}.json" + + if [ -z "${MANAGED_SIG_ID:-}" ]; then + echo "##vso[task.logissue type=error]Packer did not return a managed SIG image version ID" >&2 + exit 1 + fi + + regions="$(jq -nc \ + --arg buildLocation "${PACKER_BUILD_LOCATION:?}" \ + --arg replications "${REPLICATIONS:-}" \ + '([$buildLocation] + ($replications | split(",") | map(split("=")[0] | gsub("^\\s+|\\s+$"; "")) | map(select(length > 0)))) | unique')" + + metadata="$(jq -nc \ + --arg imageName "${SIG_IMAGE_NAME:?}" \ + --arg resourceId "${MANAGED_SIG_ID}" \ + --arg version "${CAPTURED_SIG_VERSION:?}" \ + --argjson regions "${regions}" \ + '{($imageName): {resourceId: $resourceId, version: $version, regions: $regions}}')" + + prefetch_settings="${GENERATE_PUBLISHING_INFO:-False}:${RUN_PREFETCH_OPTIMIZATION:-False}:${PREFETCH_COMPATIBLE:-False}" + case "${prefetch_settings,,}:${OS_SKU,,}:${FEATURE_FLAGS,,}" in + true:true:true:ubuntu:*cvm*|true:true:true:azurelinux:*cvm*) + # Preserve the capture entry; query the generalized version in the same gallery. + gallery_id="${MANAGED_SIG_ID%/images/*}" + generalized_id="${gallery_id}/images/${SKU_NAME:?}/versions/${CAPTURED_SIG_VERSION}" + echo "Reading generalized CVM image ${SKU_NAME}/${CAPTURED_SIG_VERSION} for E2E metadata" + if ! generalized_version="$( + az sig image-version show --ids "${generalized_id}" --expand ReplicationStatus \ + --output json --only-show-errors + )"; then + echo "##vso[task.logissue type=warning]Unable to read generalized CVM image ${generalized_id}; omitting it from E2E metadata" >&2 + elif ! generalized_entry="$(jq -ce --arg expectedId "${generalized_id}" --arg version "${CAPTURED_SIG_VERSION}" --arg buildLocation "${PACKER_BUILD_LOCATION}" ' + select((.id | type) == "string") | + select((.id | ascii_downcase) == ($expectedId | ascii_downcase)) | + select(.provisioningState == "Succeeded" and + any(.replicationStatus.summary[]?; .region == $buildLocation and .state == "Completed")) | + { + resourceId: .id, + version: $version, + regions: ([.replicationStatus.summary[]? | + select(.state == "Completed") | .region | + select(type == "string" and length > 0)] | unique) + } | + select(.regions | length > 0) + ' <<<"${generalized_version}")"; then + echo "##vso[task.logissue type=warning]Generalized CVM image ${generalized_id} is not ready with confirmed replica regions; omitting it from E2E metadata" >&2 + else + metadata="$(jq -c --arg imageName "${SKU_NAME}" --argjson entry "${generalized_entry}" \ + '. + {($imageName): $entry}' <<<"${metadata}")" + echo "Added image ready for E2Es: ${SKU_NAME}/${CAPTURED_SIG_VERSION}, replicated regions: $(jq -r '.regions | join(", ")' <<<"${generalized_entry}")" + fi + ;; + esac + + mkdir -p "$(dirname "${metadata_file}")" + printf '%s\n' "${metadata}" >"${metadata_file}" + echo "Created E2E VHD metadata: ${metadata_file}" + condition: succeeded() + displayName: Create E2E VHD metadata + env: + GENERATE_PUBLISHING_INFO: $(GENERATE_PUBLISHING_INFO) + RUN_PREFETCH_OPTIMIZATION: $(RUN_PREFETCH_OPTIMIZATION) + PREFETCH_COMPATIBLE: $(PREFETCH_COMPATIBLE) + VHD_ARTIFACT_NAME: ${{ parameters.artifactName }} + SIG_IMAGE_NAME: $(SIG_IMAGE_NAME) + SKU_NAME: $(SKU_NAME) + OS_SKU: $(OS_SKU) + FEATURE_FLAGS: $(FEATURE_FLAGS) + CAPTURED_SIG_VERSION: $(CAPTURED_SIG_VERSION) + MANAGED_SIG_ID: $(MANAGED_SIG_ID) + PACKER_BUILD_LOCATION: $(PACKER_BUILD_LOCATION) + REPLICATIONS: $(REPLICATIONS) + + - task: PublishPipelineArtifact@0 + condition: succeeded() + displayName: Publish E2E VHD metadata + inputs: + artifactName: 'e2e-vhd-metadata-${{ parameters.artifactName }}-stage$(System.StageAttempt)-job$(System.JobAttempt)' + targetPath: '$(Build.ArtifactStagingDirectory)/e2e-vhd-metadata/${{ parameters.artifactName }}.json' + - task: DownloadPipelineArtifact@2 displayName: Download Build Performance Program condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme')) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index dd44c1e13be..7d6f7eb8b24 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -35,6 +35,15 @@ var ( ) var ( + VHDUbuntu2004CVMGen2Containerd = &Image{ + Name: "2004gen2CVMcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2004CVMGen2, + Gallery: &Config.GalleryLinux, + UnsupportedLocalDns: true, + } + VHDUbuntu2204Gen2Arm64Containerd = &Image{ Name: "2204gen2arm64containerd", OS: OSUbuntu, @@ -115,6 +124,14 @@ var ( Gallery: &Config.GalleryLinux, } + VHDAzureLinuxV3CVMGen2 = &Image{ + Name: "V3gen2CVM", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3CVMGen2, + Gallery: &Config.GalleryLinux, + } + // VHDAzureLinuxV3Gen2Kata is the AzureLinux V3 Gen2 VHD built with FEATURE_FLAGS=kata. // The image definition name mirrors the SIG_IMAGE_NAME produced by the VHD builder for // OS_VERSION=V3kata + HYPERV_GENERATION=V2 (SKU_NAME=V3katagen2, prefixed with "AzureLinux"). @@ -154,6 +171,14 @@ var ( Gallery: &Config.GalleryLinux, } + VHDUbuntu2404CVMGen2Containerd = &Image{ + Name: "2404gen2CVMcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2404CVMGen2, + Gallery: &Config.GalleryLinux, + } + VHDUbuntu2404ArmContainerd = &Image{ Name: "2404gen2arm64containerd", OS: OSUbuntu, @@ -170,6 +195,14 @@ var ( Gallery: &Config.GalleryLinux, } + VHDUbuntu2604MinimalCVMGen2Containerd = &Image{ + Name: "2604minimalgen2CVMcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuMinimalContainerd2604CVMGen2, + Gallery: &Config.GalleryLinux, + } + VHDUbuntu2604MinimalArm64Gen2Containerd = &Image{ Name: "2604minimalgen2arm64containerd", OS: OSUbuntu, @@ -197,6 +230,16 @@ var ( OSDiskSizeGB: 60, } + VHDACLCVMGen2 = &Image{ + Name: "aclgen2CVM", + OS: OSACL, + Arch: "amd64", + Distro: datamodel.AKSACLCVMGen2, + Gallery: &Config.GalleryLinux, + Flatcar: true, + OSDiskSizeGB: 60, + } + VHDACLArm64Gen2TL = &Image{ Name: "aclgen2arm64TL", OS: OSACL, diff --git a/e2e/runner/filter_test.go b/e2e/runner/filter_test.go index 2cf0bbffa1e..9fff63a8792 100644 --- a/e2e/runner/filter_test.go +++ b/e2e/runner/filter_test.go @@ -122,7 +122,7 @@ func restoreRunnerConfig(t *testing.T) { } func TestMatchFiltersPreservesTagPolicy(t *testing.T) { - tags := scenario.Tags{Name: "Ubuntu2204", OS: "linux", GPU: true} + tags := scenario.Tags{Name: "Ubuntu2204", OS: "linux", GPU: true, CVM: true} for _, test := range []struct { filter string all bool @@ -132,6 +132,7 @@ func TestMatchFiltersPreservesTagPolicy(t *testing.T) { {filter: "Name=Ubuntu2204,GPU=false", all: true, match: false}, {filter: "Name=Ubuntu2204,GPU=false", all: false, match: true}, {filter: " os = LINUX , gpu = true ", all: true, match: true}, + {filter: "CVM=true", all: true, match: true}, {filter: "", all: true, match: true}, } { t.Run(test.filter, func(t *testing.T) { diff --git a/e2e/scenario/scenario_cvm.go b/e2e/scenario/scenario_cvm.go new file mode 100644 index 00000000000..4c18faa330b --- /dev/null +++ b/e2e/scenario/scenario_cvm.go @@ -0,0 +1,157 @@ +package scenario + +import ( + "context" + + aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v8" +) + +const ( + cvmLocation = "westus" + cvmSNPSKU = "Standard_DC2ads_v5" + cvmTDXSKU = "Standard_DC2eds_v6" +) + +var ( + _ = Register(newCVMScenario( + "Ubuntu2004_CVM_SNP", + "Tests that an Ubuntu 20.04 Confidential VM node can be provisioned and pass standard node validation", + config.VHDUbuntu2004CVMGen2Containerd, + ClusterKubenet, + cvmSNPSKU, + false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, + )) + _ = Register(newCVMScenario( + "Ubuntu2404_CVM_SNP", + "Tests that an Ubuntu 24.04 Confidential VM node can be provisioned and pass standard node validation", + config.VHDUbuntu2404CVMGen2Containerd, + ClusterKubenet, + cvmSNPSKU, + false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, + )) + _ = Register(newCVMScenario( + "AzureLinuxV3_CVM_SNP", + "Tests that an Azure Linux V3 Confidential VM node can be provisioned and pass standard node validation", + config.VHDAzureLinuxV3CVMGen2, + ClusterKubenet, + cvmSNPSKU, + false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, + )) + _ = Register(newCVMScenario( + "AzureContainerLinux_CVM_SNP", + "Tests that an Azure Container Linux Confidential VM node can be provisioned and pass standard node validation", + config.VHDACLCVMGen2, + ClusterKubenet, + cvmSNPSKU, + false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, + )) + _ = Register(newCVMScenario( + "Ubuntu2604Minimal_CVM_SNP", + "Tests that an Ubuntu 26.04 minimal Confidential VM node can be provisioned on SEV-SNP and pass standard node validation", + config.VHDUbuntu2604MinimalCVMGen2Containerd, + ClusterLatestKubernetesVersionKubenet, + cvmSNPSKU, + false, + armcompute.SecurityEncryptionTypesDiskWithVMGuestState, + )) + _ = Register(newCVMScenario( + "Ubuntu2604Minimal_CVM_TDX", + "Tests that an Ubuntu 26.04 minimal Confidential VM node can be provisioned on TDX and pass standard node validation", + config.VHDUbuntu2604MinimalCVMGen2Containerd, + ClusterLatestKubernetesVersionKubenet, + cvmTDXSKU, + true, + armcompute.SecurityEncryptionTypesDiskWithVMGuestState, + )) +) + +func newCVMScenario( + name string, + description string, + vhd *config.Image, + cluster func(context.Context, ClusterRequest) (*Cluster, error), + vmSize string, + useNVMe bool, + securityEncryptionType armcompute.SecurityEncryptionTypes, +) *Scenario { + s := &Scenario{ + Name: name, + Description: description, + Location: cvmLocation, + Tags: Tags{ + CVM: true, + }, + Config: Config{ + Cluster: cluster, + VHD: vhd, + UseNVMe: useNVMe, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = vmSize + if vhd.Distro == datamodel.AKSUbuntuContainerd2004CVMGen2 { + nbc.AgentPoolProfile.LocalDNSProfile = nil + } + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + configureConfidentialVM(vmss, vmSize, securityEncryptionType) + }, + }, + } + if vhd.Distro == datamodel.AKSUbuntuContainerd2004CVMGen2 { + s.AKSNodeConfigMutator = func(_ *Cluster, cfg *aksnodeconfigv1.Configuration) { + cfg.LocalDnsProfile = nil + } + } + return s +} + +func configureConfidentialVM( + vmss *armcompute.VirtualMachineScaleSet, + vmSize string, + securityEncryptionType armcompute.SecurityEncryptionTypes, +) { + if vmss.SKU == nil { + vmss.SKU = &armcompute.SKU{} + } + vmss.SKU.Name = to.Ptr(vmSize) + + if vmss.Properties == nil { + vmss.Properties = &armcompute.VirtualMachineScaleSetProperties{} + } + if vmss.Properties.VirtualMachineProfile == nil { + vmss.Properties.VirtualMachineProfile = &armcompute.VirtualMachineScaleSetVMProfile{} + } + profile := vmss.Properties.VirtualMachineProfile + if profile.SecurityProfile == nil { + profile.SecurityProfile = &armcompute.SecurityProfile{} + } + profile.SecurityProfile.SecurityType = to.Ptr(armcompute.SecurityTypesConfidentialVM) + profile.SecurityProfile.EncryptionAtHost = to.Ptr(true) + if profile.SecurityProfile.UefiSettings == nil { + profile.SecurityProfile.UefiSettings = &armcompute.UefiSettings{} + } + profile.SecurityProfile.UefiSettings.SecureBootEnabled = to.Ptr(true) + profile.SecurityProfile.UefiSettings.VTpmEnabled = to.Ptr(true) + + if profile.StorageProfile == nil { + profile.StorageProfile = &armcompute.VirtualMachineScaleSetStorageProfile{} + } + if profile.StorageProfile.OSDisk == nil { + profile.StorageProfile.OSDisk = &armcompute.VirtualMachineScaleSetOSDisk{} + } + if profile.StorageProfile.OSDisk.ManagedDisk == nil { + profile.StorageProfile.OSDisk.ManagedDisk = &armcompute.VirtualMachineScaleSetManagedDiskParameters{} + } + if profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile == nil { + profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile = &armcompute.VMDiskSecurityProfile{} + } + profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType = + to.Ptr(securityEncryptionType) +} diff --git a/e2e/scenario/scenario_cvm_test.go b/e2e/scenario/scenario_cvm_test.go new file mode 100644 index 00000000000..14a10680180 --- /dev/null +++ b/e2e/scenario/scenario_cvm_test.go @@ -0,0 +1,158 @@ +package scenario + +import ( + "testing" + + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v8" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewCVMScenarioUsesWestUSAndDefaultValidation(t *testing.T) { + s := newCVMScenario( + "test", + "description", + config.VHDUbuntu2404CVMGen2Containerd, + ClusterKubenet, + cvmSNPSKU, + false, + armcompute.SecurityEncryptionTypesVMGuestStateOnly, + ) + + assert.Equal(t, "westus", s.Location) + assert.True(t, s.Tags.CVM) + assert.Equal(t, config.VHDUbuntu2404CVMGen2Containerd, s.VHD) + assert.False(t, s.SkipDefaultValidation) + assert.Nil(t, s.Validator) + + nbc := &datamodel.NodeBootstrappingConfiguration{ + AgentPoolProfile: &datamodel.AgentPoolProfile{}, + } + s.BootstrapConfigMutator(nil, nbc) + assert.Equal(t, cvmSNPSKU, nbc.AgentPoolProfile.VMSize) +} + +func TestRegisteredCVMScenarios(t *testing.T) { + expected := map[string]struct { + vhd *config.Image + useNVMe bool + disableLocalDNS bool + securityEncryptionType armcompute.SecurityEncryptionTypes + }{ + "Ubuntu2004_CVM_SNP": { + vhd: config.VHDUbuntu2004CVMGen2Containerd, + disableLocalDNS: true, + securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, + }, + "Ubuntu2404_CVM_SNP": { + vhd: config.VHDUbuntu2404CVMGen2Containerd, + securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, + }, + "AzureLinuxV3_CVM_SNP": { + vhd: config.VHDAzureLinuxV3CVMGen2, + securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, + }, + "AzureContainerLinux_CVM_SNP": { + vhd: config.VHDACLCVMGen2, + securityEncryptionType: armcompute.SecurityEncryptionTypesVMGuestStateOnly, + }, + "Ubuntu2604Minimal_CVM_SNP": { + vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, + securityEncryptionType: armcompute.SecurityEncryptionTypesDiskWithVMGuestState, + }, + "Ubuntu2604Minimal_CVM_TDX": { + vhd: config.VHDUbuntu2604MinimalCVMGen2Containerd, + useNVMe: true, + securityEncryptionType: armcompute.SecurityEncryptionTypesDiskWithVMGuestState, + }, + } + + found := make(map[string]bool, len(expected)) + for _, s := range List() { + want, ok := expected[s.Name] + if !ok { + continue + } + found[s.Name] = true + assert.Equal(t, "westus", s.Location) + assert.Equal(t, want.vhd, s.VHD) + assert.Equal(t, want.useNVMe, s.UseNVMe) + assert.False(t, s.SkipDefaultValidation) + assert.Nil(t, s.Validator) + + nbc, err := baseTemplateLinux("westus", "1.36.5", "amd64") + require.NoError(t, err) + s.BootstrapConfigMutator(nil, nbc) + nodeConfig, err := nbcToAKSNodeConfigV1(nbc) + require.NoError(t, err) + if s.AKSNodeConfigMutator != nil { + s.AKSNodeConfigMutator(nil, nodeConfig) + } + if want.disableLocalDNS { + assert.Nil(t, nbc.AgentPoolProfile.LocalDNSProfile) + assert.Nil(t, nodeConfig.LocalDnsProfile) + assert.True(t, s.VHD.UnsupportedLocalDns) + } else { + require.NotNil(t, nbc.AgentPoolProfile.LocalDNSProfile) + assert.True(t, nbc.AgentPoolProfile.LocalDNSProfile.EnableLocalDNS) + require.NotNil(t, nodeConfig.LocalDnsProfile) + assert.True(t, nodeConfig.LocalDnsProfile.EnableLocalDns) + } + + vmss := &armcompute.VirtualMachineScaleSet{} + s.VMConfigMutator(vmss) + assert.Equal( + t, + want.securityEncryptionType, + *vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType, + ) + } + for name := range expected { + assert.True(t, found[name], "CVM scenario %q is not registered", name) + } +} + +func TestConfigureConfidentialVM(t *testing.T) { + osProfile := &armcompute.VirtualMachineScaleSetOSProfile{} + vmss := &armcompute.VirtualMachineScaleSet{ + Properties: &armcompute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &armcompute.VirtualMachineScaleSetVMProfile{ + OSProfile: osProfile, + }, + }, + } + + configureConfidentialVM(vmss, cvmTDXSKU, armcompute.SecurityEncryptionTypesDiskWithVMGuestState) + + require.NotNil(t, vmss.SKU) + require.NotNil(t, vmss.SKU.Name) + assert.Equal(t, cvmTDXSKU, *vmss.SKU.Name) + + require.NotNil(t, vmss.Properties) + require.NotNil(t, vmss.Properties.VirtualMachineProfile) + profile := vmss.Properties.VirtualMachineProfile + assert.Same(t, osProfile, profile.OSProfile) + require.NotNil(t, profile.SecurityProfile) + require.NotNil(t, profile.SecurityProfile.SecurityType) + assert.Equal(t, armcompute.SecurityTypesConfidentialVM, *profile.SecurityProfile.SecurityType) + require.NotNil(t, profile.SecurityProfile.EncryptionAtHost) + assert.True(t, *profile.SecurityProfile.EncryptionAtHost) + require.NotNil(t, profile.SecurityProfile.UefiSettings) + require.NotNil(t, profile.SecurityProfile.UefiSettings.SecureBootEnabled) + assert.True(t, *profile.SecurityProfile.UefiSettings.SecureBootEnabled) + require.NotNil(t, profile.SecurityProfile.UefiSettings.VTpmEnabled) + assert.True(t, *profile.SecurityProfile.UefiSettings.VTpmEnabled) + + require.NotNil(t, profile.StorageProfile) + require.NotNil(t, profile.StorageProfile.OSDisk) + require.NotNil(t, profile.StorageProfile.OSDisk.ManagedDisk) + require.NotNil(t, profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile) + require.NotNil(t, profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType) + assert.Equal( + t, + armcompute.SecurityEncryptionTypesDiskWithVMGuestState, + *profile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile.SecurityEncryptionType, + ) +} diff --git a/e2e/scenario/types.go b/e2e/scenario/types.go index 33b87ce9a40..27a2c4456dd 100644 --- a/e2e/scenario/types.go +++ b/e2e/scenario/types.go @@ -24,6 +24,7 @@ type Tags struct { NetworkIsolated bool NonAnonymousACR bool GPU bool + CVM bool WASM bool Kata bool BootstrapTokenFallback bool diff --git a/vhdbuilder/packer/publish-generalized-cvm-if-required.sh b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh new file mode 100755 index 00000000000..3646f2ab4a1 --- /dev/null +++ b/vhdbuilder/packer/publish-generalized-cvm-if-required.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SOURCE_VHD_URI="${1:?source VHD URI is required}" + +if [[ "${FEATURE_FLAGS,,}" != *cvm* || ! "${OS_SKU,,}" =~ ^(ubuntu|azurelinux)$ ]]; then + echo "Generalized CVM publication is not required for OS_SKU=${OS_SKU:-unset}, FEATURE_FLAGS=${FEATURE_FLAGS:-unset}" + exit 0 +fi + +gallery_args=( + --resource-group "${AZURE_RESOURCE_GROUP_NAME:?}" + --gallery-name "${SIG_GALLERY_NAME:?}" + --only-show-errors +) +definition_args=("${gallery_args[@]}" --gallery-image-definition "${SKU_NAME:?}") +version_args=("${definition_args[@]}" --gallery-image-version "${CAPTURED_SIG_VERSION:?}") + +if [[ ! "${SOURCE_VHD_URI}" =~ ^https://([a-z0-9]{3,24})\.blob\.[^/]+/.+ ]]; then + echo "Source VHD must be an Azure Storage blob URL" >&2 + exit 1 +fi +source_storage_account="${BASH_REMATCH[1]}" +report_publication_failure() { + local exit_code="${1:-$?}" + echo "ERROR: Generalized CVM publication failed (exit ${exit_code})" >&2 + exit "${exit_code}" +} +trap report_publication_failure ERR + +echo "Publishing generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} in resource group ${AZURE_RESOURCE_GROUP_NAME}" +echo "Resolving source storage account ${source_storage_account}" +source_storage_info="$( + az storage account list \ + --query "[?name=='${source_storage_account}'].{id:id,location:location} | [0]" \ + --output json \ + --only-show-errors +)" +source_storage_id="$(jq -r '.id // empty' <<<"${source_storage_info}")" +source_location="$(jq -r '.location // empty' <<<"${source_storage_info}")" +if [[ -z "${source_storage_id}" || -z "${source_location}" ]]; then + echo "Unable to resolve source storage account ${source_storage_account} and its location" >&2 + exit 1 +fi +echo "Resolved source storage account ${source_storage_account} in ${source_location}" + +echo "Looking up image definition ${SIG_GALLERY_NAME}/${SKU_NAME}" +definition_id="$(az sig image-definition list "${gallery_args[@]}" --query "[?name=='${SKU_NAME}'].id | [0]" --output tsv)" +if [[ -z "${definition_id}" || "${definition_id}" = "None" ]]; then + echo "Image definition not found; creating generalized image definition ${SIG_GALLERY_NAME}/${SKU_NAME}" + definition_id="$( + az sig image-definition create "${definition_args[@]}" \ + --publisher microsoft-aks --offer "${SIG_GALLERY_NAME}" --sku "${SKU_NAME}" \ + --os-type Linux --os-state Generalized --architecture x64 --hyper-v-generation V2 \ + --location "${PACKER_BUILD_LOCATION:?}" \ + --features "DiskControllerTypes=SCSI,NVMe" "SecurityType=ConfidentialVMSupported" \ + --query id --output tsv + )" + echo "Created image definition ${definition_id}" + echo "Enabling allowUpdateImage on ${definition_id}" + az resource update --ids "${definition_id}" --set properties.allowUpdateImage=true --only-show-errors >/dev/null + echo "Enabled allowUpdateImage on ${definition_id}" +else + echo "Using existing image definition ${definition_id}" +fi + +CREATE_TIME="$(date +%s)" +version_tags=( + "buildDefinitionName=${BUILD_DEFINITION_NAME:?}" + "buildNumber=${BUILD_NUMBER:?}" + "buildId=${BUILD_BUILDID:?}" + "SkipLinuxAzSecPack=true" + "os=Linux" + "now=${CREATE_TIME}" + "createdBy=aks-vhd-pipeline" + "image_sku=${IMG_SKU:?}" + "branch=${BRANCH:?}" +) +echo "Creating image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} with buildId=${BUILD_BUILDID:?}, now=${CREATE_TIME}, source location ${source_location}, target region ${PACKER_BUILD_LOCATION:?}" +echo "Image version tags: ${version_tags[*]}" +az sig image-version create "${version_args[@]}" \ + --location "${source_location}" \ + --os-vhd-uri "${SOURCE_VHD_URI}" \ + --os-vhd-storage-account "${source_storage_id}" \ + --target-regions "${PACKER_BUILD_LOCATION}" \ + --tags "${version_tags[@]}" >/dev/null +echo "Image version creation completed for ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}; checking replication" + +echo "Waiting for image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replication to ${PACKER_BUILD_LOCATION} to complete" +echo "Verifying image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} is ready in ${PACKER_BUILD_LOCATION}" +replication_deadline=$((SECONDS + 900)) +while true; do + image_status="$( + az sig image-version show "${version_args[@]}" --expand ReplicationStatus \ + --query '{state:provisioningState,replication:replicationStatus}' --output json + )" || { show_status=$?; [[ "${show_status}" -eq 3 ]] || report_publication_failure "${show_status}"; } + echo "Image version status: ${image_status:-not found}" + readiness="$(jq -r --arg region "${PACKER_BUILD_LOCATION}" ' + if .state == "Succeeded" and + any(.replication.summary[]?; .region == $region and .state == "Completed") then "ready" + elif .state == "Failed" or .state == "Canceled" or + any(.replication.summary[]?; .region == $region and .state == "Failed") then "failed" + else "pending" end + ' <<<"${image_status:-null}")" + [[ "${readiness}" != "ready" ]] || break + [[ "${readiness}" != "failed" ]] || { echo "ERROR: Image provisioning or replication failed" >&2; exit 1; } + ((SECONDS < replication_deadline)) || { echo "ERROR: Image is not ready in ${PACKER_BUILD_LOCATION} after waiting 900 seconds for replication" >&2; exit 1; } + echo "Image version is not ready in ${PACKER_BUILD_LOCATION}; retrying in 60 seconds" + sleep 60 +done + +echo "Image version ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION} replicated successfully to ${PACKER_BUILD_LOCATION}" +echo "Generalized CVM image ${SIG_GALLERY_NAME}/${SKU_NAME}/${CAPTURED_SIG_VERSION}: image ready in ${PACKER_BUILD_LOCATION}" diff --git a/vhdbuilder/prefetch/scripts/optimize.sh b/vhdbuilder/prefetch/scripts/optimize.sh index f7ca0890757..939209ec644 100755 --- a/vhdbuilder/prefetch/scripts/optimize.sh +++ b/vhdbuilder/prefetch/scripts/optimize.sh @@ -68,7 +68,22 @@ main() { # image builder staging storage account, then copy that VHD into the target storage account ensure_image_builder_rg || exit $? run_image_builder_template || exit $? - copy_optimized_vhd || exit $? + + local artifact_uri storage_pid gallery_pid storage_status=0 + artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv)" || return $? + if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then + echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" + return 1 + fi + + copy_optimized_vhd "${artifact_uri}" & + storage_pid=$! + copy_optimized_vhd_to_packer_gallery "${artifact_uri}" & + gallery_pid=$! + + wait "${storage_pid}" || storage_status=$? + wait "${gallery_pid}" || echo "##vso[task.logissue type=warning]Generalized CVM gallery publication failed (exit $?)" + return "${storage_status}" } check_for_existing_vhd() { @@ -308,6 +323,7 @@ create_temp_storage() { # 1. azcopy the optimized VHD into the staging container (STORAGE_ACCOUNT_BLOB_URL_STAGING) # 2. perform a single server-side blob copy from the staging container into the immutable vhd container copy_optimized_vhd() { + local artifact_uri="${1:?Image Builder VHD URI is required}" # The staging container should never have an immutability policy, so copying from image builder staging storage is # fully retriable. If a previous run already produced a complete staging blob (marked with the # prefetched metadata flag) but failed before finishing the server-side copy into the @@ -323,11 +339,6 @@ copy_optimized_vhd() { delete_staging_vhd || return $? fi - artifact_uri="$(az resource show --ids "${IMAGE_BUILDER_TEMPLATE_ID}/runOutputs/${DISTRIBUTE_RUN_OUTPUT_NAME}" --api-version "${IMAGE_BUILDER_API_VERSION}" --query "properties.artifactUri" -o tsv 2>/dev/null)" - if [ -z "${artifact_uri}" ] || [ "${artifact_uri,,}" = "null" ] || [ "${artifact_uri,,}" = "none" ]; then - echo "unable to determine artifactUri for run output ${DISTRIBUTE_RUN_OUTPUT_NAME}, cannot continue" - return 1 - fi echo "setting azcopy environment variables with pool identity: ${IMAGE_BUILDER_IDENTITY_ID}" export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" export AZCOPY_CONCURRENCY_VALUE="AUTO" @@ -364,6 +375,11 @@ copy_optimized_vhd() { echo "optimized VHD has been published to: ${VHD_URI}" } +copy_optimized_vhd_to_packer_gallery() { + local artifact_uri="${1:?Image Builder VHD URI is required}" + "$(dirname "${BASH_SOURCE[0]}")/../../packer/publish-generalized-cvm-if-required.sh" "${artifact_uri}" +} + wait_for_vhd_copy() { copy_status="$(az storage blob show --blob-url "${VHD_URI}" --auth-mode login | jq -r '.properties.copy.status')" while [ "${copy_status,,}" = "pending" ]; do