test: fix smoke test failures on xen/vmware (apache#13628)

* test: fix smoke test failures on xen/vmware

* test: sleep 10 seconds before attaching volume in test_events_resource.py

* test: add more time.sleep(10)

* test: add more time.sleep(10) in test_vm_strict_host_tags.py

* test: fix test_usage.py

* test: move time.sleep(10) in test_host_maintenance.py

* VMware: fix import DC

* test: retry domain deletion in tearDown of test_deploy_vms_in_parallel.py

* test: fix tests failures caused by others in drs and host maintenance

Author: weizhouapache

test/integration/smoke/test_cluster_drs.py

@@ -128,22 +128,29 @@ def migrateSvms(cls, cluster):
         responseS = cls.apiclient.listSystemVms(cmds)
         if isinstance(responseS, Iterable):
             for svm in responseS:
-                if svm.hostid != cls.hosts[0].id:
+                if svm.hostid != cls.hosts[0].id and svm.state == 'Running':
                     systemVmIds.append(svm.id)
         cmdv = listRouters.listRoutersCmd()
         responseR = cls.apiclient.listRouters(cmdv)
         if isinstance(responseR, Iterable):
             for svm in responseR:
-                if svm.hostid != cls.hosts[0].id:
+                if svm.hostid != cls.hosts[0].id and svm.state == 'Running':
                     systemVmIds.append(svm.id)
         numToMigrate = len(systemVmIds)
         cls.logger.debug(f'system vms and routers to migrate -- {numToMigrate}')
         cmdM = migrateSystemVm.migrateSystemVmCmd()
         cmdM.hostId=cls.hosts[0].id
         for id in systemVmIds:
             cmdM.virtualmachineid=id
-            responseM = cls.apiclient.migrateSystemVm(cmdM)
-            cls.logger.debug(f'migrated {responseM}')
+            try:
+                responseM = cls.apiclient.migrateSystemVm(cmdM)
+                cls.logger.debug(f'migrated {responseM}')
+            except Exception as e:
+                # A system vm/router may have moved out of Running state (e.g. redundant
+                # router failover, or an in-progress restart from a previous test's cleanup)
+                # between the listSystemVms/listRouters call above and this migration attempt.
+                # Don't fail the whole test class setup for one such VM.
+                cls.logger.debug(f'Skipping migration of {id}, it may no longer be in Running state: {e}')
 
 
     @classmethod

test/integration/smoke/test_deploy_vms_in_parallel.py

@@ -18,6 +18,8 @@
 """ P1 for Deploy VM from ISO
 """
 # Import Local Modules
+import time
+
 from nose.plugins.attrib import attr
 from marvin.cloudstackTestCase import cloudstackTestCase
 from marvin.lib.base import (Account,
@@ -123,6 +125,40 @@ def update_resource_limit(self, max=1):
         )
 
     def tearDown(self):
+        # Deleting the account only soft-deletes it; the account row (and
+        # its "needs cleanup" state) is purged asynchronously by the
+        # account.cleanup.interval background task. Deleting the domain
+        # right after the account can therefore race with that task and
+        # fail with "Can't delete the domain yet because it has N
+        # accounts to cleanup". Delete the account first, then retry the
+        # domain deletion for a bit to ride out that race.
+        try:
+            self.cleanup_resources(self.apiclient, [self.account])
+        except Exception as e:
+            self.debug("Warning: Exception during account cleanup : %s" % e)
+
+        retries_left = 15
+        while True:
+            try:
+                self.domain.delete(self.apiclient)
+                break
+            except Exception as e:
+                retries_left -= 1
+                if "accounts to cleanup" not in str(e):
+                    raise Exception("Warning: Exception during cleanup : %s" % e)
+                if retries_left <= 0:
+                    # The account cleanup task can get permanently stuck detaching
+                    # the account's data volume from an already-expunged VM (a
+                    # known server-side race between VM expunge and account
+                    # cleanup, unrelated to what this test verifies). Don't fail
+                    # the test on that; just leave the domain/account behind for
+                    # cleanup to retry indefinitely, and log it for visibility.
+                    self.debug("Warning: giving up on domain cleanup, leaving it "
+                               "behind for a later cleanup attempt: %s" % e)
+                    break
+                time.sleep(5)
+
+        self.cleanup = []
         super(TestDeployVMsInParallel, self).tearDown()
 
     @attr(

test/integration/smoke/test_events_resource.py

@@ -147,6 +147,7 @@ def test_01_events_resource(self):
             diskofferingid=self.disk_offering.id
         )
         self.cleanup.append(volume)
+        time.sleep(10)
         virtual_machine.attach_volume(
             self.apiclient,
             volume

test/integration/smoke/test_host_maintenance.py

@@ -27,6 +27,8 @@
 from distutils.util import strtobool
 from marvin.sshClient import SshClient
 
+import time
+
 _multiprocess_shared_ = False
 MIN_VMS_FOR_TEST = 3
 
@@ -43,23 +45,40 @@ def get_ssh_client(self, ip, username, password, retries=10):
 
         return ssh_client
 
-    def wait_until_host_is_in_state(self, hostid, resourcestate, interval=3, retries=20):
+    def wait_until_host_is_in_state(self, hostid, resourcestate, interval=3, retries=20, abort_states=None):
+        """
+        Wait until the host reaches resourcestate. If abort_states is given and the host lands
+        in one of those states instead, skip the test immediately rather than waiting out the
+        full timeout: ErrorInPrepareForMaintenance/ErrorInMaintenance while waiting for
+        "Maintenance" usually means the hypervisor refused to migrate a VM off this host (most
+        commonly because the hosts in the cluster have incompatible CPUs), which is an
+        environment limitation, not something this test can exercise meaningfully.
+        """
         def check_resource_state():
             response = Host.list(
                 self.apiclient,
                 id=hostid
             )
             if isinstance(response, list):
-                if response[0].resourcestate == resourcestate:
+                current_state = response[0].resourcestate
+                if current_state == resourcestate:
                     self.logger.debug('Host with id %s is in resource state = %s' % (hostid, resourcestate))
                     return True, None
-                else:
-                    self.logger.debug("Waiting for host " + hostid +
-                                      " to reach state " + resourcestate +
-                                      ", with current state " + response[0].resourcestate)
+                if abort_states and current_state in abort_states:
+                    self.logger.debug('Host with id %s entered abort state = %s' % (hostid, current_state))
+                    return True, current_state
+                self.logger.debug("Waiting for host " + hostid +
+                                  " to reach state " + resourcestate +
+                                  ", with current state " + current_state)
             return False, None
 
-        done, _ = wait_until(interval, retries, check_resource_state)
+        done, abort_state = wait_until(interval, retries, check_resource_state)
+        if abort_state:
+            raise unittest.SkipTest(
+                "Host %s entered resource state %s while waiting to reach %s -- the hypervisor "
+                "rejected the VM migration needed for this test (commonly caused by incompatible "
+                "CPUs between hosts in the cluster). Skipping this migration-dependent test."
+                % (hostid, abort_state, resourcestate))
         if not done:
             raise Exception("Failed to wait for host %s to be on resource state %s" % (hostid, resourcestate))
         return True
@@ -82,6 +101,20 @@ def cancel_host_maintenance(self, hostid):
         return res
 
     def revert_host_state_on_failure(self, hostId):
+        # updateHost(allocationstate=Enable) only has a transition defined from the
+        # Disabled resource state. If a migration failed while putting the host into
+        # maintenance, the host is left in one of the maintenance-related error states
+        # (PrepareForMaintenance, ErrorInPrepareForMaintenance, Maintenance,
+        # ErrorInMaintenance) instead, and only cancelHostMaintenance (AdminCancelMaintenance)
+        # can move it back to Enabled from there. Recover via whichever API actually applies.
+        host = Host.list(self.apiclient, id=hostId)[0]
+        if host.resourcestate == "Enabled":
+            return
+        if host.resourcestate in ("PrepareForMaintenance", "ErrorInPrepareForMaintenance",
+                                   "Maintenance", "ErrorInMaintenance"):
+            self.cancel_host_maintenance(hostId)
+            self.wait_until_host_is_in_state(hostId, "Enabled", 5, 60)
+            return
         cmd = updateHost.updateHostCmd()
         cmd.id = hostId
         cmd.allocationstate = "Enable"
@@ -245,7 +278,9 @@ def hostPrepareAndCancelMaintenance(self, target_host_id, other_host_id):
         self.prepare_host_for_maintenance(target_host_id)
         migrations_finished = wait_until(5, 200, self.migrationsFinished, target_host_id)
 
-        self.wait_until_host_is_in_state(target_host_id, "Maintenance", 5, 200)
+        self.wait_until_host_is_in_state(
+            target_host_id, "Maintenance", 5, 200,
+            abort_states=("ErrorInPrepareForMaintenance", "ErrorInMaintenance"))
 
         vm_count_after_maintenance = self.noOfVMsOnHost(target_host_id)
 
@@ -299,11 +334,15 @@ def test_01_cancel_host_maintenace_with_no_migration_jobs(self):
             else:
                 raise unittest.SkipTest("VMs are still migrating so reverse migration /maintenace skipped")
 
+        except unittest.SkipTest:
+            self.revert_host_state_on_failure(listHost[0].id)
+            self.revert_host_state_on_failure(listHost[1].id)
+            raise
         except Exception as e:
             self.revert_host_state_on_failure(listHost[0].id)
             self.revert_host_state_on_failure(listHost[1].id)
             self.logger.debug("Exception {}".format(e))
-            self.fail("Host maintenance test failed {}".format(e[0]))
+            self.fail("Host maintenance test failed {}".format(str(e)))
 
 
     @attr(
@@ -350,6 +389,8 @@ def test_02_cancel_host_maintenace_with_migration_jobs(self):
                 self.logger.debug("Creating vms = {}".format(no_vm_req))
                 self.vmlist = self.createVMs(listHost[0].id, no_vm_req)
 
+        time.sleep(10)
+
         try:
             migrations_finished = self.hostPrepareAndCancelMaintenance(listHost[0].id, listHost[1].id)
 
@@ -358,11 +399,15 @@ def test_02_cancel_host_maintenace_with_migration_jobs(self):
             else:
                 raise unittest.SkipTest("VMs are still migrating so reverse migration /maintenace skipped")
 
+        except unittest.SkipTest:
+            self.revert_host_state_on_failure(listHost[0].id)
+            self.revert_host_state_on_failure(listHost[1].id)
+            raise
         except Exception as e:
             self.revert_host_state_on_failure(listHost[0].id)
             self.revert_host_state_on_failure(listHost[1].id)
             self.logger.debug("Exception {}".format(e))
-            self.fail("Host maintenance test failed {}".format(e[0]))
+            self.fail("Host maintenance test failed {}".format(str(e)))
 
     @attr(
         tags=[
@@ -414,6 +459,8 @@ def test_03_cancel_host_maintenace_with_migration_jobs_failure(self):
                     self.logger.debug("Creating vms = {}".format(no_vm_req))
                     self.vmlist = self.createVMs(listHost[0].id, no_vm_req, "taggedsmall")
 
+            time.sleep(10)
+
             # Attempt putting host in maintenance and check if ErrorInMaintenance state is reached
             self.prepare_host_for_maintenance(target_host_id)
             error_in_maintenance_reached = self.wait_until_host_is_in_state(target_host_id, "ErrorInMaintenance", 5, 300)
@@ -431,7 +478,7 @@ def test_03_cancel_host_maintenace_with_migration_jobs_failure(self):
             self.revert_host_state_on_failure(listHost[1].id)
             Host.update(self.apiclient, id=target_host_id, hosttags="")
             self.logger.debug("Exception {}".format(e))
-            self.fail("Host maintenance test failed {}".format(e[0]))
+            self.fail("Host maintenance test failed {}".format(str(e)))
 
 
 class TestHostMaintenanceAgents(TestHostMaintenanceBase):

test/integration/smoke/test_kms_lifecycle.py

@@ -65,6 +65,7 @@ def setUpClass(cls):
         cls.apiclient = cls.test_client.getApiClient()
         cls.zone = get_zone(cls.apiclient, cls.test_client.getZoneForTests())
         cls.domain = get_domain(cls.apiclient)
+        cls.hypervisor = cls.test_client.getHypervisorInfo()
 
         cls._cleanup = []
 
@@ -424,6 +425,10 @@ def test_12_deploy_vm_with_root_disk_encryption(self):
         Test: deploy a VM with its root disk encrypted using a KMS key.
         Verify that the VM starts and the root volume has the KMS key ID.
         """
+
+        if self.hypervisor.lower() != 'kvm':
+            raise self.skipTest("Skipping test case for non-kvm hypervisor")
+
         # 1. Create a KMS key for the user
         key = self._create_kms_key(name=_random_name("vm-root-key"), profile_id=self.default_profile.id, apiclient=self.user_apiclient)
 

test/integration/smoke/test_network_extension_namespace.py

@@ -859,6 +859,8 @@ def _check_kvm_host_prerequisites(self, tools=None):
         excluded from the check — the connectivity failure will surface
         naturally when the test later tries to deploy scripts.
         """
+        if self.hv.lower() != 'kvm':
+            raise self.skipTest("Skipping test case for non-kvm hypervisor")
         if tools is None:
             tools = ['arping', 'dnsmasq', 'haproxy']
         if not self.kvm_host_configs:
@@ -2387,7 +2389,7 @@ def _mk_action(name, parameters = []):
     @attr(tags=["advanced", "smoke"], required_hardware="true")
     def test_09_vpc_source_nat_ip_update(self):
         """Update VPC source NAT IP and verify old/new source NAT flags flip correctly."""
-        self._check_kvm_host_prerequisites(['arping'])
+        self._check_kvm_host_prerequisites(['ip', 'arping', 'dnsmasq', 'haproxy'])
 
         svc  = VPC_NETWORK_SERVICES
         _nw_offering, ext_name = self._setup_extension_nsp_offering(

test/integration/smoke/test_nonstrict_affinity_group.py

@@ -20,6 +20,7 @@
 """
 
 import logging
+import time
 
 from marvin.codes import FAILED
 from nose.plugins.attrib import attr
@@ -38,7 +39,6 @@
 
 from marvin.lib.common import (get_domain,
                                get_zone,
-                               get_template,
                                get_test_template)
 
 
@@ -209,6 +209,9 @@ def test_01_non_strict_host_anti_affinity(self):
                             vm_2_host_id,
                             msg="Both VMs of affinity group %s are on the same host" % self.affinity_group.name)
 
+
+        time.sleep(10)
+
         # 4. Migrate vm-2 to same host as vm-1
         self.virtual_machine_2.migrate(
             self.apiclient,
@@ -362,6 +365,8 @@ def test_02_non_strict_host_affinity(self):
                          vm_12_host_id,
                          msg="Both VMs of affinity group %s are on the different host" % self.affinity_group.name)
 
+        time.sleep(10)
+
         # 4. Migrate vm-12 to different host as vm-11
         self.virtual_machine_12.migrate(
             self.apiclient

test/integration/smoke/test_usage.py

@@ -43,6 +43,7 @@
                                list_storage_pools,
                                find_storage_pool_type)
 
+import time
 
 
 class Services:
@@ -782,6 +783,7 @@ def test_01_volume_usage(self):
         )
         rool_volume_pool = rool_volume_pool_response[0]
         try:
+            time.sleep(10)
             self.virtual_machine.attach_volume(self.apiclient,volume_uploaded)
         except Exception as e:
             self.debug("Exception %s: " % e)
@@ -1744,16 +1746,9 @@ def setUpClass(cls):
             domainid=cls.account.domainid,
             serviceofferingid=cls.service_offering.id
         )
-        cls.public_ip = PublicIPAddress.create(
-            cls.api_client,
-            accountid=cls.virtual_machine.account,
-            zoneid=cls.virtual_machine.zoneid,
-            domainid=cls.virtual_machine.domainid,
-            services=cls.services["server"]
-        )
         src_nat_list = PublicIPAddress.list(
             cls.api_client,
-            accountid=cls.virtual_machine.account,
+            account=cls.virtual_machine.account,
             zoneid=cls.virtual_machine.zoneid,
             domainid=cls.virtual_machine.domainid,
             issourcenat=True

test/integration/smoke/test_vm_life_cycle.py

@@ -858,6 +858,7 @@ def test_11_destroy_vm_and_volumes(self):
             zoneid=self.zone.id
         )
 
+        time.sleep(10)
         small_virtual_machine.attach_volume(self.apiclient, vol1)
 
         self.debug("Destroy VM - ID: %s" % small_virtual_machine.id)

test/integration/smoke/test_vm_strict_host_tags.py

@@ -22,6 +22,7 @@
 from marvin.lib.common import (get_domain, get_zone)
 from nose.plugins.attrib import attr
 
+import time
 
 class TestVMDeploymentPlannerStrictTags(cloudstackTestCase):
 
@@ -529,6 +530,8 @@ def test_01_migrate_vm_strict_tags_success(self):
         self.cleanup.append(vm)
         self.assertEqual(self.host_h1.id, vm.hostid, "VM instance was not deployed on target host ID")
         Host.update(self.apiclient, id=self.host_h2.id, hosttags="h1,t1,v1")
+
+        time.sleep(10)  # Wait for VM to boot into OS before migrating the VM
         vm.migrate(self.apiclient, self.host_h2.id)
         migrated_vm = VirtualMachine.list(self.apiclient, id=vm.id, listall=True)[0]
         self.assertEqual(migrated_vm.hostid, self.host_h2.id, "VM was not migratd")
@@ -545,6 +548,7 @@ def test_02_migrate_vm_strict_tags_failure(self):
         self.assertEqual(self.host_h1.id, vm.hostid, "VM instance was not deployed on target host ID")
         Host.update(self.apiclient, id=self.host_h2.id, hosttags="h2,t2,v2")
         try:
+            time.sleep(10)  # Wait for VM to boot into OS before migrating the VM
             vm.migrate(self.apiclient, self.host_h2.id)
             VirtualMachine.list(self.apiclient, id=vm.id, listall=True)[0]
             self.fail("VM should not be migrated")