Skip to content

Commit fdcace1

Browse files
Pearl1594Pearl1594
committed
add host HA support and retry transient lock refusals on activation
1 parent 68cff97 commit fdcace1

18 files changed

Lines changed: 1009 additions & 50 deletions

File tree

‎api/src/main/java/com/cloud/agent/api/to/HostTO.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ public class HostTO {
2525
private NetworkTO storageNetwork1;
2626
private NetworkTO storageNetwork2;
2727
private String parent;
28+
private Long sanlockHostId;
2829

2930
protected HostTO() {
3031
}
@@ -46,6 +47,14 @@ public HostTO(Host vo) {
4647
}
4748
}
4849

50+
public Long getSanlockHostId() {
51+
return sanlockHostId;
52+
}
53+
54+
public void setSanlockHostId(Long sanlockHostId) {
55+
this.sanlockHostId = sanlockHostId;
56+
}
57+
4958
public String getGuid() {
5059
return guid;
5160
}

‎api/src/main/java/com/cloud/host/Host.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ public static String[] toStrings(Host.Type... types) {
6060
String HOST_RBD_VOLUME_ENCRYPTION = "host.volume.encryption.rbd";
6161
String HOST_INSTANCE_CONVERSION = "host.instance.conversion";
6262
String HOST_VDDK_SUPPORT = "host.vddk.support";
63+
String HOST_SANLOCK_ID = "host.sanlock.id";
6364
String HOST_VDDK_LIB_DIR = "vddk.lib.dir";
6465
String HOST_VDDK_VERSION = "host.vddk.version";
6566
String HOST_OVFTOOL_VERSION = "host.ovftool.version";

‎engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
*/
3434
public interface HighAvailabilityManager extends Manager {
3535

36-
List<StoragePoolType> LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint);
36+
List<StoragePoolType> LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint, StoragePoolType.CLVM, StoragePoolType.CLVM_NG);
3737

3838
ConfigKey<Boolean> ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false",
3939
"Force High-Availability to happen even if the VM says no.", true, Cluster);

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -881,6 +881,7 @@ private Long getClvmLockHostFromVmVolumes(Long vmId) {
881881
return null;
882882
}
883883

884+
long recoveryDeadlineMs = clvmPoolManager.newLockRecoveryDeadlineMs();
884885
for (VolumeVO volume : vmVolumes) {
885886
if (volume.getPoolId() == null) {
886887
continue;
@@ -895,7 +896,8 @@ private Long getClvmLockHostFromVmVolumes(Long vmId) {
895896
volume.getUuid(),
896897
volume.getPath(),
897898
pool,
898-
true
899+
true,
900+
recoveryDeadlineMs
899901
);
900902
if (lockHostId != null) {
901903
logger.debug("Found actual CLVM lock host {} from volume {} of VM {} via LVM query",
@@ -912,6 +914,7 @@ private void transferClvmLocksForVmStart(List<VolumeVO> volumes, Long destHostId
912914
return;
913915
}
914916

917+
long recoveryDeadlineMs = clvmPoolManager.newLockRecoveryDeadlineMs();
915918
for (VolumeVO volume : volumes) {
916919
if (volume.getPoolId() == null) {
917920
continue;
@@ -927,7 +930,8 @@ private void transferClvmLocksForVmStart(List<VolumeVO> volumes, Long destHostId
927930
volume.getUuid(),
928931
volume.getPath(),
929932
pool,
930-
true
933+
true,
934+
recoveryDeadlineMs
931935
);
932936

933937
if (currentLockHost == null) {

‎engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -671,9 +671,9 @@ public void testTransferClvmLocksForVmStart_WithClvmVolumes() throws Exception {
671671

672672
ClvmPoolManager clvmPoolManager = Mockito.mock(ClvmPoolManager.class);
673673
Mockito.when(clvmPoolManager.getClvmLockHostId(Mockito.eq(101L), Mockito.anyString(),
674-
Mockito.anyString(), Mockito.any(), Mockito.eq(true))).thenReturn(currentHostId);
674+
Mockito.anyString(), Mockito.any(), Mockito.eq(true), Mockito.anyLong())).thenReturn(currentHostId);
675675
Mockito.when(clvmPoolManager.getClvmLockHostId(Mockito.eq(102L), Mockito.anyString(),
676-
Mockito.anyString(), Mockito.any(), Mockito.eq(true))).thenReturn(currentHostId);
676+
Mockito.anyString(), Mockito.any(), Mockito.eq(true), Mockito.anyLong())).thenReturn(currentHostId);
677677
Mockito.when(clvmPoolManager.transferClvmVolumeLock(Mockito.anyString(), Mockito.anyLong(),
678678
Mockito.anyString(), Mockito.any(), Mockito.anyLong(), Mockito.anyLong())).thenReturn(true);
679679

@@ -743,7 +743,7 @@ public void testTransferClvmLocksForVmStart_NoLockTransferNeeded() throws Except
743743

744744
ClvmPoolManager clvmPoolManager = Mockito.mock(ClvmPoolManager.class);
745745
Mockito.when(clvmPoolManager.getClvmLockHostId(Mockito.eq(101L), ArgumentMatchers.nullable(String.class),
746-
ArgumentMatchers.nullable(String.class), Mockito.any(), Mockito.eq(true))).thenReturn(destHostId);
746+
ArgumentMatchers.nullable(String.class), Mockito.any(), Mockito.eq(true), Mockito.anyLong())).thenReturn(destHostId);
747747

748748
Mockito.when(storagePoolDao.findById(poolId)).thenReturn(clvmPool);
749749

@@ -776,7 +776,7 @@ public void testTransferClvmLocksForVmStart_EmptyVolumeList() throws Exception {
776776
method.invoke(volumeOrchestrator, new ArrayList<VolumeVO>(), destHostId, vmInstance);
777777

778778
Mockito.verify(clvmPoolManager, Mockito.never()).getClvmLockHostId(Mockito.anyLong(), Mockito.anyString(),
779-
Mockito.anyString(), Mockito.any(), Mockito.anyBoolean());
779+
Mockito.anyString(), Mockito.any(), Mockito.anyBoolean(), Mockito.anyLong());
780780
}
781781

782782
@Test
@@ -817,7 +817,7 @@ public void testTransferClvmLocksForVmStart_SetInitialLockHost() throws Exceptio
817817

818818
ClvmPoolManager clvmPoolManager = Mockito.mock(ClvmPoolManager.class);
819819
Mockito.when(clvmPoolManager.getClvmLockHostId(Mockito.eq(101L), ArgumentMatchers.nullable(String.class),
820-
ArgumentMatchers.nullable(String.class), Mockito.any(), Mockito.eq(true))).thenReturn(null);
820+
ArgumentMatchers.nullable(String.class), Mockito.any(), Mockito.eq(true), Mockito.anyLong())).thenReturn(null);
821821

822822
Mockito.when(storagePoolDao.findById(poolId)).thenReturn(clvmPool);
823823

@@ -863,7 +863,7 @@ public void testTransferClvmLocksForVmStart_MixedVolumes() throws Exception {
863863

864864
ClvmPoolManager clvmPoolManager = Mockito.mock(ClvmPoolManager.class);
865865
Mockito.when(clvmPoolManager.getClvmLockHostId(Mockito.eq(101L), Mockito.anyString(),
866-
Mockito.anyString(), Mockito.any(), Mockito.eq(true))).thenReturn(currentHostId);
866+
Mockito.anyString(), Mockito.any(), Mockito.eq(true), Mockito.anyLong())).thenReturn(currentHostId);
867867
Mockito.when(clvmPoolManager.transferClvmVolumeLock(Mockito.anyString(), Mockito.anyLong(),
868868
Mockito.anyString(), Mockito.any(), Mockito.anyLong(), Mockito.anyLong())).thenReturn(true);
869869

@@ -904,7 +904,7 @@ public void testTransferClvmLocksForVmStart_TransferFails() throws Throwable {
904904

905905
ClvmPoolManager clvmPoolManager = Mockito.mock(ClvmPoolManager.class);
906906
Mockito.when(clvmPoolManager.getClvmLockHostId(Mockito.eq(101L), Mockito.anyString(),
907-
Mockito.anyString(), Mockito.any(), Mockito.eq(true))).thenReturn(currentHostId);
907+
Mockito.anyString(), Mockito.any(), Mockito.eq(true), Mockito.anyLong())).thenReturn(currentHostId);
908908
Mockito.when(clvmPoolManager.transferClvmVolumeLock(Mockito.anyString(), Mockito.anyLong(),
909909
Mockito.anyString(), Mockito.any(), Mockito.anyLong(), Mockito.anyLong())).thenReturn(false);
910910

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,7 @@ public Boolean hasHeartBeat() {
4444
storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", ")));
4545
logger.debug("Checking heart beat with KVMHAChecker for {}", hostAndPools);
4646

47-
boolean heartBeatCheckResult = false;
48-
for (HAStoragePool pool : storagePools) {
49-
heartBeatCheckResult = pool.getPool().hasHeartBeat(pool, host);
50-
if (reportIfHeartBeatFailedForOneStoragePool && !heartBeatCheckResult) {
51-
break;
52-
}
53-
}
47+
boolean heartBeatCheckResult = checkPoolsHeartBeat();
5448

5549
if (!heartBeatCheckResult) {
5650
logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools);
@@ -59,6 +53,22 @@ public Boolean hasHeartBeat() {
5953
return heartBeatCheckResult;
6054
}
6155

56+
/*
57+
* With reportIfHeartBeatFailedForOneStoragePool the host is dead as soon as one pool has no heart beat,
58+
* otherwise it is dead only when none of the pools has one.
59+
*/
60+
private boolean checkPoolsHeartBeat() {
61+
boolean anyPoolHasHeartBeat = false;
62+
for (HAStoragePool pool : storagePools) {
63+
boolean poolHasHeartBeat = pool.getPool().hasHeartBeat(pool, host);
64+
if (reportIfHeartBeatFailedForOneStoragePool && !poolHasHeartBeat) {
65+
return false;
66+
}
67+
anyPoolHasHeartBeat = anyPoolHasHeartBeat || poolHasHeartBeat;
68+
}
69+
return anyPoolHasHeartBeat;
70+
}
71+
6272
@Override
6373
public Boolean call() throws Exception {
6474
// logger.addAppender(new org.apache.log4j.ConsoleAppender(new

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
import com.cloud.agent.properties.AgentProperties;
2020
import com.cloud.agent.properties.AgentPropertiesFileHandler;
21-
import com.cloud.ha.HighAvailabilityManager;
21+
import com.cloud.storage.Storage.StoragePoolType;
2222
import com.cloud.utils.script.Script;
2323
import org.libvirt.Connect;
2424
import org.libvirt.LibvirtException;
@@ -34,6 +34,8 @@
3434

3535
public class KVMHAMonitor extends KVMHABase implements Runnable {
3636

37+
private static final List<StoragePoolType> MOUNT_BASED_POOL_TYPES = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint);
38+
3739
private final Map<String, HAStoragePool> haStoragePools = new ConcurrentHashMap<>();
3840
private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout;
3941

@@ -54,7 +56,9 @@ public void removeStoragePool(String uuid) {
5456
synchronized (haStoragePools) {
5557
HAStoragePool pool = haStoragePools.get(uuid);
5658
if (pool != null) {
57-
Script.runSimpleBashScript("umount " + pool.getMountDestPath());
59+
if (isMountBased(pool)) {
60+
Script.runSimpleBashScript("umount " + pool.getMountDestPath());
61+
}
5862
haStoragePools.remove(uuid);
5963
}
6064
}
@@ -72,12 +76,16 @@ public HAStoragePool getStoragePool(String uuid) {
7276
}
7377
}
7478

79+
private boolean isMountBased(HAStoragePool pool) {
80+
return MOUNT_BASED_POOL_TYPES.contains(pool.getPool().getType());
81+
}
82+
7583
protected void runHeartBeat() {
7684
synchronized (haStoragePools) {
7785
Set<String> removedPools = new HashSet<>();
7886
for (String uuid : haStoragePools.keySet()) {
7987
HAStoragePool primaryStoragePool = haStoragePools.get(uuid);
80-
if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(primaryStoragePool.getPool().getType())) {
88+
if (isMountBased(primaryStoragePool)) {
8189
checkForNotExistingLibvirtStoragePools(removedPools, uuid);
8290
if (removedPools.contains(uuid)) {
8391
continue;

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import static com.cloud.host.Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM;
2222
import static com.cloud.host.Host.HOST_OVFTOOL_VERSION;
2323
import static com.cloud.host.Host.HOST_RBD_VOLUME_ENCRYPTION;
24+
import static com.cloud.host.Host.HOST_SANLOCK_ID;
2425
import static com.cloud.host.Host.HOST_VDDK_LIB_DIR;
2526
import static com.cloud.host.Host.HOST_VDDK_SUPPORT;
2627
import static com.cloud.host.Host.HOST_VDDK_VERSION;
@@ -217,6 +218,7 @@
217218
import com.cloud.hypervisor.kvm.resource.rolling.maintenance.RollingMaintenanceServiceExecutor;
218219
import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtRequestWrapper;
219220
import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper;
221+
import com.cloud.hypervisor.kvm.storage.ClvmSanlockHaChecker;
220222
import com.cloud.hypervisor.kvm.storage.IscsiStorageCleanupMonitor;
221223
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
222224
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
@@ -4430,6 +4432,7 @@ public StartupCommand[] initialize() {
44304432
cmd.setCluster(clusterId);
44314433
cmd.setGatewayIpAddress(localGateway);
44324434
cmd.setIqn(getIqn());
4435+
addSanlockHostId(cmd);
44334436
cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption()));
44344437
cmd.getHostDetails().put(HOST_RBD_VOLUME_ENCRYPTION, String.valueOf(hostSupportsRbdVolumeEncryption()));
44354438
cmd.setHostTags(getHostTags());
@@ -6216,6 +6219,13 @@ public boolean isHostSecured() {
62166219
return true;
62176220
}
62186221

6222+
private void addSanlockHostId(StartupRoutingCommand cmd) {
6223+
Long sanlockHostId = ClvmSanlockHaChecker.readLocalHostId();
6224+
if (sanlockHostId != null) {
6225+
cmd.getHostDetails().put(HOST_SANLOCK_ID, String.valueOf(sanlockHostId));
6226+
}
6227+
}
6228+
62196229
/**
62206230
* Test host for qemu-native LUKS volume encryption (qemu-img LUKS support + cryptsetup),
62216231
* reported as {@code host.volume.encryption}. RBD/librbd encryption support is a separate

0 commit comments

Comments
 (0)