From dd2e26c28270599a4644f9e1139b825c42aac0b0 Mon Sep 17 00:00:00 2001 From: Brad House Date: Sat, 7 Nov 2026 10:47:03 +0000 Subject: [PATCH 1/3] Stop concurrent CKS operations clobbering each other, recover stuck states A Kubernetes cluster's state was written by id alone, so two operations started on the same cluster at once both got past their first transition. Each then ran its steps against VMs the other was changing, and whichever finished last decided the final state. The status scanner could also overwrite the state of an operation that started after it read the cluster. The state is now compared and set in one update, as the VM and volume states are. Each operation moves the cluster into its own state before it changes anything, and fails without touching the cluster when another operation got there first. From then on it only moves the cluster while the cluster is in a state the operation moved it to. Starting a cluster in Alert now moves it to Starting like any other start. A start from Error, a stop from Created or Error, and a retried destroy still go on without a state change, as they did before; the start and stop then leave the state alone when they end. From any other state with no transition for it, an operation is now refused instead of running alongside the one that holds the cluster. The scanner only completes a stop or a recovery that it started itself. A management server that stopped during an operation left the cluster in that operation's state. Nothing moved it on, and every later operation on it was refused until the database was edited by hand. The scanner now fails the operation of a cluster in an operation's state when no job is in progress for it on any management server. The existing Alert reconciliation then returns a healthy cluster to Running. To find those jobs, the start, stop, scale and upgrade jobs now record their cluster, as the create, delete and node jobs already did. A cluster named by a job in progress on a management server of an older version, which does not record it, is left alone. This replaces the scanner's first-run handling of Starting clusters, which tried a transition that does not exist. No schema or API change. Signed-off-by: Brad House (cherry picked from commit fee9834af471dc88ed4b3b86e6732a66e063bf6c) --- .../kubernetes/cluster/KubernetesCluster.java | 1 + .../cluster/KubernetesClusterManagerImpl.java | 98 ++++++++++++---- .../KubernetesClusterActionWorker.java | 85 +++++++++++++- .../KubernetesClusterAddWorker.java | 12 +- .../KubernetesClusterDestroyWorker.java | 5 +- .../KubernetesClusterRemoveWorker.java | 2 +- ...esClusterResourceModifierActionWorker.java | 39 ++++++- .../KubernetesClusterScaleWorker.java | 32 +++--- .../KubernetesClusterStartWorker.java | 13 ++- .../KubernetesClusterStopWorker.java | 3 +- .../KubernetesClusterUpgradeWorker.java | 2 +- .../cluster/dao/KubernetesClusterDao.java | 1 + .../cluster/dao/KubernetesClusterDaoImpl.java | 54 +++++++-- .../cluster/ScaleKubernetesClusterCmd.java | 5 + .../cluster/StartKubernetesClusterCmd.java | 5 + .../cluster/StopKubernetesClusterCmd.java | 5 + .../cluster/UpgradeKubernetesClusterCmd.java | 5 + .../KubernetesClusterManagerImplTest.java | 108 ++++++++++++++++++ .../KubernetesClusterActionWorkerTest.java | 107 +++++++++++++++++ ...usterResourceModifierActionWorkerTest.java | 56 +++++++++ .../dao/KubernetesClusterDaoImplTest.java | 78 +++++++++++++ 21 files changed, 653 insertions(+), 63 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImplTest.java diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 80f6a6045c72..c06f8709d710 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -88,6 +88,7 @@ enum State { s_fsm.addTransition(State.Stopping, Event.OperationFailed, State.Alert); s_fsm.addTransition(State.Stopped, Event.StartRequested, State.Starting); + s_fsm.addTransition(State.Alert, Event.StartRequested, State.Starting); s_fsm.addTransition(State.Stopped, Event.OperationSucceeded, State.Stopped); s_fsm.addTransition(State.Running, Event.OperationSucceeded, State.Running); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index eb4ebb78ee48..69d8ee6ce409 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -122,6 +121,8 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.commons.beanutils.BeanUtils; @@ -360,6 +361,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected VMInstanceDao vmInstanceDao; @Inject + protected AsyncJobDao asyncJobDao; + @Inject protected UserVmJoinDao userVmJoinDao; @Inject protected NetworkOfferingDao networkOfferingDao; @@ -2909,8 +2912,11 @@ public void reallyRun() { startWorker = ComponentContext.inject(startWorker); startWorker.reconcileAlertCluster(); } else if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + // move the cluster as listed, so that an operation started on it since then, which may + // also allow a stop, is left alone + if (_stateMachine.transitTo(kubernetesCluster, KubernetesCluster.Event.StopRequested, null, kubernetesClusterDao)) { + _stateMachine.transitTo(kubernetesCluster, KubernetesCluster.Event.OperationSucceeded, null, kubernetesClusterDao); + } } } catch (Exception e) { logger.warn("Failed to run Kubernetes cluster Alert state scanner on Kubernetes cluster: {} status scanner", kubernetesCluster, e); @@ -2918,26 +2924,15 @@ public void reallyRun() { } + // move clusters left in an operation's state by a management server that stopped to a state the + // reconciliation above can work from + try { + recoverOrphanedOperationStates(); + } catch (Exception e) { + logger.warn("Failed to recover Kubernetes clusters left in an operation's state", e); + } + if (firstRun) { - // run through Kubernetes clusters in 'Starting' state and reconcile state as 'Alert' or 'Error' if the VM's are running - List startingKubernetesClusters = kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.Starting); - for (KubernetesCluster kubernetesCluster : startingKubernetesClusters) { - if ((new Date()).getTime() - kubernetesCluster.getCreated().getTime() < 10*60*1000) { - continue; - } - if (logger.isInfoEnabled()) { - logger.info("Running Kubernetes cluster state scanner on Kubernetes cluster: {} for state: {}", kubernetesCluster, KubernetesCluster.State.Starting.toString()); - } - try { - if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); - } else { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } catch (Exception e) { - logger.warn("Failed to run Kubernetes cluster Starting state scanner on Kubernetes cluster: {} status scanner", kubernetesCluster, e); - } - } List destroyingKubernetesClusters = kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.Destroying); for (KubernetesCluster kubernetesCluster : destroyingKubernetesClusters) { if (logger.isInfoEnabled()) { @@ -2957,6 +2952,65 @@ public void reallyRun() { } } + /** + * States which a cluster holds only while an operation runs on it, and which nothing else moves it out of. + */ + protected static final List OPERATION_STATES = List.of(KubernetesCluster.State.Starting, + KubernetesCluster.State.Stopping, KubernetesCluster.State.Scaling, KubernetesCluster.State.ScalingStoppedCluster, + KubernetesCluster.State.Upgrading, KubernetesCluster.State.Importing, KubernetesCluster.State.RemovingNodes, + KubernetesCluster.State.Recovering); + + /** + * Fails the operation of each cluster in an operation's state which has no job in progress on any management + * server. Such a cluster was left there by a management server that stopped during the operation: the jobs of a + * stopped management server are failed when it starts again or when another one sees it leave, but nothing moved + * the cluster on, and every later operation on it was refused. + */ + protected void recoverOrphanedOperationStates() { + List clusters = kubernetesClusterDao.findManagedKubernetesClustersInStates(OPERATION_STATES); + if (clusters.isEmpty()) { + return; + } + // A management server of an older version does not record the cluster of its start, stop, scale and upgrade + // jobs, but their parameters name it by its UUID. A cluster that such a job may be acting on is left alone. + List untaggedJobsCmdInfo = new ArrayList<>(); + for (AsyncJobVO job : asyncJobDao.findInstancePendingAsyncJobs(ApiCommandResourceType.KubernetesCluster.toString(), null)) { + if (job.getInstanceId() == null) { + untaggedJobsCmdInfo.add(StringUtils.defaultString(job.getCmdInfo())); + } + } + for (KubernetesClusterVO cluster : clusters) { + if (untaggedJobsCmdInfo.stream().anyMatch(cmdInfo -> cmdInfo.isEmpty() || cmdInfo.contains(cluster.getUuid()))) { + logger.debug("Not recovering Kubernetes cluster {} from state {} while a job of an older management server may be acting on it", cluster, cluster.getState()); + continue; + } + try { + recoverOrphanedOperationState(cluster.getId()); + } catch (Exception e) { + logger.warn("Failed to recover Kubernetes cluster {} from state {}", cluster, cluster.getState(), e); + } + } + } + + /** + * The cluster is read before its jobs. An operation that ends between the two reads moves the cluster to another + * state, so the compare-and-set of the state as read misses. For it to hit a live operation instead, one operation + * would have to end and another start and reach the same state between the two reads and the update right after. + */ + protected void recoverOrphanedOperationState(long clusterId) throws NoTransitionException { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(clusterId); + if (cluster == null || !OPERATION_STATES.contains(cluster.getState())) { + return; + } + if (asyncJobDao.findInstancePendingAsyncJob(ApiCommandResourceType.KubernetesCluster.toString(), clusterId) != null) { + return; + } + final KubernetesCluster.State state = cluster.getState(); + if (_stateMachine.transitTo(cluster, KubernetesCluster.Event.OperationFailed, null, kubernetesClusterDao)) { + logger.warn("Kubernetes cluster {} was in state {} with no operation in progress, as its operation ended without moving it on, for example because the management server running it stopped; marked the operation as failed", cluster, state); + } + } + // checks if Kubernetes cluster is in desired state private boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 07c9f12a50e2..23b8ac7a2407 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -164,6 +164,10 @@ public class KubernetesClusterActionWorker { protected final static List CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER, CONTROL, ETCD); protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); + // whether this worker has started its operation with transitToOperationState, and the state the operation moved + // the cluster to and has not moved it out of yet + private boolean operationStarted = false; + private KubernetesCluster.State ownedState; @Inject protected CAManager caManager; @@ -770,10 +774,31 @@ protected void updateLoginUserDetails(List clusterVMs) { } } + /** + * Moves the cluster on with the given event. Once this worker has started its operation with + * {@link #transitToOperationState}, it only moves the cluster out of the state that the operation last moved it + * to: a cluster it never moved, or that has moved on since, may be held by another operation by now. + */ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + logger.warn("Unable to find Kubernetes cluster with ID {} to apply event {}", kubernetesClusterId, e); + return false; + } + if (operationStarted) { + if (ownedState == null || kubernetesCluster.getState() != ownedState) { + logger.debug("Not applying event {} to Kubernetes cluster {} in state {}, which its operation does not hold", e, kubernetesCluster, kubernetesCluster.getState()); + return false; + } + } try { - return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + boolean transitioned = _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + if (operationStarted) { + // the operation holds the state it moved the cluster to, until another operation moves it on; on a + // miss, another operation already has + ownedState = transitioned ? kubernetesCluster.getState() : null; + } + return transitioned; } catch (NoTransitionException nte) { logger.warn(String.format("Failed to transition state of the Kubernetes cluster : %s in state %s on event %s", kubernetesCluster.getName(), kubernetesCluster.getState().toString(), e.toString()), nte); @@ -781,6 +806,64 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve } } + /** + * Moves the cluster into the state of the operation that is starting, failing if another operation got there first. + * The state is compared and set in one update, so of two operations started on the cluster at once only one gets + * past this point. The other fails without changing the state, and without undoing anything that the operation in + * progress may still need. + *

+ * From then on the worker only moves the cluster out of that state, see {@link #stateTransitTo}. + * + * @param statesToGoOnWithout the states with no transition for the event in which the operation still goes on, + * without a state change, as it did before; in any other such state it fails. Going on + * this way, the operation holds no state, and does not move the cluster when it ends. + * @return whether the cluster was moved to the operation's state + */ + protected boolean transitToOperationState(final KubernetesCluster.Event event, final KubernetesCluster.State... statesToGoOnWithout) throws CloudRuntimeException { + operationStarted = true; + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster == null) { + throw new CloudRuntimeException(String.format("Unable to find Kubernetes cluster %s", kubernetesCluster.getName())); + } + final KubernetesCluster.State state = cluster.getState(); + boolean transitioned; + try { + transitioned = _stateMachine.transitTo(cluster, event, null, kubernetesClusterDao); + } catch (NoTransitionException e) { + if (Arrays.asList(statesToGoOnWithout).contains(state)) { + return false; + } + String message = String.format("Kubernetes cluster %s is in state %s, which does not allow %s", kubernetesCluster.getName(), state, event); + logger.warn(message); + throw new CloudRuntimeException(message); + } + if (!transitioned) { + KubernetesClusterVO current = kubernetesClusterDao.findById(cluster.getId()); + throw operationRaced(state, current == null ? null : current.getState(), event); + } + ownedState = cluster.getState(); + return true; + } + + /** + * Makes the operation hold a state it went on in without moving the cluster, so that it moves the cluster out of + * it when it ends: a retried destroy completes the Destroying state. + */ + protected void holdOperationState(final KubernetesCluster.State state) { + ownedState = state; + } + + protected boolean isOperationStarted() { + return operationStarted; + } + + protected CloudRuntimeException operationRaced(final KubernetesCluster.State from, final KubernetesCluster.State to, final KubernetesCluster.Event event) { + String message = String.format("Kubernetes cluster %s changed from state %s to %s before %s could start; another operation may be in progress, please retry once it has finished", + kubernetesCluster.getName(), from, to, event); + logger.warn(message); + return new CloudRuntimeException(message); + } + protected boolean createCloudStackSecret(String[] keys) { File pkFile = getManagementServerSshPublicKeyFile(); Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java index 8b694adf1cca..53012b6c93fc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java @@ -86,8 +86,9 @@ public boolean addNodesToCluster(List nodeIds, boolean mountCksIsoOnVr, bo } catch (ManagementServerException e) { throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName())); } + // take the cluster before attaching the ISO, so that an add which is refused leaves nothing behind + transitToOperationState(KubernetesCluster.Event.AddNodeRequested); attachCksIsoForNodesAdditionToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AddNodeRequested); String controlNodeGuestIp = getControlVmPrivateIp(); Ternary nodesAddedAndMemory = importNodeToCluster(nodeIds, network, publicIp, controlNodeGuestIp, mountCksIsoOnVr); int nodesAdded = nodesAddedAndMemory.first(); @@ -113,7 +114,14 @@ public boolean addNodesToCluster(List nodeIds, boolean mountCksIsoOnVr, bo description, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); return true; } catch (Exception e) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + // a failure before this operation took the cluster must not end the operation that holds it, and after, + // the cluster is only moved while this operation still holds it, see stateTransitTo + if (isOperationStarted()) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + if (e instanceof CloudRuntimeException) { + throw (CloudRuntimeException) e; + } throw new CloudRuntimeException(e); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index ac6a64e2fb0c..4380f35bba76 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -328,7 +328,10 @@ public boolean destroy() throws CloudRuntimeException { if (logger.isInfoEnabled()) { logger.info("Destroying Kubernetes cluster : {}", kubernetesCluster); } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.DestroyRequested); + // a cluster already being destroyed is having its destroy retried, which completes it + if (!transitToOperationState(KubernetesCluster.Event.DestroyRequested, KubernetesCluster.State.Destroying)) { + holdOperationState(KubernetesCluster.State.Destroying); + } boolean vmsDestroyed = destroyClusterVMs(); // if there are VM's that were not expunged, we can not delete the network if (vmsDestroyed) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java index 07d062e23a19..275911466ee3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java @@ -70,7 +70,7 @@ public boolean removeNodesFromCluster(List nodeIds) { } catch (ManagementServerException e) { throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName())); } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RemoveNodeRequested); + transitToOperationState(KubernetesCluster.Event.RemoveNodeRequested); boolean result = removeNodesFromCluster(nodeIds, network, publicIp); if (!result) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 72dd9816858d..c56c87d33f1b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -121,6 +121,9 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClusterActionWorker { + private final KubernetesCluster.State initialState; + private boolean scalingStateEntered = false; + @Inject protected CapacityManager capacityManager; @Inject @@ -166,6 +169,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected KubernetesClusterResourceModifierActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); + this.initialState = kubernetesCluster.getState(); } protected void init() { @@ -953,10 +957,34 @@ private KubernetesClusterVO updateKubernetesClusterEntry(final Boolean autoscale return kubernetesClusterVO; } - protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AutoscaleRequested); + /** + * Moves the cluster into its scaling state the first time a scaling step needs it, failing if another operation + * moved the cluster first. Later steps of the same operation find the cluster already in that state. + */ + protected void enterScalingState(KubernetesCluster.Event event) throws CloudRuntimeException { + if (scalingStateEntered) { + return; + } + // a cluster which was never started has no scaling states: only its stored settings change. Created has no + // transition to compare and set, so this only checks that no start has taken the cluster since the request + // was accepted, which leaves a narrow window open. + if (KubernetesCluster.State.Created.equals(initialState)) { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster == null || !KubernetesCluster.State.Created.equals(cluster.getState())) { + throw operationRaced(initialState, cluster == null ? null : cluster.getState(), event); + } + return; } + transitToOperationState(event); + scalingStateEntered = true; + } + + protected boolean isScalingStateEntered() { + return scalingStateEntered; + } + + protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { + enterScalingState(KubernetesCluster.Event.AutoscaleRequested); File pkFile = getManagementServerSshPublicKeyFile(); Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); @@ -1001,10 +1029,13 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { return true; } catch (Exception e) { String msg = String.format("Failed to autoscale Kubernetes cluster: %s : %s", kubernetesCluster.getName(), e.getMessage()); + // fail the scaling state this operation entered; a cluster never started holds none + if (isScalingStateEntered()) { + logTransitStateAndThrow(Level.ERROR, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } logAndThrow(Level.ERROR, msg); return false; } finally { - // Deploying the autoscaler might fail but it can be deployed manually too, so no need to go to an alert state updateLoginUserDetails(null); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 003a852f3dbe..535069d497da 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -113,8 +113,8 @@ protected void init() { } private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message, final Exception e) throws CloudRuntimeException { - KubernetesCluster cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); - if (cluster != null && KubernetesCluster.State.Scaling.equals(cluster.getState())) { + // only fail the scaling this worker started: before it moved the cluster, a scaling state belongs to another one + if (isScalingStateEntered()) { logTransitStateAndThrow(logLevel, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } else { logAndThrow(logLevel, message, e); @@ -364,10 +364,7 @@ private void validateKubernetesClusterScaleSizeParameters(KubernetesClusterNodeT private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, ServiceOffering serviceOffering, boolean updateNodeOffering, boolean updateClusterOffering) throws CloudRuntimeException { validateKubernetesClusterScaleOfferingParameters(); - List scalingStates = List.of(KubernetesCluster.State.Scaling, KubernetesCluster.State.ScalingStoppedCluster); - if (!scalingStates.contains(kubernetesCluster.getState())) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); - } + enterScalingState(KubernetesCluster.Event.ScaleUpRequested); if (KubernetesCluster.State.Created.equals(originalState)) { kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); return; @@ -440,9 +437,7 @@ private void removeNodesFromCluster(List vmMaps) throw } private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); - } + enterScalingState(KubernetesCluster.Event.ScaleDownRequested); List vmList; if (this.nodeIds != null) { vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds).stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList()); @@ -498,9 +493,7 @@ private void cleanupNewlyCreatedVms(Set originalVmIds) { } private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRuntimeException { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); - } + enterScalingState(KubernetesCluster.Event.ScaleUpRequested); Set originalVmIds = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()) .stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toSet()); List clusterVMs = new ArrayList<>(); @@ -542,9 +535,7 @@ private void scaleKubernetesClusterSize(KubernetesClusterNodeType nodeType) thro final long originalClusterSize = kubernetesCluster.getNodeCount(); final long newVmRequiredCount = clusterSize - originalClusterSize; if (KubernetesCluster.State.Created.equals(originalState)) { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), newVmRequiredCount > 0 ? KubernetesCluster.Event.ScaleUpRequested : KubernetesCluster.Event.ScaleDownRequested); - } + enterScalingState(newVmRequiredCount > 0 ? KubernetesCluster.Event.ScaleUpRequested : KubernetesCluster.Event.ScaleDownRequested); kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOfferingNodeTypeMap.get(nodeType.name()), false, false); return; } @@ -620,7 +611,7 @@ public boolean scaleCluster() throws CloudRuntimeException { if (autoScaled && isNodeOfferingScalingNeeded) { scaleKubernetesClusterOffering(nodeType, scalingServiceOffering, updateNodeOffering, updateClusterOffering); } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + completeScaling(); return autoScaled; } final boolean clusterSizeScalingNeeded = isWorkerNode && clusterSize != null && clusterSize != originalClusterSize; @@ -639,10 +630,17 @@ public boolean scaleCluster() throws CloudRuntimeException { } } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + completeScaling(); return true; } + private void completeScaling() { + // a cluster this worker did not move, such as one never started, may now be held by another operation + if (isScalingStateEntered()) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } + } + private ServiceOffering getExistingServiceOfferingForNodeType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) { Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster); if (existingOfferingId == null) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index a15086c60d6c..3c1508c11766 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -774,7 +774,7 @@ public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId, Lon logger.info("Starting Kubernetes cluster: {}", kubernetesCluster); } final long startTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterStartTimeout.value() * 1000; - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); + transitToOperationState(KubernetesCluster.Event.StartRequested); DeployDestination dest = null; try { VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId()); @@ -907,7 +907,9 @@ public boolean startStoppedKubernetesCluster(Long domainId, Long accountId) thro logger.info("Starting Kubernetes cluster: {}", kubernetesCluster); } final long startTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterStartTimeout.value() * 1000; - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); + // a cluster in Error, whose create failed, is started without taking a state, so that it stays in Error if the + // start fails too, rather than going to Alert where it could be marked Running without being set up + transitToOperationState(KubernetesCluster.Event.StartRequested, KubernetesCluster.State.Error); startKubernetesClusterVMs(domainId, accountId); try { InetAddress address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); @@ -978,8 +980,11 @@ public boolean reconcileAlertCluster() { if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { return false; } - // mark the cluster to be running - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); + // mark the cluster to be running, unless an operation started on it while the checks above ran: the + // operation now owns the cluster's state, and completing it here would end that state early + if (!stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested)) { + return false; + } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java index 59d74751dff9..4a1643d5db4f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java @@ -40,7 +40,8 @@ public boolean stop() throws CloudRuntimeException { if (logger.isInfoEnabled()) { logger.info("Stopping Kubernetes cluster: {}", kubernetesCluster); } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); + // a cluster that was never started or failed to start may still have VMs to stop; it is left in its state + transitToOperationState(KubernetesCluster.Event.StopRequested, KubernetesCluster.State.Created, KubernetesCluster.State.Error); List clusterVMs = getKubernetesClusterVMs(); for (UserVm vm : clusterVMs) { if (vm == null) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 4c2725fc2a2f..aa0f967b7852 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -174,7 +174,7 @@ public boolean upgradeCluster() throws CloudRuntimeException { } filterOutManualUpgradeNodesFromClusterUpgrade(); retrieveScriptFiles(); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested); + transitToOperationState(KubernetesCluster.Event.UpgradeRequested); attachIsoKubernetesVMs(clusterVMs, upgradeVersion); upgradeKubernetesClusterNodes(); detachIsoKubernetesVMs(clusterVMs); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java index 7df6a6b1dce7..dd23f0f451fc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java @@ -29,6 +29,7 @@ public interface KubernetesClusterDao extends GenericDao findKubernetesClustersToGarbageCollect(); List findManagedKubernetesClustersInState(KubernetesCluster.State state); + List findManagedKubernetesClustersInStates(List states); List listByNetworkId(long networkId); List listAllByKubernetesVersion(long kubernetesVersionId); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java index 7bec98d5d25b..55304e2ec080 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java @@ -25,7 +25,7 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.db.UpdateBuilder; @Component public class KubernetesClusterDaoImpl extends GenericDaoBase implements KubernetesClusterDao { @@ -36,6 +36,8 @@ public class KubernetesClusterDaoImpl extends GenericDaoBase ManagedStateSearch; private final SearchBuilder SameNetworkSearch; private final SearchBuilder KubernetesVersionSearch; + private final SearchBuilder StateChangeSearch; + private final SearchBuilder ManagedStatesSearch; public KubernetesClusterDaoImpl() { CleanupAccountIdSearch = createSearchBuilder(); @@ -61,6 +63,11 @@ public KubernetesClusterDaoImpl() { ManagedStateSearch.and("cluster_type", ManagedStateSearch.entity().getClusterType(), SearchCriteria.Op.EQ); ManagedStateSearch.done(); + ManagedStatesSearch = createSearchBuilder(); + ManagedStatesSearch.and("state", ManagedStatesSearch.entity().getState(), SearchCriteria.Op.IN); + ManagedStatesSearch.and("cluster_type", ManagedStatesSearch.entity().getClusterType(), SearchCriteria.Op.EQ); + ManagedStatesSearch.done(); + SameNetworkSearch = createSearchBuilder(); SameNetworkSearch.and("network_id", SameNetworkSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); SameNetworkSearch.done(); @@ -68,6 +75,11 @@ public KubernetesClusterDaoImpl() { KubernetesVersionSearch = createSearchBuilder(); KubernetesVersionSearch.and("kubernetesVersionId", KubernetesVersionSearch.entity().getKubernetesVersionId(), SearchCriteria.Op.EQ); KubernetesVersionSearch.done(); + + StateChangeSearch = createSearchBuilder(); + StateChangeSearch.and("id", StateChangeSearch.entity().getId(), SearchCriteria.Op.EQ); + StateChangeSearch.and("state", StateChangeSearch.entity().getState(), SearchCriteria.Op.EQ); + StateChangeSearch.done(); } @Override @@ -104,18 +116,42 @@ public List findManagedKubernetesClustersInState(Kubernetes return listBy(sc); } + @Override + public List findManagedKubernetesClustersInStates(List states) { + SearchCriteria sc = ManagedStatesSearch.create(); + sc.setParameters("state", states.toArray()); + sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged); + return listBy(sc); + } + + /** + * Moves the cluster to the next state only if it is still in the given current state, so that of two operations + * started on the same cluster at once only one gets past its first transition. + */ @Override public boolean updateState(KubernetesCluster.State currentState, KubernetesCluster.Event event, KubernetesCluster.State nextState, KubernetesCluster vo, Object data) { - // TODO: ensure this update is correct - TransactionLegacy txn = TransactionLegacy.currentTxn(); - txn.start(); - KubernetesClusterVO ccVo = (KubernetesClusterVO)vo; - ccVo.setState(nextState); - super.update(ccVo.getId(), ccVo); - - txn.commit(); + SearchCriteria sc = StateChangeSearch.create(); + sc.setParameters("id", ccVo.getId()); + sc.setParameters("state", currentState); + + UpdateBuilder builder = getUpdateBuilder(ccVo); + builder.set(ccVo, "state", nextState); + int rows = update(builder, sc, null); + if (rows == 0) { + // the builder set the next state on the cluster object too: put back the state the caller read, which + // the database no longer has either, without leaving it as a change for a later update to write + builder.set(ccVo, "state", currentState); + builder.clear(); + if (!logger.isDebugEnabled()) { + return false; + } + KubernetesClusterVO dbCluster = findByIdIncludingRemoved(ccVo.getId()); + logger.debug("Unable to move Kubernetes cluster {} from state {} to {} on event {}, its state is now {}", ccVo, currentState, nextState, event, + dbCluster == null ? null : dbCluster.getState()); + return false; + } return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index 1cff2649428d..5d345ef47f76 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -164,6 +164,11 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public Long getApiResourceId() { + return getId(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java index dae367548190..8ab6e3b450de 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java @@ -94,6 +94,11 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public Long getApiResourceId() { + return getId(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java index 5abeb27f59b1..c14f8d7fefbe 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java @@ -92,6 +92,11 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public Long getApiResourceId() { + return getId(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java index fb576e9ef825..620b98621cea 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java @@ -95,6 +95,11 @@ public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.KubernetesCluster; } + @Override + public Long getApiResourceId() { + return getId(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 1fab5420c3c3..cd33970ec483 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -51,12 +51,15 @@ import com.cloud.host.dao.HostDao; import org.apache.cloudstack.affinity.AffinityGroupVO; import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.commons.collections.MapUtils; import org.junit.After; import org.junit.Assert; @@ -118,6 +121,9 @@ public class KubernetesClusterManagerImplTest { @Mock private HostDao hostDao; + @Mock + private AsyncJobDao asyncJobDao; + @Spy @InjectMocks KubernetesClusterManagerImpl kubernetesClusterManager; @@ -914,4 +920,106 @@ public void testValidateNodeAffinityGroupsAffinityNoExistingWorkers() { Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()); } + private static final String CLUSTER_TYPE = ApiCommandResourceType.KubernetesCluster.toString(); + + private KubernetesClusterVO mockClusterInState(long id, KubernetesCluster.State state) { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.lenient().when(cluster.getId()).thenReturn(id); + Mockito.lenient().when(cluster.getState()).thenReturn(state); + Mockito.lenient().when(cluster.getUuid()).thenReturn("uuid-" + id); + Mockito.lenient().when(kubernetesClusterDao.findById(id)).thenReturn(cluster); + return cluster; + } + + private AsyncJobVO mockPendingJob(Long instanceId) { + AsyncJobVO job = Mockito.mock(AsyncJobVO.class); + Mockito.when(job.getInstanceId()).thenReturn(instanceId); + return job; + } + + @Test + public void recoverOrphanedOperationStatesFailsOnlyOperationsWithoutJob() { + KubernetesClusterVO orphaned = mockClusterInState(1L, KubernetesCluster.State.Upgrading); + KubernetesClusterVO busy = mockClusterInState(2L, KubernetesCluster.State.Scaling); + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInStates(KubernetesClusterManagerImpl.OPERATION_STATES)).thenReturn(List.of(orphaned, busy)); + AsyncJobVO job = mockPendingJob(2L); + Mockito.when(asyncJobDao.findInstancePendingAsyncJobs(CLUSTER_TYPE, null)).thenReturn(List.of(job)); + Mockito.when(asyncJobDao.findInstancePendingAsyncJob(CLUSTER_TYPE, 2L)).thenReturn(job); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Upgrading, KubernetesCluster.Event.OperationFailed, + KubernetesCluster.State.Alert, orphaned, null)).thenReturn(true); + + kubernetesClusterManager.recoverOrphanedOperationStates(); + + Mockito.verify(kubernetesClusterDao, Mockito.times(1)).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void recoverOrphanedOperationStatesLeavesClusterNamedByJobOfOlderManagementServer() { + KubernetesClusterVO named = mockClusterInState(1L, KubernetesCluster.State.Starting); + KubernetesClusterVO orphaned = mockClusterInState(2L, KubernetesCluster.State.Upgrading); + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInStates(KubernetesClusterManagerImpl.OPERATION_STATES)).thenReturn(List.of(named, orphaned)); + AsyncJobVO job = mockPendingJob(null); + Mockito.when(job.getCmdInfo()).thenReturn("{\"id\":\"uuid-1\"}"); + Mockito.when(asyncJobDao.findInstancePendingAsyncJobs(CLUSTER_TYPE, null)).thenReturn(List.of(job)); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Upgrading, KubernetesCluster.Event.OperationFailed, + KubernetesCluster.State.Alert, orphaned, null)).thenReturn(true); + + kubernetesClusterManager.recoverOrphanedOperationStates(); + + Mockito.verify(kubernetesClusterDao, Mockito.times(1)).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.verify(kubernetesClusterDao).updateState(KubernetesCluster.State.Upgrading, KubernetesCluster.Event.OperationFailed, + KubernetesCluster.State.Alert, orphaned, null); + } + + @Test + public void recoverOrphanedOperationStatesLeavesEveryClusterForJobOfOlderManagementServerWithoutParameters() { + KubernetesClusterVO cluster = mockClusterInState(1L, KubernetesCluster.State.Starting); + Mockito.when(kubernetesClusterDao.findManagedKubernetesClustersInStates(KubernetesClusterManagerImpl.OPERATION_STATES)).thenReturn(List.of(cluster)); + AsyncJobVO job = mockPendingJob(null); + Mockito.when(asyncJobDao.findInstancePendingAsyncJobs(CLUSTER_TYPE, null)).thenReturn(List.of(job)); + + kubernetesClusterManager.recoverOrphanedOperationStates(); + + Mockito.verify(kubernetesClusterDao, Mockito.never()).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void recoverOrphanedOperationStatesSkipsJobLookupWithoutClusters() { + kubernetesClusterManager.recoverOrphanedOperationStates(); + + Mockito.verifyNoInteractions(asyncJobDao); + } + + @Test + public void recoverOrphanedOperationStateLeavesClusterWhoseOperationEnded() throws Exception { + mockClusterInState(1L, KubernetesCluster.State.Running); + + kubernetesClusterManager.recoverOrphanedOperationState(1L); + + Mockito.verifyNoInteractions(asyncJobDao); + Mockito.verify(kubernetesClusterDao, Mockito.never()).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void recoverOrphanedOperationStateLeavesClusterWithJobStartedSinceTheScan() throws Exception { + mockClusterInState(1L, KubernetesCluster.State.Scaling); + Mockito.when(asyncJobDao.findInstancePendingAsyncJob(CLUSTER_TYPE, 1L)).thenReturn(Mockito.mock(AsyncJobVO.class)); + + kubernetesClusterManager.recoverOrphanedOperationState(1L); + + Mockito.verify(kubernetesClusterDao, Mockito.never()).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void recoverOrphanedOperationStateFailsStart() throws Exception { + KubernetesClusterVO cluster = mockClusterInState(1L, KubernetesCluster.State.Starting); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Starting, KubernetesCluster.Event.OperationFailed, + KubernetesCluster.State.Alert, cluster, null)).thenReturn(true); + + kubernetesClusterManager.recoverOrphanedOperationState(1L); + + Mockito.verify(kubernetesClusterDao).updateState(KubernetesCluster.State.Starting, KubernetesCluster.Event.OperationFailed, + KubernetesCluster.State.Alert, cluster, null); + } + } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java index a25ec55cc04a..1f0451942e31 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java @@ -38,6 +38,7 @@ import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; @@ -46,6 +47,7 @@ import com.cloud.network.Network; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; +import com.cloud.utils.exception.CloudRuntimeException; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterActionWorkerTest { @@ -229,4 +231,109 @@ public void testGetMergedAffinityGroupIdsExplicitDedicationAlreadyInList() { Assert.assertTrue(result.contains(99L)); Assert.assertTrue(result.contains(2L)); } + + private KubernetesClusterVO mockClusterInState(KubernetesCluster.State state) { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getState()).thenReturn(state); + Mockito.when(kubernetesClusterDao.findById(DEFAULT_ID)).thenReturn(cluster); + return cluster; + } + + @Test + public void testTransitToOperationStateMovesCluster() { + KubernetesClusterVO cluster = mockClusterInState(KubernetesCluster.State.Running); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Running, KubernetesCluster.Event.UpgradeRequested, + KubernetesCluster.State.Upgrading, cluster, null)).thenReturn(true); + Assert.assertTrue(actionWorker.transitToOperationState(KubernetesCluster.Event.UpgradeRequested)); + } + + @Test(expected = CloudRuntimeException.class) + public void testTransitToOperationStateFailsWhenAnotherOperationMovedClusterFirst() { + KubernetesClusterVO cluster = mockClusterInState(KubernetesCluster.State.Running); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Running, KubernetesCluster.Event.StopRequested, + KubernetesCluster.State.Stopping, cluster, null)).thenReturn(false); + // the compare-and-set misses even where the operation could go on without a transition: the other operation owns the state now + actionWorker.transitToOperationState(KubernetesCluster.Event.StopRequested, KubernetesCluster.State.Running); + } + + @Test(expected = CloudRuntimeException.class) + public void testTransitToOperationStateFailsWithoutTransitionInOtherStates() { + mockClusterInState(KubernetesCluster.State.Scaling); + actionWorker.transitToOperationState(KubernetesCluster.Event.DestroyRequested, KubernetesCluster.State.Created, KubernetesCluster.State.Destroying); + } + + @Test + public void testTransitToOperationStateGoesOnWithoutTransitionInGivenStates() { + mockClusterInState(KubernetesCluster.State.Destroying); + Assert.assertFalse(actionWorker.transitToOperationState(KubernetesCluster.Event.DestroyRequested, KubernetesCluster.State.Created, KubernetesCluster.State.Destroying)); + Mockito.verify(kubernetesClusterDao, Mockito.never()).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testStateTransitToFollowsTheStatesTheOperationMovesTheClusterTo() { + KubernetesClusterVO cluster = mockClusterInState(KubernetesCluster.State.Running); + Mockito.when(kubernetesClusterDao.updateState(Mockito.eq(KubernetesCluster.State.Running), Mockito.eq(KubernetesCluster.Event.UpgradeRequested), + Mockito.eq(KubernetesCluster.State.Upgrading), Mockito.eq(cluster), Mockito.isNull())).thenAnswer(invocation -> { + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Upgrading); + return true; + }); + Mockito.when(kubernetesClusterDao.updateState(Mockito.eq(KubernetesCluster.State.Upgrading), Mockito.eq(KubernetesCluster.Event.OperationFailed), + Mockito.eq(KubernetesCluster.State.Alert), Mockito.eq(cluster), Mockito.isNull())).thenAnswer(invocation -> { + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Alert); + return true; + }); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Alert, KubernetesCluster.Event.OperationSucceeded, + KubernetesCluster.State.Running, cluster, null)).thenReturn(true); + + actionWorker.transitToOperationState(KubernetesCluster.Event.UpgradeRequested); + Assert.assertTrue(actionWorker.stateTransitTo(DEFAULT_ID, KubernetesCluster.Event.OperationFailed)); + // a step that failed and was retried successfully still completes the operation + Assert.assertTrue(actionWorker.stateTransitTo(DEFAULT_ID, KubernetesCluster.Event.OperationSucceeded)); + } + + @Test + public void testStateTransitToLeavesClusterAnotherOperationMovedOn() { + KubernetesClusterVO cluster = mockClusterInState(KubernetesCluster.State.Running); + Mockito.when(kubernetesClusterDao.updateState(Mockito.eq(KubernetesCluster.State.Running), Mockito.eq(KubernetesCluster.Event.UpgradeRequested), + Mockito.eq(KubernetesCluster.State.Upgrading), Mockito.eq(cluster), Mockito.isNull())).thenAnswer(invocation -> { + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Upgrading); + return true; + }); + actionWorker.transitToOperationState(KubernetesCluster.Event.UpgradeRequested); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Scaling); + + Assert.assertFalse(actionWorker.stateTransitTo(DEFAULT_ID, KubernetesCluster.Event.OperationFailed)); + } + + @Test + public void testStateTransitToLeavesClusterTheOperationWentOnWithoutMoving() { + mockClusterInState(KubernetesCluster.State.Error); + Assert.assertFalse(actionWorker.transitToOperationState(KubernetesCluster.Event.StopRequested, KubernetesCluster.State.Error)); + Assert.assertFalse(actionWorker.stateTransitTo(DEFAULT_ID, KubernetesCluster.Event.OperationSucceeded)); + Mockito.verify(kubernetesClusterDao, Mockito.never()).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testStartTakesClusterInAlert() { + KubernetesClusterVO cluster = mockClusterInState(KubernetesCluster.State.Alert); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Alert, KubernetesCluster.Event.StartRequested, + KubernetesCluster.State.Starting, cluster, null)).thenReturn(true); + Assert.assertTrue(actionWorker.transitToOperationState(KubernetesCluster.Event.StartRequested)); + } + + @Test + public void testStateTransitToKeepsTheStateAfterAnEventWithoutTransition() { + KubernetesClusterVO cluster = mockClusterInState(KubernetesCluster.State.Running); + Mockito.when(kubernetesClusterDao.updateState(Mockito.eq(KubernetesCluster.State.Running), Mockito.eq(KubernetesCluster.Event.ScaleUpRequested), + Mockito.eq(KubernetesCluster.State.Scaling), Mockito.eq(cluster), Mockito.isNull())).thenAnswer(invocation -> { + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Scaling); + return true; + }); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Scaling, KubernetesCluster.Event.OperationFailed, + KubernetesCluster.State.Running, cluster, null)).thenReturn(true); + + actionWorker.transitToOperationState(KubernetesCluster.Event.ScaleUpRequested); + Assert.assertFalse(actionWorker.stateTransitTo(DEFAULT_ID, KubernetesCluster.Event.CreateFailed)); + Assert.assertTrue(actionWorker.stateTransitTo(DEFAULT_ID, KubernetesCluster.Event.OperationFailed)); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java index c220a3468afb..2af43f1ad8ad 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorkerTest.java @@ -19,10 +19,12 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.utils.exception.CloudRuntimeException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -135,4 +137,58 @@ public void getKubernetesClusterNodeNamePrefixTestNormalizedPrefixShouldNotStart Mockito.when(kubernetesClusterMock.getName()).thenReturn(originalPrefix); Assert.assertEquals(expectedPrefix, kubernetesClusterResourceModifierActionWorker.getKubernetesClusterNodeNamePrefix()); } + + @Test + public void enterScalingStateTransitionsOnlyOnceForAllStepsOfAnOperation() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Running); + Mockito.when(kubernetesClusterMock.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterMock.getState()).thenReturn(KubernetesCluster.State.Running); + kubernetesClusterResourceModifierActionWorker = new KubernetesClusterResourceModifierActionWorker(kubernetesClusterMock, kubernetesClusterManagerMock); + Mockito.when(kubernetesClusterDaoMock.findById(1L)).thenReturn(cluster); + Mockito.when(kubernetesClusterDaoMock.updateState(KubernetesCluster.State.Running, KubernetesCluster.Event.ScaleUpRequested, + KubernetesCluster.State.Scaling, cluster, null)).thenReturn(true); + + kubernetesClusterResourceModifierActionWorker.enterScalingState(KubernetesCluster.Event.ScaleUpRequested); + kubernetesClusterResourceModifierActionWorker.enterScalingState(KubernetesCluster.Event.ScaleDownRequested); + + Mockito.verify(kubernetesClusterDaoMock, Mockito.times(1)).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test(expected = CloudRuntimeException.class) + public void enterScalingStateFailsWhenAnotherOperationMovedTheClusterFirst() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getState()).thenReturn(KubernetesCluster.State.Upgrading); + Mockito.when(kubernetesClusterMock.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterMock.getState()).thenReturn(KubernetesCluster.State.Running); + kubernetesClusterResourceModifierActionWorker = new KubernetesClusterResourceModifierActionWorker(kubernetesClusterMock, kubernetesClusterManagerMock); + Mockito.when(kubernetesClusterDaoMock.findById(1L)).thenReturn(cluster); + + kubernetesClusterResourceModifierActionWorker.enterScalingState(KubernetesCluster.Event.ScaleUpRequested); + } + + private void mockNeverStartedClusterNowIn(KubernetesCluster.State state) { + Mockito.when(kubernetesClusterMock.getId()).thenReturn(1L); + Mockito.when(kubernetesClusterMock.getState()).thenReturn(KubernetesCluster.State.Created); + kubernetesClusterResourceModifierActionWorker = new KubernetesClusterResourceModifierActionWorker(kubernetesClusterMock, kubernetesClusterManagerMock); + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getState()).thenReturn(state); + Mockito.when(kubernetesClusterDaoMock.findById(1L)).thenReturn(cluster); + } + + @Test + public void enterScalingStateLeavesNeverStartedClusterInItsState() { + mockNeverStartedClusterNowIn(KubernetesCluster.State.Created); + + kubernetesClusterResourceModifierActionWorker.enterScalingState(KubernetesCluster.Event.ScaleUpRequested); + + Mockito.verify(kubernetesClusterDaoMock, Mockito.never()).updateState(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test(expected = CloudRuntimeException.class) + public void enterScalingStateFailsForNeverStartedClusterTakenByAStart() { + mockNeverStartedClusterNowIn(KubernetesCluster.State.Starting); + + kubernetesClusterResourceModifierActionWorker.enterScalingState(KubernetesCluster.Event.ScaleUpRequested); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImplTest.java new file mode 100644 index 000000000000..6f87cb635e33 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImplTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.dao; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.UpdateBuilder; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterDaoImplTest { + + @Spy + KubernetesClusterDaoImpl kubernetesClusterDao; + + private boolean updateStateWithRowsUpdated(int rows, KubernetesClusterVO cluster, UpdateBuilder builder) { + Mockito.doReturn(rows).when(kubernetesClusterDao).update(Mockito.eq(builder), Mockito.any(SearchCriteria.class), Mockito.isNull()); + try (MockedStatic dao = Mockito.mockStatic(GenericDaoBase.class)) { + dao.when(() -> GenericDaoBase.getUpdateBuilder(cluster)).thenReturn(builder); + return kubernetesClusterDao.updateState(KubernetesCluster.State.Running, KubernetesCluster.Event.ScaleUpRequested, + KubernetesCluster.State.Scaling, cluster, null); + } + } + + @Test + public void updateStateSetsStateOnlyWhereClusterIsStillInCurrentState() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getId()).thenReturn(1L); + UpdateBuilder builder = Mockito.mock(UpdateBuilder.class); + + Assert.assertTrue(updateStateWithRowsUpdated(1, cluster, builder)); + + Mockito.verify(builder).set(cluster, "state", KubernetesCluster.State.Scaling); + Mockito.verify(kubernetesClusterDao).update(Mockito.eq(builder), Mockito.argThat(sc -> { + String where = sc.getWhereClause(); + return where.contains("id") && where.contains("state"); + }), Mockito.isNull()); + } + + @Test + public void updateStateFailsWhenClusterLeftCurrentState() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.lenient().doReturn(null).when(kubernetesClusterDao).findByIdIncludingRemoved(1L); + + UpdateBuilder builder = Mockito.mock(UpdateBuilder.class); + Assert.assertFalse(updateStateWithRowsUpdated(0, cluster, builder)); + + // the cluster object gets back the state it still has, with no change left for a later update + org.mockito.InOrder inOrder = Mockito.inOrder(builder); + inOrder.verify(builder).set(cluster, "state", KubernetesCluster.State.Scaling); + inOrder.verify(builder).set(cluster, "state", KubernetesCluster.State.Running); + inOrder.verify(builder).clear(); + } +} From 63f98244271687e2eff2c444f00e70efd8bf0cfa Mon Sep 17 00:00:00 2001 From: Brad House Date: Sat, 7 Nov 2026 10:44:41 +0000 Subject: [PATCH 2/3] CKS: fix node drain on scale-down and upgrade, make a failed upgrade resumable Scale-down and upgrade both drain a node before touching it, each with its own copy of the drain loop, and neither works well: - Scale-down gives up at once when kubectl drain exits non-zero (a PodDisruptionBudget that does not allow an eviction yet, say), and when it does retry after an exception it counts the attempt twice. The node is left cordoned, and the cluster goes to Alert although nothing has been removed and every node still works. - Upgrade retries straight away, with no wait for the eviction to become possible, and on the last failure leaves the node cordoned. - In both, kubectl drain has no --timeout, so it is cut off by a fixed 60s SSH timeout rather than a deadline of its own. Both workers now use one drain helper in KubernetesClusterActionWorker: one count per attempt, a retry (after a wait) on a non-zero exit or an exception, kubectl drain --timeout from the new setting cloud.kubernetes.cluster.node.drain.timeout (300s), cut down so that no attempt runs past the operation's deadline, with the SSH wait derived from it, and the end of the drain output in the error. The caller uncordons the node, and only if it cordoned it. Scale-down cordons every node it is going to remove (leaving alone the ones that were cordoned already), drains them, and if one cannot be drained uncordons them all and puts the cluster back in the state it was in, with an error that names the node, instead of Alert; the same happens if the scale times out before every node is drained. Once they are drained, the removal is not stopped part way by the timeout any more. If removing a node fails, the nodes still in the cluster are uncordoned. The upgrade had two more problems: - It only waited for the first control node to be Ready, so the next node could be drained while the previous one was still coming back. It now waits for every node, after it has been uncordoned. - It reached node i on sshPort + i. The port-forwarding rules are not created in the order of the upgrade's VM list (manually upgraded nodes are left out of that list, and nodes added later get the next free port), so the script could run on the wrong node. It now looks up the node's own SSH port-forwarding rule, as the remove worker already does. A failed upgrade left the cluster in Alert with no way to finish it, as upgradeKubernetesCluster only accepts a Running cluster. An upgrade now records the version it is upgrading to as a cluster detail (an existing key/value table) and removes it once the cluster is Running or Stopped again. upgradeKubernetesCluster accepts a cluster in Alert (Alert -> Upgrading) when that detail is the requested version (or another one with the same semantic version), which is a real upgrade of the cluster's own version, and all of the cluster's VMs are running, and refuses any other Alert cluster with a clear message. For an upgrade that failed before this change, a cluster still at the old version with nodes recorded at the target version (kubernetesnodeversion, which is set as each node is done) is accepted too. Nodes that are already at the target version, both as recorded and as reported by kubectl get node, and that are Ready, are skipped. The first control node is still upgraded first; if it is already done, kubeadm upgrade apply is not run again. The node version check now compares versions exactly, so 1.28.1 no longer matches 1.28.10, line by line, as the SSH output can hold stderr too, and ignoring semver build metadata. Failure messages name the node the upgrade stopped on. The upgrade stops if it can't move the cluster to Upgrading, and the Alert state scanner no longer marks a cluster Running when it couldn't move it to Recovering, and a failed upgrade detaches its ISO before going to Alert, so a retry and the scanner or the failing job don't undo each other's work. No schema change. Signed-off-by: Brad House (cherry picked from commit 0dcaa8f982216527468a4965f183e92608b876f3) --- .../kubernetes/cluster/KubernetesCluster.java | 1 + .../cluster/KubernetesClusterManagerImpl.java | 58 +++++- .../cluster/KubernetesClusterService.java | 7 + .../KubernetesClusterActionWorker.java | 155 ++++++++++++++- .../KubernetesClusterScaleWorker.java | 176 ++++++++++++------ .../KubernetesClusterUpgradeWorker.java | 116 ++++++++---- .../cluster/utils/KubernetesClusterUtil.java | 22 ++- .../KubernetesClusterManagerImplTest.java | 106 +++++++++++ .../KubernetesClusterActionWorkerTest.java | 163 ++++++++++++++++ .../KubernetesClusterScaleWorkerTest.java | 145 +++++++++++++++ .../KubernetesClusterUpgradeWorkerTest.java | 72 +++++++ .../utils/KubernetesClusterUtilTest.java | 8 + ui/src/config/section/compute.js | 2 +- 13 files changed, 929 insertions(+), 102 deletions(-) diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index c06f8709d710..2be99f3df589 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -104,6 +104,7 @@ enum State { s_fsm.addTransition(State.ScalingStoppedCluster, Event.OperationFailed, State.Alert); s_fsm.addTransition(State.Running, Event.UpgradeRequested, State.Upgrading); + s_fsm.addTransition(State.Alert, Event.UpgradeRequested, State.Upgrading); s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 69d8ee6ce409..b6e2ccfda628 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -131,6 +131,7 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.Level; import com.cloud.api.ApiDBUtils; @@ -175,6 +176,7 @@ import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStopWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterUpgradeWorker; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; @@ -1570,6 +1572,40 @@ protected boolean isAnyNodeOfferingEmpty(Map map) { return map.values().stream().anyMatch(Objects::isNull); } + /** + * Tells whether upgrading a cluster in Alert state to a version retries an upgrade to that version that failed. + * An upgrade records the version it is upgrading to as a detail of the cluster, which is removed once the cluster + * is Running again. + * For an upgrade that failed before that detail was recorded, the nodes' versions are used: the version of the cluster is only changed once every node has been upgraded, while the version recorded for + * each node is changed as soon as that node has been upgraded. So, a cluster that is still at an older version + * but has nodes at the target version is one whose upgrade to that version stopped part way. Nodes that are + * upgraded manually are left out, as they may be at any version. + */ + protected boolean isRetryOfFailedUpgrade(KubernetesCluster kubernetesCluster, KubernetesSupportedVersion upgradeVersion) { + if (Objects.equals(upgradeVersion.getId(), kubernetesCluster.getKubernetesVersionId())) { + return false; + } + // Another version with the same semantic version as the cluster's is not an upgrade either + KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (clusterVersion == null || KubernetesClusterUtil.isSameNodeVersion(clusterVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion())) { + return false; + } + KubernetesClusterDetailsVO upgradeDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), KubernetesClusterUpgradeWorker.UPGRADE_VERSION_DETAIL); + if (upgradeDetail != null) { + if (String.valueOf(upgradeVersion.getId()).equals(upgradeDetail.getValue())) { + return true; + } + // Another version with the same semantic version, for example one registered with a fixed ISO + KubernetesSupportedVersionVO failedVersion = NumberUtils.isDigits(upgradeDetail.getValue()) ? + kubernetesSupportedVersionDao.findByIdIncludingRemoved(Long.parseLong(upgradeDetail.getValue())) : null; + return failedVersion != null && KubernetesClusterUtil.isSameNodeVersion(failedVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion()); + } + List vmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + return vmMaps != null && vmMaps.stream() + .filter(vmMap -> !vmMap.isManualUpgrade()) + .anyMatch(vmMap -> KubernetesClusterUtil.isSameNodeVersion(vmMap.getNodeVersion(), upgradeVersion.getSemanticVersion())); + } + private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) { // Validate parameters ApiServiceConfiguration.validateEndpointUrl(); @@ -1590,7 +1626,8 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster throw new InvalidParameterValueException(String.format("Upgrade kubernetes cluster is not supported for an externally managed cluster (%s)", kubernetesCluster.getName())); } accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); - if (!KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + boolean inAlert = KubernetesCluster.State.Alert.equals(kubernetesCluster.getState()); + if (!inAlert && !KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s is not in running state", kubernetesCluster.getName())); } final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); @@ -1604,6 +1641,16 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster if (!KubernetesSupportedVersion.State.Enabled.equals(upgradeVersion.getState())) { throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s for upgrade is in %s state", upgradeVersion.getUuid(), upgradeVersion.getState())); } + if (inAlert) { + if (!isRetryOfFailedUpgrade(kubernetesCluster, upgradeVersion)) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s is in Alert state. It can be upgraded in that state only " + + "to retry an upgrade that failed, to the same Kubernetes version as that upgrade", kubernetesCluster.getName())); + } + if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s is in Alert state and not all of its VMs are running. " + + "Start them before retrying the upgrade", kubernetesCluster.getName())); + } + } KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (clusterVersion == null || clusterVersion.getRemoved() != null) { throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster : %s", @@ -1640,7 +1687,13 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); try { - return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + boolean transited = _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + if (transited && (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) || + KubernetesCluster.State.Stopped.equals(kubernetesCluster.getState()))) { + // As in KubernetesClusterActionWorker.stateTransitTo: no longer in Alert because of a failed upgrade + kubernetesClusterDetailsDao.removeDetail(kubernetesClusterId, KubernetesClusterUpgradeWorker.UPGRADE_VERSION_DETAIL); + } + return transited; } catch (NoTransitionException nte) { logger.warn("Failed to transition state of the Kubernetes cluster: {} in state {} on event {}", kubernetesCluster, kubernetesCluster.getState().toString(), e.toString(), nte); return false; @@ -3135,6 +3188,7 @@ public ConfigKey[] getConfigKeys() { KubernetesClusterScaleTimeout, KubernetesClusterUpgradeTimeout, KubernetesClusterUpgradeRetries, + KubernetesClusterNodeDrainTimeout, KubernetesClusterAddNodeTimeout, KubernetesClusterRemoveNodeTimeout, KubernetesClusterExperimentalFeaturesEnabled, diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 1d19127dc54a..d0a5c6bc5f39 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -88,6 +88,13 @@ public interface KubernetesClusterService extends PluggableService, Configurable "The number of retries if fail to upgrade kubernetes cluster due to some reasons (e.g. drain node, etcdserver leader changed)", true, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesClusterNodeDrainTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.node.drain.timeout", + "300", + "Timeout interval (in seconds) for draining a Kubernetes node before it is removed from a cluster or upgraded. " + + "A drain that does not finish in time, for example because a PodDisruptionBudget does not allow an eviction, is retried, and the node is uncordoned if it still fails. Must be at least 1, smaller values use the default", + true, + KubernetesServiceEnabled.key()); static final ConfigKey KubernetesClusterAddNodeTimeout = new ConfigKey("Advanced", Long.class, "cloud.kubernetes.cluster.add.node.timeout", "3600", diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 23b8ac7a2407..1722d37d45b9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -154,6 +155,12 @@ public class KubernetesClusterActionWorker { public static final int ETCD_NODE_CLIENT_REQUEST_PORT = 2379; public static final int ETCD_NODE_PEER_COMM_PORT = 2380; public static final int CLUSTER_NODES_DEFAULT_SSH_PORT_SG = DEFAULT_SSH_PORT; + /** How long to wait before retrying a failed drain of a node, in milliseconds. */ + protected static final long DRAIN_RETRY_WAIT = 30000; + /** How much of the output of a failed drain to put in the error, in characters. */ + protected static final int MAX_DRAIN_OUTPUT_IN_ERROR = 2000; + /** How much longer than kubectl drain's own timeout the SSH call waits, in seconds. */ + protected static final long DRAIN_SSH_MARGIN = 60; public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup"; public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes"; @@ -372,8 +379,9 @@ protected void logMessage(final Level logLevel, final String message, final Exce protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster, final List clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { logMessage(logLevel, message, e); - stateTransitTo(kubernetesCluster.getId(), event); + // Detach first: once the cluster is in Alert, a retry of the upgrade may attach the ISO again detachIsoKubernetesVMs(clusterVMs); + stateTransitTo(kubernetesCluster.getId(), event); if (e == null) { throw new CloudRuntimeException(message); } @@ -774,6 +782,146 @@ protected void updateLoginUserDetails(List clusterVMs) { } } + /** Runs a command on a cluster node over SSH, as the cluster's login user with the management server's key. */ + protected Pair executeSshCommand(final String ipAddress, final int port, final String command, + final int connectTimeout, final int kexTimeout, final int waitTimeout) throws Exception { + return SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), null, + command, connectTimeout, kexTimeout, waitTimeout); + } + + /** Waits before a retry. Returns false if the thread was interrupted, in which case the caller should stop retrying. */ + protected boolean sleepBeforeRetry(final long millis) { + try { + Thread.sleep(millis); + return true; + } catch (InterruptedException ie) { + logger.warn("Interrupted while waiting to retry an operation on Kubernetes cluster: {}", kubernetesCluster, ie); + Thread.currentThread().interrupt(); + return false; + } + } + + protected static String getNodeName(final UserVm vm) { + String hostName = vm.getHostName(); + return StringUtils.isNotEmpty(hostName) ? hostName.toLowerCase() : hostName; + } + + /** + * Drains a Kubernetes node through the cluster's API endpoint, so that it can be removed or upgraded. + * Each attempt runs "kubectl drain" with the timeout of the cloud.kubernetes.cluster.node.drain.timeout setting, + * and the SSH call waits a little longer than that. A failed attempt, whether the command fails or the SSH call + * does, is retried after waiting, until the attempts are used up or the deadline has passed. No attempt runs + * past the deadline. A failed drain leaves the node cordoned: the caller uncordons it if it cordoned it, as only + * the caller knows whether the node was cordoned before. + * + * @return whether the node was drained, and the output of the last attempt, which explains a failure + */ + protected Pair drainKubernetesClusterNode(final String ipAddress, final int port, final UserVm vm, + final int attempts, final long retryWait, final long deadline) { + final String nodeName = getNodeName(vm); + Long configuredTimeout = KubernetesClusterService.KubernetesClusterNodeDrainTimeout.value(); + final long drainTimeout = configuredTimeout != null && configuredTimeout >= 1 ? configuredTimeout : + Long.parseLong(KubernetesClusterService.KubernetesClusterNodeDrainTimeout.defaultValue()); + String output = "no time was left to drain the node before the operation timed out"; + for (int attempt = 1; attempt <= attempts; attempt++) { + if (attempt > 1) { + // Only wait if there will still be time for an attempt afterwards + if (System.currentTimeMillis() + retryWait + (DRAIN_SSH_MARGIN + 1) * 1000 > deadline || !sleepBeforeRetry(retryWait)) { + break; + } + } + // Don't let an attempt run past the deadline of the operation, which would evict pods for nothing + final long secondsLeft = (deadline - System.currentTimeMillis()) / 1000 - DRAIN_SSH_MARGIN; + if (secondsLeft < 1) { + break; + } + final long timeout = Math.min(drainTimeout, secondsLeft); + final String command = String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data --timeout=%ds", + nodeName, timeout); + final int sshWaitTimeout = (int) Math.min(Integer.MAX_VALUE, (timeout + DRAIN_SSH_MARGIN) * 1000); + try { + Pair result = executeSshCommand(ipAddress, port, command, 10000, 10000, sshWaitTimeout); + output = result.second(); + if (Boolean.TRUE.equals(result.first())) { + return new Pair<>(true, output); + } + logger.warn("Draining node: {} on VM: {} in Kubernetes cluster: {} failed (attempt {} of {}): {}", + nodeName, vm, kubernetesCluster, attempt, attempts, output); + } catch (Exception e) { + output = e.getMessage(); + logger.warn("Draining node: {} on VM: {} in Kubernetes cluster: {} failed (attempt {} of {})", + nodeName, vm, kubernetesCluster, attempt, attempts, e); + } + } + // A drain blocked by a PodDisruptionBudget logs every retry of every eviction; the end is what explains it + return new Pair<>(false, StringUtils.right(output, MAX_DRAIN_OUTPUT_IN_ERROR)); + } + + /** + * Tells whether a node is cordoned already. If that can't be found out, it is taken as cordoned, so that a + * rollback doesn't uncordon a node that someone else may have cordoned. + */ + protected boolean isNodeCordoned(final UserVm vm) { + try { + Pair result = executeSshCommand(publicIpAddress, sshPort, + String.format("sudo /opt/bin/kubectl get node %s -o jsonpath='{.spec.unschedulable}'", getNodeName(vm)), 10000, 10000, 30000); + if (!Boolean.TRUE.equals(result.first()) || result.second() == null) { + logger.warn("Unable to find out whether node: {} on VM: {} in Kubernetes cluster: {} is cordoned, taking it as cordoned: {}", + getNodeName(vm), vm, kubernetesCluster, result.second()); + return true; + } + // The output also holds anything written to stderr, such as a warning from sudo + return result.second().lines().anyMatch(line -> "true".equals(line.trim())); + } catch (Exception e) { + logger.warn("Unable to find out whether node: {} on VM: {} in Kubernetes cluster: {} is cordoned, taking it as cordoned", getNodeName(vm), vm, kubernetesCluster, e); + return true; + } + } + + /** Uncordons nodes, only logging a warning for the ones that can't be uncordoned. */ + protected void uncordonNodes(final List vms) { + for (UserVm vm : vms) { + if (!uncordonKubernetesClusterNode(publicIpAddress, sshPort, vm)) { + logger.warn("Unable to uncordon node: {} on VM: {} in Kubernetes cluster: {}", getNodeName(vm), vm, kubernetesCluster); + } + } + } + + /** Uncordons a node, trying for up to a minute. */ + protected boolean uncordonKubernetesClusterNode(final String ipAddress, final int port, final UserVm vm) { + return KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, ipAddress, port, getControlNodeLoginUser(), + getManagementServerSshPublicKeyFile(), vm, System.currentTimeMillis() + 60000, 10000); + } + + /** + * Returns the address and port to reach a node over SSH. On a network with port forwarding, the port is the + * one the node's port-forwarding rule actually uses. The rules are not always in the order of the cluster's VMs: + * external nodes and nodes left out of an operation, such as manual-upgrade nodes, shift the order. When the node + * has no such rule, the port is derived from its position in the list of nodes, as before. + */ + protected Pair getNodeSshAddress(final UserVm vm, final int index) { + if (sshPort == DEFAULT_SSH_PORT) { + return new Pair<>(index > 0 ? vm.getPrivateIpAddress() : publicIpAddress, DEFAULT_SSH_PORT); + } + List rules = portForwardingRulesDao.listByVm(vm.getId()); + if (rules != null) { + Optional rule = rules.stream() + .filter(r -> !FirewallRule.State.Revoke.equals(r.getState())) + .filter(r -> r.getDestinationPortStart() == DEFAULT_SSH_PORT && r.getSourcePortStart() >= CLUSTER_NODES_DEFAULT_START_SSH_PORT) + .filter(r -> isRuleOnAddress(r, publicIpAddress)) + .findFirst(); + if (rule.isPresent()) { + return new Pair<>(publicIpAddress, rule.get().getSourcePortStart()); + } + } + return new Pair<>(publicIpAddress, sshPort + index); + } + + private boolean isRuleOnAddress(final PortForwardingRuleVO rule, final String address) { + IPAddressVO ip = ipAddressDao.findById(rule.getSourceIpAddressId()); + return ip != null && ip.getAddress() != null && ip.getAddress().addr().equals(address); + } + /** * Moves the cluster on with the given event. Once this worker has started its operation with * {@link #transitToOperationState}, it only moves the cluster out of the state that the operation last moved it @@ -798,6 +946,11 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve // miss, another operation already has ownedState = transitioned ? kubernetesCluster.getState() : null; } + if (transitioned && (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) || + KubernetesCluster.State.Stopped.equals(kubernetesCluster.getState()))) { + // A cluster that is Running or Stopped again is no longer in Alert because of a failed upgrade + kubernetesClusterDetailsDao.removeDetail(kubernetesClusterId, KubernetesClusterUpgradeWorker.UPGRADE_VERSION_DETAIL); + } return transitioned; } catch (NoTransitionException nte) { logger.warn(String.format("Failed to transition state of the Kubernetes cluster : %s in state %s on event %s", diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 535069d497da..3037794e189a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -17,7 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -59,7 +58,6 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.UserVmVO; import org.apache.logging.log4j.Level; @@ -78,7 +76,9 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private Long minSize; private Long maxSize; private Boolean isAutoscalingEnabled; - private long scaleTimeoutTime; + protected long scaleTimeoutTime; + /** How many times to try to drain a node that is being removed, as scale-down did before. */ + private static final int SCALE_DOWN_DRAIN_ATTEMPTS = 3; protected KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); @@ -280,46 +280,89 @@ protected Pair calculateNodesCapacity(ServiceOffering offering, long return new Pair<>(offering.getCpu() * nodeCount, offering.getRamSize() * nodeCount); } + /** + * Deletes the Kubernetes node object of a VM that is being removed from the cluster. The node has been drained + * already, see {@link #drainNodesToRemove(List)}. A failed attempt is retried after waiting. + */ private boolean removeKubernetesClusterNode(final String ipAddress, final int port, final UserVm userVm, final int retries, final int waitDuration) { - File pkFile = getManagementServerSshPublicKeyFile(); - int retryCounter = 0; - String hostName = userVm.getHostName(); - if (StringUtils.isNotEmpty(hostName)) { - hostName = hostName.toLowerCase(); - } - while (retryCounter < retries) { - retryCounter++; + final String hostName = getNodeName(userVm); + for (int attempt = 1; attempt <= retries; attempt++) { + if (attempt > 1 && !sleepBeforeRetry(waitDuration)) { + break; + } try { - Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), - pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), - 10000, 10000, 60000); - if (!result.first()) { - logger.warn("Draining node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); - } else { - result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), - pkFile, null, String.format("sudo /opt/bin/kubectl delete node %s", hostName), - 10000, 10000, 30000); - if (result.first()) { - return true; - } else { - logger.warn("Deleting node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); - } + Pair result = executeSshCommand(ipAddress, port, + String.format("sudo /opt/bin/kubectl delete node %s --ignore-not-found", hostName), 10000, 10000, 30000); + if (result.first()) { + return true; } - break; + logger.warn("Deleting node: {} on VM: {} in Kubernetes cluster: {} unsuccessful: {}", hostName, userVm, kubernetesCluster, result.second()); } catch (Exception e) { - String msg = String.format("Failed to remove Kubernetes cluster: %s node: %s on VM: %s", kubernetesCluster, hostName, userVm); - logger.warn(msg, e); + logger.warn("Failed to remove Kubernetes cluster: {} node: {} on VM: {}", kubernetesCluster, hostName, userVm, e); } + } + return false; + } + + /** + * Drains the nodes that are being removed, before any of them is removed. All of them are cordoned first, so + * that the pods evicted from one are not moved to another one that is going away. If a node can't be drained, + * all of them are uncordoned and the scale-down stops with nothing removed: the nodes are still working members + * of the cluster, so the cluster goes back to the state it was in, rather than to Alert. + * + * @return the nodes that were cordoned here, as opposed to the ones that were cordoned already + */ + protected List drainNodesToRemove(final List vms) throws CloudRuntimeException { + List cordoned = new ArrayList<>(); + String failure = null; + for (UserVmVO vm : vms) { + // A node cordoned by someone else has to stay cordoned if the scale-down is rolled back + final boolean cordonedAlready = isNodeCordoned(vm); + String output; try { - Thread.sleep(waitDuration); - } catch (InterruptedException ie) { - logger.error("Error while waiting for Kubernetes cluster: {} node: {} on VM: {} removal", kubernetesCluster, hostName, userVm, ie); + Pair result = executeSshCommand(publicIpAddress, sshPort, + String.format("sudo /opt/bin/kubectl cordon %s", getNodeName(vm)), 10000, 10000, 30000); + if (result.first()) { + if (!cordonedAlready) { + cordoned.add(vm); + } + continue; + } + output = result.second(); + } catch (Exception e) { + output = e.getMessage(); + } + failure = String.format("unable to cordon Kubernetes node: %s on VM: %s: %s", getNodeName(vm), vm, output); + break; + } + if (failure == null) { + for (UserVmVO vm : vms) { + if (System.currentTimeMillis() > scaleTimeoutTime) { + failure = String.format("scaling action timed out before Kubernetes node: %s on VM: %s was drained", getNodeName(vm), vm); + break; + } + Pair drained = drainKubernetesClusterNode(publicIpAddress, sshPort, vm, SCALE_DOWN_DRAIN_ATTEMPTS, DRAIN_RETRY_WAIT, scaleTimeoutTime); + if (!drained.first()) { + failure = String.format("unable to drain Kubernetes node: %s on VM: %s: %s", getNodeName(vm), vm, drained.second()); + break; + } } - retryCounter++; } - return false; + if (failure == null) { + return cordoned; + } + uncordonNodes(cordoned); + String message = String.format("Scaling failed for Kubernetes cluster %s, %s. No node was removed", kubernetesCluster, failure); + logger.error(message); + // Nothing has changed, so the cluster goes back to the state it was in: Scaling goes back to Running (and + // ScalingStoppedCluster to Stopped) on OperationSucceeded. Scaling a cluster in Alert is not allowed today, + // but if it is, such a cluster has to go back to Alert, not to Running. + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.State.Alert.equals(originalState) ? + KubernetesCluster.Event.OperationFailed : KubernetesCluster.Event.OperationSucceeded); + throw new CloudRuntimeException(message); } + private void validateKubernetesClusterScaleOfferingParameters() throws CloudRuntimeException { if (KubernetesCluster.State.Created.equals(originalState)) { return; @@ -392,34 +435,29 @@ private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, } private void removeNodesFromCluster(List vmMaps) throws CloudRuntimeException { - for (KubernetesClusterVmMapVO vmMapVO : vmMaps) { - UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); - logger.info("Removing vm {} from cluster {}", userVM, kubernetesCluster); - if (!removeKubernetesClusterNode(publicIpAddress, sshPort, userVM, 3, 30000)) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes" + - " cluster %s, failed to remove Kubernetes node: %s running on VM : %s", - kubernetesCluster, userVM.getHostName(), userVM), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - CallContext vmContext = CallContext.register(CallContext.current(), - ApiCommandResourceType.VirtualMachine); - vmContext.setEventResourceId(userVM.getId()); - try { - UserVm vm = userVmService.destroyVm(userVM.getId(), true); - if (!userVmManager.expunge(userVM)) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to expunge VM '%s'." - , kubernetesCluster, vm), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + List vms = vmMaps.stream().map(vmMapVO -> userVmDao.findById(vmMapVO.getVmId())).collect(Collectors.toList()); + List cordoned = drainNodesToRemove(vms); + // Index of the first VM whose Kubernetes node is still in the cluster + int firstInCluster = 0; + try { + for (int i = 0; i < vmMaps.size(); i++) { + // No timeout check here: the nodes are drained, and stopping part way would leave the cluster + // with some of them removed and the node count not updated + UserVmVO userVM = vms.get(i); + logger.info("Removing vm {} from cluster {}", userVM, kubernetesCluster); + if (!removeKubernetesClusterNode(publicIpAddress, sshPort, userVM, 3, 30000)) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes" + + " cluster %s, failed to remove Kubernetes node: %s running on VM : %s", + kubernetesCluster, userVM.getHostName(), userVM), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - } catch (ResourceUnavailableException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to remove VM ID: %s", - kubernetesCluster, userVM), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); - } finally { - CallContext.unregister(); - } - kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); - if (System.currentTimeMillis() > scaleTimeoutTime) { - logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster %s failed, scaling action timed out", - kubernetesCluster), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + firstInCluster = i + 1; + destroyRemovedNodeVm(vmMaps.get(i), userVM); } + } catch (CloudRuntimeException e) { + // The nodes that are still in the cluster were drained above and would stay unschedulable + List inCluster = vms.subList(firstInCluster, vms.size()); + uncordonNodes(cordoned.stream().filter(inCluster::contains).collect(Collectors.toList())); + throw e; } // Scale network rules to update firewall rule @@ -436,6 +474,26 @@ private void removeNodesFromCluster(List vmMaps) throw } } + /** Destroys and expunges the VM of a node that has been deleted from the Kubernetes cluster. */ + private void destroyRemovedNodeVm(final KubernetesClusterVmMapVO vmMapVO, final UserVmVO userVM) throws CloudRuntimeException { + CallContext vmContext = CallContext.register(CallContext.current(), + ApiCommandResourceType.VirtualMachine); + vmContext.setEventResourceId(userVM.getId()); + try { + UserVm vm = userVmService.destroyVm(userVM.getId(), true); + if (!userVmManager.expunge(userVM)) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to expunge VM '%s'." + , kubernetesCluster, vm), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } catch (ResourceUnavailableException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to remove VM ID: %s", + kubernetesCluster, userVM), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } finally { + CallContext.unregister(); + } + kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); + } + private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { enterScalingState(KubernetesCluster.Event.ScaleDownRequested); List vmList; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index aa0f967b7852..1ece29f23115 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -19,6 +19,7 @@ import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -43,6 +44,11 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker { + /** Cluster detail holding the id of the Kubernetes version that an upgrade, not finished yet, is upgrading to. */ + public static final String UPGRADE_VERSION_DETAIL = "upgradeKubernetesVersionId"; + /** Prints the STATUS of a node, such as Ready, NotReady or Ready,SchedulingDisabled. */ + protected static final String NODE_STATUS_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\") print $2}'"; + protected List clusterVMs = new ArrayList<>(); private KubernetesSupportedVersion upgradeVersion; private final String upgradeScriptFilename = "upgrade-kubernetes.sh"; @@ -64,8 +70,9 @@ protected void retrieveScriptFiles() { } private Pair runInstallScriptOnVM(final UserVm vm, final int index) throws Exception { - int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index; - String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; + Pair nodeSshAddress = getNodeSshAddress(vm, index); + String nodeAddress = nodeSshAddress.first(); + int nodeSshPort = nodeSshAddress.second(); SshHelper.scpTo(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, "~/", upgradeScriptFile.getAbsolutePath(), "0755"); String cmdStr = String.format("sudo ./%s %s %s %s %s %s", @@ -82,40 +89,35 @@ private Pair runInstallScriptOnVM(final UserVm vm, final int in private void upgradeKubernetesClusterNodes() { for (int i = 0; i < clusterVMs.size(); ++i) { UserVm vm = clusterVMs.get(i); - String hostName = vm.getHostName(); - if (StringUtils.isNotEmpty(hostName)) { - hostName = hostName.toLowerCase(); - } + String hostName = getNodeName(vm); Pair result; + if (isNodeUpgraded(vm, hostName)) { + // A previous, failed, upgrade to this version got this far. On the first control node this also + // means that kubeadm upgrade apply has run, so the other nodes only need kubeadm upgrade node. + logger.info("Skipping node {} on VM {} in Kubernetes cluster {}, it is already at Kubernetes version {}", hostName, vm, kubernetesCluster, upgradeVersion); + continue; + } if (logger.isInfoEnabled()) { - logger.info("Upgrading node on VM {} in Kubernetes cluster {} with Kubernetes version {}", vm, kubernetesCluster, upgradeVersion); + logger.info("Upgrading node {} on VM {} in Kubernetes cluster {} with Kubernetes version {}", hostName, vm, kubernetesCluster, upgradeVersion); } - String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to drain Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()); - for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) { - try { - result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), - 10000, 10000, 60000); - if (result.first()) { - break; - } - if (retry > 0) { - logger.error(String.format("%s, retries left: %s", errorMessage, retry)); - } else { - logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); - } - } catch (Exception e) { - if (retry > 0) { - logger.error(String.format("%s due to %s, retries left: %s", errorMessage, e, retry)); - } else { - logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); - } - } + if (System.currentTimeMillis() > upgradeTimeoutTime) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out on Kubernetes node : %s on VM : %s", kubernetesCluster.getName(), hostName, vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } + // A node cordoned by someone else stays cordoned if the upgrade stops before upgrading it + final List toUncordon = isNodeCordoned(vm) ? List.of() : List.of(vm); + Pair drained = drainKubernetesClusterNode(publicIpAddress, sshPort, vm, + KubernetesClusterService.KubernetesClusterUpgradeRetries.value() + 1, DRAIN_RETRY_WAIT, upgradeTimeoutTime); + if (!drained.first()) { + uncordonNodes(toUncordon); + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to drain Kubernetes node : %s on VM : %s: %s", + kubernetesCluster.getName(), hostName, vm.getDisplayName(), drained.second()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (System.currentTimeMillis() > upgradeTimeoutTime) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + // Nothing has been changed on the node, so don't leave it unschedulable + uncordonNodes(toUncordon); + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out on Kubernetes node : %s on VM : %s", kubernetesCluster.getName(), hostName, vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } - errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()); + String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node : %s on VM : %s", kubernetesCluster.getName(), hostName, vm.getDisplayName()); for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) { try { deployProvider(); @@ -137,18 +139,19 @@ private void upgradeKubernetesClusterNodes() { } } if (System.currentTimeMillis() > upgradeTimeoutTime) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + // The node has been upgraded, so don't leave it unschedulable, unless someone else cordoned it + uncordonNodes(toUncordon); + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out on Kubernetes node : %s on VM : %s", kubernetesCluster.getName(), hostName, vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (!KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), vm, upgradeTimeoutTime, 15000)) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to uncordon Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to uncordon Kubernetes node : %s on VM : %s", kubernetesCluster.getName(), hostName, vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } - if (i == 0) { // Wait for control node to get in Ready state - if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); - } + // Wait for every node, not only the first control node, to be Ready before the next one is drained + if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node : %s on VM : %s in ready state", kubernetesCluster.getName(), hostName, vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000, vm.getId(), kubernetesClusterVmMapDao)) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node : %s on VM : %s upgraded to version %s", kubernetesCluster.getName(), hostName, vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (logger.isInfoEnabled()) { logger.info("Successfully upgraded node on VM {} in Kubernetes cluster {} with Kubernetes version {}", vm, kubernetesCluster, upgradeVersion); @@ -156,6 +159,42 @@ private void upgradeKubernetesClusterNodes() { } } + /** + * Tells whether a node was upgraded to the target version by an earlier upgrade of the cluster that failed on a + * later node, so that a retry can carry on from where that one stopped. The version recorded for the node is + * only updated once the node has been upgraded, uncordoned and found Ready, but it is not trusted on its own: + * the node must also report the target version to Kubernetes, and still be Ready and schedulable. + */ + protected boolean isNodeUpgraded(final UserVm vm, final String hostName) { + KubernetesClusterVmMapVO vmMapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(vm.getId()); + String version = upgradeVersion.getSemanticVersion(); + if (vmMapVO == null || !KubernetesClusterUtil.isSameNodeVersion(vmMapVO.getNodeVersion(), version)) { + return false; + } + try { + Pair result = executeSshCommand(publicIpAddress, sshPort, + String.format(KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, hostName), 10000, 10000, 20000); + if (!Boolean.TRUE.equals(KubernetesClusterUtil.clusterNodeVersionMatches(result, version).first())) { + logger.info("Node {} on VM {} in Kubernetes cluster {} is recorded at Kubernetes version {}, but reports: {}", + hostName, vm, kubernetesCluster, version, result.second()); + return false; + } + // Only a node that is also Ready is done; any other one is upgraded again. A node that is Ready but + // cordoned (Ready,SchedulingDisabled) was cordoned by someone else after its upgrade, and is left as it is. + result = executeSshCommand(publicIpAddress, sshPort, + String.format(NODE_STATUS_COMMAND, hostName), 10000, 10000, 20000); + if (Boolean.TRUE.equals(result.first()) && result.second() != null && + result.second().lines().anyMatch(line -> Arrays.asList(line.trim().split(",")).contains("Ready"))) { + return true; + } + logger.info("Node {} on VM {} in Kubernetes cluster {} is at Kubernetes version {}, but is not Ready, so it is upgraded again", + hostName, vm, kubernetesCluster, version); + } catch (Exception e) { + logger.warn("Unable to get the Kubernetes version of node {} on VM {} in Kubernetes cluster {}", hostName, vm, kubernetesCluster, e); + } + return false; + } + public boolean upgradeCluster() throws CloudRuntimeException { init(); if (logger.isInfoEnabled()) { @@ -175,6 +214,9 @@ public boolean upgradeCluster() throws CloudRuntimeException { filterOutManualUpgradeNodesFromClusterUpgrade(); retrieveScriptFiles(); transitToOperationState(KubernetesCluster.Event.UpgradeRequested); + // Kept until the cluster is Running again (see stateTransitTo), so that a failed upgrade can be told apart + // from other causes of Alert + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), UPGRADE_VERSION_DETAIL, String.valueOf(upgradeVersion.getId()), false); attachIsoKubernetesVMs(clusterVMs, upgradeVersion); upgradeKubernetesClusterNodes(); detachIsoKubernetesVMs(clusterVMs); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 023fc9ec7fad..46a3f71c136a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -377,11 +377,29 @@ public static boolean clusterNodeVersionMatches(final String version, return false; } - protected static Pair clusterNodeVersionMatches(final Pair result, final String version) { + /** + * Compares a node version, as recorded for a cluster VM or as printed by kubectl (with a leading "v"), with a + * semantic version. The SSH output also holds whatever the command wrote to stderr, such as a warning from sudo, + * so any line of it that is exactly the version, ignoring build metadata such as "+vendor.1", matches. + */ + public static boolean isSameNodeVersion(final String nodeVersion, final String version) { + if (StringUtils.isBlank(nodeVersion) || StringUtils.isBlank(version)) { + return false; + } + final String expected = withoutBuildMetadata(StringUtils.removeStart(version.trim(), "v")); + return nodeVersion.lines().anyMatch(line -> withoutBuildMetadata(StringUtils.removeStart(line.trim(), "v")).equals(expected)); + } + + /** Semantic versions that only differ in their build metadata, after a "+", are the same version. */ + private static String withoutBuildMetadata(final String version) { + return StringUtils.substringBefore(version, "+"); + } + + public static Pair clusterNodeVersionMatches(final Pair result, final String version) { if (result == null || Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) { return new Pair<>(false, null); } String response = result.second(); - return new Pair<>(response.contains(String.format("v%s", version)), response); + return new Pair<>(isSameNodeVersion(response, version), response); } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index cd33970ec483..79f051541535 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -27,9 +27,13 @@ import com.cloud.exception.PermissionDeniedException; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterUpgradeWorker; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; import com.cloud.network.Network; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRule; @@ -103,6 +107,12 @@ public class KubernetesClusterManagerImplTest { @Mock KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Mock + KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + + @Mock + KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Mock VMInstanceDao vmInstanceDao; @@ -1022,4 +1032,100 @@ public void recoverOrphanedOperationStateFailsStart() throws Exception { KubernetesCluster.State.Alert, cluster, null); } + private KubernetesCluster mockClusterForUpgradeRetry(String... nodeVersions) { + KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.when(cluster.getKubernetesVersionId()).thenReturn(10L); + List vmMaps = new ArrayList<>(); + for (String nodeVersion : nodeVersions) { + KubernetesClusterVmMapVO vmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(vmMap.getNodeVersion()).thenReturn(nodeVersion); + vmMaps.add(vmMap); + } + Mockito.when(kubernetesClusterVmMapDao.listByClusterId(1L)).thenReturn(vmMaps); + mockClusterVersion("1.28.9"); + return cluster; + } + + private void mockClusterVersion(String semanticVersion) { + KubernetesSupportedVersionVO clusterVersion = Mockito.mock(KubernetesSupportedVersionVO.class); + Mockito.when(clusterVersion.getSemanticVersion()).thenReturn(semanticVersion); + Mockito.when(kubernetesSupportedVersionDao.findById(10L)).thenReturn(clusterVersion); + } + + private KubernetesSupportedVersion mockUpgradeVersion(long id) { + KubernetesSupportedVersion version = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(version.getId()).thenReturn(id); + Mockito.lenient().when(version.getSemanticVersion()).thenReturn("1.29.2"); + return version; + } + + @Test + public void testIsRetryOfFailedUpgradeWithNodeAtTargetVersion() { + KubernetesCluster cluster = mockClusterForUpgradeRetry("v1.29.2", "1.28.9", "1.28.9"); + Assert.assertTrue(kubernetesClusterManager.isRetryOfFailedUpgrade(cluster, mockUpgradeVersion(11L))); + } + + @Test + public void testIsRetryOfFailedUpgradeWithNoNodeAtTargetVersion() { + KubernetesCluster cluster = mockClusterForUpgradeRetry("1.28.9", "v1.28.9"); + Assert.assertFalse(kubernetesClusterManager.isRetryOfFailedUpgrade(cluster, mockUpgradeVersion(11L))); + } + + @Test + public void testIsRetryOfFailedUpgradeIgnoresManuallyUpgradedNodes() { + KubernetesCluster cluster = mockClusterForUpgradeRetry("1.28.9"); + KubernetesClusterVmMapVO manualNode = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(manualNode.isManualUpgrade()).thenReturn(true); + Mockito.lenient().when(manualNode.getNodeVersion()).thenReturn("v1.29.2"); + List vmMaps = new ArrayList<>(kubernetesClusterVmMapDao.listByClusterId(1L)); + vmMaps.add(manualNode); + Mockito.when(kubernetesClusterVmMapDao.listByClusterId(1L)).thenReturn(vmMaps); + Assert.assertFalse(kubernetesClusterManager.isRetryOfFailedUpgrade(cluster, mockUpgradeVersion(11L))); + } + + @Test + public void testIsRetryOfFailedUpgradeToCurrentVersion() { + KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + Mockito.when(cluster.getKubernetesVersionId()).thenReturn(10L); + KubernetesSupportedVersion version = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(version.getId()).thenReturn(10L); + Assert.assertFalse(kubernetesClusterManager.isRetryOfFailedUpgrade(cluster, version)); + Mockito.verifyNoInteractions(kubernetesClusterVmMapDao); + } + + @Test + public void testIsRetryOfFailedUpgradeUsesRecordedUpgradeVersion() { + KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class); + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.when(cluster.getKubernetesVersionId()).thenReturn(10L); + Mockito.when(kubernetesClusterDetailsDao.findDetail(1L, KubernetesClusterUpgradeWorker.UPGRADE_VERSION_DETAIL)) + .thenReturn(new KubernetesClusterDetailsVO(1L, KubernetesClusterUpgradeWorker.UPGRADE_VERSION_DETAIL, "11", false)); + mockClusterVersion("1.28.9"); + KubernetesSupportedVersion sameVersion = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(sameVersion.getId()).thenReturn(11L); + Mockito.when(sameVersion.getSemanticVersion()).thenReturn("1.29.2"); + KubernetesSupportedVersion otherVersion = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(otherVersion.getId()).thenReturn(12L); + Mockito.when(otherVersion.getSemanticVersion()).thenReturn("1.29.3"); + // No node has to be at the version, for example when the first control node failed + KubernetesSupportedVersionVO failedVersion = Mockito.mock(KubernetesSupportedVersionVO.class); + Mockito.when(failedVersion.getSemanticVersion()).thenReturn("1.29.2"); + Mockito.when(kubernetesSupportedVersionDao.findByIdIncludingRemoved(11L)).thenReturn(failedVersion); + KubernetesSupportedVersion equivalentVersion = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(equivalentVersion.getId()).thenReturn(13L); + Mockito.when(equivalentVersion.getSemanticVersion()).thenReturn("1.29.2"); + Assert.assertTrue(kubernetesClusterManager.isRetryOfFailedUpgrade(cluster, sameVersion)); + Assert.assertTrue(kubernetesClusterManager.isRetryOfFailedUpgrade(cluster, equivalentVersion)); + Assert.assertFalse(kubernetesClusterManager.isRetryOfFailedUpgrade(cluster, otherVersion)); + Mockito.verifyNoInteractions(kubernetesClusterVmMapDao); + } + + @Test + public void testIsRetryOfFailedUpgradeToSameSemanticVersion() { + // Another version row with the cluster's own semantic version, with every node recorded at it + KubernetesCluster cluster = mockClusterForUpgradeRetry("1.29.2", "1.29.2"); + mockClusterVersion("1.29.2"); + Assert.assertFalse(kubernetesClusterManager.isRetryOfFailedUpgrade(cluster, mockUpgradeVersion(11L))); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java index 1f0451942e31..bb43ac6087a3 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java @@ -48,6 +48,11 @@ import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.net.Ip; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterActionWorkerTest { @@ -76,6 +81,9 @@ public class KubernetesClusterActionWorkerTest { @Mock AffinityGroupDao affinityGroupDao; + @Mock + PortForwardingRulesDao portForwardingRulesDao; + KubernetesClusterActionWorker actionWorker = null; final static Long DEFAULT_ID = 1L; @@ -92,6 +100,7 @@ public void setUp() throws Exception { actionWorker = new KubernetesClusterActionWorker(kubernetesCluster, kubernetesClusterManager); actionWorker.ipAddressDao = ipAddressDao; actionWorker.affinityGroupDao = affinityGroupDao; + actionWorker.portForwardingRulesDao = portForwardingRulesDao; } @Test @@ -336,4 +345,158 @@ public void testStateTransitToKeepsTheStateAfterAnEventWithoutTransition() { Assert.assertFalse(actionWorker.stateTransitTo(DEFAULT_ID, KubernetesCluster.Event.CreateFailed)); Assert.assertTrue(actionWorker.stateTransitTo(DEFAULT_ID, KubernetesCluster.Event.OperationFailed)); } + + private UserVm mockNodeVm() { + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getHostName()).thenReturn("Node-1"); + return vm; + } + + private static final String DRAIN_COMMAND = "sudo /opt/bin/kubectl drain node-1 --ignore-daemonsets --delete-emptydir-data --timeout=300s"; + + @Test + public void testDrainKubernetesClusterNodeRetriesUntilSuccess() throws Exception { + KubernetesClusterActionWorker worker = Mockito.spy(actionWorker); + UserVm vm = mockNodeVm(); + Mockito.doReturn(new Pair<>(false, "Cannot evict pod as it would violate the pod's disruption budget"), new Pair<>(true, "node/node-1 drained")) + .when(worker).executeSshCommand("10.0.0.1", 2222, DRAIN_COMMAND, 10000, 10000, 360000); + Mockito.doReturn(true).when(worker).sleepBeforeRetry(Mockito.anyLong()); + + Pair result = worker.drainKubernetesClusterNode("10.0.0.1", 2222, vm, 3, 5, Long.MAX_VALUE); + + Assert.assertTrue(result.first()); + Assert.assertEquals("node/node-1 drained", result.second()); + Mockito.verify(worker, Mockito.times(2)).executeSshCommand("10.0.0.1", 2222, DRAIN_COMMAND, 10000, 10000, 360000); + Mockito.verify(worker, Mockito.times(1)).sleepBeforeRetry(5); + } + + @Test + public void testDrainKubernetesClusterNodeUsesEveryAttempt() throws Exception { + KubernetesClusterActionWorker worker = Mockito.spy(actionWorker); + UserVm vm = mockNodeVm(); + Mockito.doThrow(new RuntimeException("connection refused")).doReturn(new Pair<>(false, "pdb")) + .when(worker).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(true).when(worker).sleepBeforeRetry(Mockito.anyLong()); + + Pair result = worker.drainKubernetesClusterNode("10.0.0.1", 2222, vm, 3, 5, Long.MAX_VALUE); + + Assert.assertFalse(result.first()); + Assert.assertEquals("pdb", result.second()); + Mockito.verify(worker, Mockito.times(3)).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.verify(worker, Mockito.times(2)).sleepBeforeRetry(5); + // The caller uncordons the node, as only it knows whether it cordoned it + Mockito.verify(worker, Mockito.never()).uncordonKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any()); + } + + @Test + public void testDrainKubernetesClusterNodeStopsRetryingAtDeadline() throws Exception { + KubernetesClusterActionWorker worker = Mockito.spy(actionWorker); + UserVm vm = mockNodeVm(); + Mockito.doReturn(new Pair<>(false, "timed out")) + .when(worker).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + + // 90 seconds left: one attempt, with kubectl's timeout cut to what is left after the SSH margin + Pair result = worker.drainKubernetesClusterNode("10.0.0.1", 2222, vm, 3, 60000, System.currentTimeMillis() + 90500); + + Assert.assertFalse(result.first()); + Mockito.verify(worker, Mockito.times(1)).executeSshCommand(Mockito.eq("10.0.0.1"), Mockito.eq(2222), + Mockito.matches(".*--timeout=(29|30)s"), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.verify(worker, Mockito.never()).sleepBeforeRetry(Mockito.anyLong()); + } + + @Test + public void testDrainKubernetesClusterNodeWithNoTimeLeft() throws Exception { + KubernetesClusterActionWorker worker = Mockito.spy(actionWorker); + UserVm vm = mockNodeVm(); + + Pair result = worker.drainKubernetesClusterNode("10.0.0.1", 2222, vm, 3, 60000, System.currentTimeMillis() + 30000); + + Assert.assertFalse(result.first()); + Assert.assertTrue(result.second().contains("timed out")); + Mockito.verify(worker, Mockito.never()).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + } + + @Test + public void testDrainKubernetesClusterNodeStopsRetryingWhenInterrupted() throws Exception { + KubernetesClusterActionWorker worker = Mockito.spy(actionWorker); + UserVm vm = mockNodeVm(); + Mockito.doReturn(new Pair<>(false, "pdb")) + .when(worker).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(false).when(worker).sleepBeforeRetry(Mockito.anyLong()); + + Assert.assertFalse(worker.drainKubernetesClusterNode("10.0.0.1", 2222, vm, 3, 5, Long.MAX_VALUE).first()); + Mockito.verify(worker, Mockito.times(1)).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + } + + @Test + public void testGetNodeSshAddressIgnoresRevokedRule() { + actionWorker.publicIpAddress = "10.0.0.1"; + actionWorker.sshPort = 2222; + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getId()).thenReturn(7L); + mockIpAddress(1L, "10.0.0.1"); + PortForwardingRuleVO revoked = Mockito.mock(PortForwardingRuleVO.class); + Mockito.when(revoked.getState()).thenReturn(com.cloud.network.rules.FirewallRule.State.Revoke); + PortForwardingRuleVO active = mockSshRule(1L, 2226); + Mockito.when(portForwardingRulesDao.listByVm(7L)).thenReturn(List.of(revoked, active)); + + Assert.assertEquals(Integer.valueOf(2226), actionWorker.getNodeSshAddress(vm, 1).second()); + } + + private PortForwardingRuleVO mockSshRule(long ipId, int sourcePort) { + PortForwardingRuleVO rule = Mockito.mock(PortForwardingRuleVO.class); + Mockito.when(rule.getDestinationPortStart()).thenReturn(22); + Mockito.when(rule.getSourcePortStart()).thenReturn(sourcePort); + Mockito.when(rule.getSourceIpAddressId()).thenReturn(ipId); + return rule; + } + + private void mockIpAddress(long id, String address) { + IPAddressVO ip = Mockito.mock(IPAddressVO.class); + Mockito.when(ip.getAddress()).thenReturn(new Ip(address)); + Mockito.when(ipAddressDao.findById(id)).thenReturn(ip); + } + + @Test + public void testGetNodeSshAddressUsesPortForwardingRule() { + actionWorker.publicIpAddress = "10.0.0.1"; + actionWorker.sshPort = 2222; + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getId()).thenReturn(7L); + mockIpAddress(1L, "10.0.0.1"); + mockIpAddress(2L, "10.0.0.2"); + List rules = List.of(mockSshRule(2L, 2230), mockSshRule(1L, 2225)); + Mockito.when(portForwardingRulesDao.listByVm(7L)).thenReturn(rules); + + Pair address = actionWorker.getNodeSshAddress(vm, 1); + + Assert.assertEquals("10.0.0.1", address.first()); + Assert.assertEquals(Integer.valueOf(2225), address.second()); + } + + @Test + public void testGetNodeSshAddressWithoutRuleFallsBackToIndex() { + actionWorker.publicIpAddress = "10.0.0.1"; + actionWorker.sshPort = 2222; + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getId()).thenReturn(7L); + Mockito.when(portForwardingRulesDao.listByVm(7L)).thenReturn(Collections.emptyList()); + + Pair address = actionWorker.getNodeSshAddress(vm, 2); + + Assert.assertEquals("10.0.0.1", address.first()); + Assert.assertEquals(Integer.valueOf(2224), address.second()); + } + + @Test + public void testGetNodeSshAddressOnSharedNetwork() { + actionWorker.publicIpAddress = "10.0.0.1"; + actionWorker.sshPort = 22; + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getPrivateIpAddress()).thenReturn("192.168.0.5"); + + Assert.assertEquals(new Pair<>("10.0.0.1", 22), actionWorker.getNodeSshAddress(vm, 0)); + Assert.assertEquals(new Pair<>("192.168.0.5", 22), actionWorker.getNodeSshAddress(vm, 1)); + Mockito.verifyNoInteractions(portForwardingRulesDao); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index c9299bdbaa67..4c95f4d0fe0e 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -24,6 +24,7 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.UserVmVO; import com.cloud.vm.dao.UserVmDao; import org.junit.Assert; @@ -187,4 +188,148 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { Assert.assertTrue(toRemove.isEmpty()); } + + private UserVmVO mockNodeVm(String hostName) { + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.when(vm.getHostName()).thenReturn(hostName); + return vm; + } + + @Test + public void testDrainNodesToRemoveFailureLeavesClusterUnchanged() throws Exception { + Mockito.when(kubernetesCluster.getId()).thenReturn(10L); + worker.publicIpAddress = "10.0.0.1"; + worker.sshPort = 2222; + worker.scaleTimeoutTime = Long.MAX_VALUE; + KubernetesClusterScaleWorker spy = Mockito.spy(worker); + UserVmVO node1 = mockNodeVm("node-1"); + UserVmVO node2 = mockNodeVm("node-2"); + Mockito.doReturn(new Pair<>(true, "")).when(spy).executeSshCommand(Mockito.eq("10.0.0.1"), Mockito.eq(2222), + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(new Pair<>(true, "drained")).when(spy).drainKubernetesClusterNode(Mockito.eq("10.0.0.1"), Mockito.eq(2222), + Mockito.eq(node1), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(new Pair<>(false, "Cannot evict pod")).when(spy).drainKubernetesClusterNode(Mockito.eq("10.0.0.1"), Mockito.eq(2222), + Mockito.eq(node2), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(true).when(spy).uncordonKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any()); + Mockito.doReturn(true).when(spy).stateTransitTo(Mockito.anyLong(), Mockito.any()); + + try { + spy.drainNodesToRemove(Arrays.asList(node1, node2)); + Assert.fail("Expected the scale-down to stop"); + } catch (CloudRuntimeException e) { + Assert.assertTrue(e.getMessage().contains("node-2")); + Assert.assertTrue(e.getMessage().contains("Cannot evict pod")); + } + Mockito.verify(spy).uncordonKubernetesClusterNode("10.0.0.1", 2222, node1); + Mockito.verify(spy).uncordonKubernetesClusterNode("10.0.0.1", 2222, node2); + Mockito.verify(spy).stateTransitTo(10L, KubernetesCluster.Event.OperationSucceeded); + Mockito.verify(spy, Mockito.never()).stateTransitTo(10L, KubernetesCluster.Event.OperationFailed); + } + + @Test + public void testDrainNodesToRemoveCordonsAllBeforeDraining() throws Exception { + worker.publicIpAddress = "10.0.0.1"; + worker.sshPort = 2222; + worker.scaleTimeoutTime = Long.MAX_VALUE; + KubernetesClusterScaleWorker spy = Mockito.spy(worker); + UserVmVO node1 = mockNodeVm("node-1"); + UserVmVO node2 = mockNodeVm("node-2"); + Mockito.doReturn(new Pair<>(true, "")).when(spy).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(new Pair<>(true, "drained")).when(spy).drainKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), + Mockito.any(), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + + spy.drainNodesToRemove(Arrays.asList(node1, node2)); + + org.mockito.InOrder inOrder = Mockito.inOrder(spy); + inOrder.verify(spy).executeSshCommand("10.0.0.1", 2222, "sudo /opt/bin/kubectl cordon node-1", 10000, 10000, 30000); + inOrder.verify(spy).executeSshCommand("10.0.0.1", 2222, "sudo /opt/bin/kubectl cordon node-2", 10000, 10000, 30000); + inOrder.verify(spy).drainKubernetesClusterNode(Mockito.eq("10.0.0.1"), Mockito.eq(2222), Mockito.eq(node1), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + inOrder.verify(spy).drainKubernetesClusterNode(Mockito.eq("10.0.0.1"), Mockito.eq(2222), Mockito.eq(node2), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.verify(spy, Mockito.never()).uncordonKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any()); + Mockito.verify(spy, Mockito.never()).stateTransitTo(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void testDrainNodesToRemoveFailureKeepsAlertCluster() throws Exception { + Mockito.when(kubernetesCluster.getId()).thenReturn(10L); + Mockito.when(kubernetesCluster.getState()).thenReturn(KubernetesCluster.State.Alert); + KubernetesClusterScaleWorker alertWorker = new KubernetesClusterScaleWorker(kubernetesCluster, null, 1L, null, null, null, null, clusterManager); + alertWorker.publicIpAddress = "10.0.0.1"; + alertWorker.sshPort = 2222; + KubernetesClusterScaleWorker spy = Mockito.spy(alertWorker); + UserVmVO node1 = mockNodeVm("node-1"); + Mockito.doReturn(new Pair<>(false, "connection refused")).when(spy).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(true).when(spy).stateTransitTo(Mockito.anyLong(), Mockito.any()); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, () -> spy.drainNodesToRemove(List.of(node1))); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("unable to cordon Kubernetes node: node-1")); + Mockito.verify(spy).stateTransitTo(10L, KubernetesCluster.Event.OperationFailed); + Mockito.verify(spy, Mockito.never()).uncordonKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any()); + } + + @Test + public void testDrainNodesToRemoveStopsAtDeadline() throws Exception { + Mockito.when(kubernetesCluster.getId()).thenReturn(10L); + worker.publicIpAddress = "10.0.0.1"; + worker.sshPort = 2222; + worker.scaleTimeoutTime = System.currentTimeMillis() - 1; + KubernetesClusterScaleWorker spy = Mockito.spy(worker); + UserVmVO node1 = mockNodeVm("node-1"); + Mockito.doReturn(new Pair<>(true, "")).when(spy).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(true).when(spy).uncordonKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any()); + Mockito.doReturn(true).when(spy).stateTransitTo(Mockito.anyLong(), Mockito.any()); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, () -> spy.drainNodesToRemove(List.of(node1))); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + Mockito.verify(spy, Mockito.never()).drainKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.verify(spy).uncordonKubernetesClusterNode("10.0.0.1", 2222, node1); + Mockito.verify(spy).stateTransitTo(10L, KubernetesCluster.Event.OperationSucceeded); + } + + @Test + public void testDrainNodesToRemoveKeepsNodeCordonedBeforehand() throws Exception { + Mockito.when(kubernetesCluster.getId()).thenReturn(10L); + worker.publicIpAddress = "10.0.0.1"; + worker.sshPort = 2222; + worker.scaleTimeoutTime = Long.MAX_VALUE; + KubernetesClusterScaleWorker spy = Mockito.spy(worker); + UserVmVO node1 = mockNodeVm("node-1"); + UserVmVO node2 = mockNodeVm("node-2"); + Mockito.doReturn(new Pair<>(true, "")).when(spy).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(new Pair<>(true, "sudo: unable to resolve host node-1\ntrue")).when(spy).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), + Mockito.eq("sudo /opt/bin/kubectl get node node-1 -o jsonpath='{.spec.unschedulable}'"), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(new Pair<>(false, "Cannot evict pod")).when(spy).drainKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), + Mockito.any(), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(true).when(spy).uncordonKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any()); + Mockito.doReturn(true).when(spy).stateTransitTo(Mockito.anyLong(), Mockito.any()); + + Assert.assertThrows(CloudRuntimeException.class, () -> spy.drainNodesToRemove(Arrays.asList(node1, node2))); + Mockito.verify(spy, Mockito.never()).uncordonKubernetesClusterNode("10.0.0.1", 2222, node1); + Mockito.verify(spy).uncordonKubernetesClusterNode("10.0.0.1", 2222, node2); + } + + @Test + public void testDrainNodesToRemoveTakesUnknownCordonAsCordoned() throws Exception { + Mockito.when(kubernetesCluster.getId()).thenReturn(10L); + worker.publicIpAddress = "10.0.0.1"; + worker.sshPort = 2222; + worker.scaleTimeoutTime = Long.MAX_VALUE; + KubernetesClusterScaleWorker spy = Mockito.spy(worker); + UserVmVO node1 = mockNodeVm("node-1"); + Mockito.doReturn(new Pair<>(true, "")).when(spy).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doThrow(new RuntimeException("timed out")).when(spy).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), + Mockito.startsWith("sudo /opt/bin/kubectl get node"), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + Mockito.doReturn(new Pair<>(false, "Cannot evict pod")).when(spy).drainKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), + Mockito.any(), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(true).when(spy).stateTransitTo(Mockito.anyLong(), Mockito.any()); + + Assert.assertThrows(CloudRuntimeException.class, () -> spy.drainNodesToRemove(List.of(node1))); + Mockito.verify(spy).executeSshCommand("10.0.0.1", 2222, "sudo /opt/bin/kubectl cordon node-1", 10000, 10000, 30000); + Mockito.verify(spy, Mockito.never()).uncordonKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any()); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java index ff8f875d06e1..60a5853d9525 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java @@ -21,7 +21,9 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -80,4 +82,74 @@ public void testFilterOutManualUpgradeNodesFromClusterUpgrade() { Assert.assertTrue(ids.contains(controlNodeId) && ids.contains(workerNode1Id)); Assert.assertFalse(ids.contains(workerNode2Id)); } + + private static final String VERSION_COMMAND = String.format(KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, "node-1"); + private static final String READY_COMMAND = String.format(KubernetesClusterUpgradeWorker.NODE_STATUS_COMMAND, "node-1"); + + private KubernetesClusterUpgradeWorker mockNodeAtVersion(String storedVersion) { + Mockito.when(kubernetesSupportedVersion.getSemanticVersion()).thenReturn("1.29.2"); + KubernetesClusterVmMapVO vmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(vmMap.getNodeVersion()).thenReturn(storedVersion); + Mockito.when(kubernetesClusterVmMapDao.getClusterMapFromVmId(5L)).thenReturn(vmMap); + worker.publicIpAddress = "10.0.0.1"; + worker.sshPort = 2222; + return Mockito.spy(worker); + } + + private UserVm mockVm() { + UserVm vm = Mockito.mock(UserVm.class); + Mockito.when(vm.getId()).thenReturn(5L); + return vm; + } + + @Test + public void testIsNodeUpgradedWhenRecordedAndReported() throws Exception { + KubernetesClusterUpgradeWorker spy = mockNodeAtVersion("v1.29.2"); + Mockito.doReturn(new Pair<>(true, "v1.29.2\n")).when(spy).executeSshCommand("10.0.0.1", 2222, VERSION_COMMAND, 10000, 10000, 20000); + Mockito.doReturn(new Pair<>(true, "Ready\n")).when(spy).executeSshCommand("10.0.0.1", 2222, READY_COMMAND, 10000, 10000, 20000); + Assert.assertTrue(spy.isNodeUpgraded(mockVm(), "node-1")); + } + + @Test + public void testIsNodeUpgradedWhenCordonedAfterUpgrade() throws Exception { + KubernetesClusterUpgradeWorker spy = mockNodeAtVersion("v1.29.2"); + Mockito.doReturn(new Pair<>(true, "v1.29.2\n")).when(spy).executeSshCommand("10.0.0.1", 2222, VERSION_COMMAND, 10000, 10000, 20000); + Mockito.doReturn(new Pair<>(true, "Ready,SchedulingDisabled\n")).when(spy).executeSshCommand("10.0.0.1", 2222, READY_COMMAND, 10000, 10000, 20000); + Assert.assertTrue(spy.isNodeUpgraded(mockVm(), "node-1")); + } + + @Test + public void testIsNodeUpgradedNotWhenNodeNotReady() throws Exception { + KubernetesClusterUpgradeWorker spy = mockNodeAtVersion("v1.29.2"); + Mockito.doReturn(new Pair<>(true, "v1.29.2\n")).when(spy).executeSshCommand("10.0.0.1", 2222, VERSION_COMMAND, 10000, 10000, 20000); + Mockito.doReturn(new Pair<>(true, "NotReady")).when(spy).executeSshCommand("10.0.0.1", 2222, READY_COMMAND, 10000, 10000, 20000); + Assert.assertFalse(spy.isNodeUpgraded(mockVm(), "node-1")); + } + + @Test + public void testIsNodeUpgradedNotTrustingRecordedVersionAlone() throws Exception { + KubernetesClusterUpgradeWorker spy = mockNodeAtVersion("v1.29.2"); + Mockito.doReturn(new Pair<>(true, "v1.28.9")).when(spy).executeSshCommand("10.0.0.1", 2222, VERSION_COMMAND, 10000, 10000, 20000); + Assert.assertFalse(spy.isNodeUpgraded(mockVm(), "node-1")); + } + + @Test + public void testIsNodeUpgradedWhenNodeUnreachable() throws Exception { + KubernetesClusterUpgradeWorker spy = mockNodeAtVersion("1.29.2"); + Mockito.doThrow(new RuntimeException("connection refused")).when(spy).executeSshCommand("10.0.0.1", 2222, VERSION_COMMAND, 10000, 10000, 20000); + Assert.assertFalse(spy.isNodeUpgraded(mockVm(), "node-1")); + } + + @Test + public void testIsNodeUpgradedWhenRecordedAtOldVersion() throws Exception { + KubernetesClusterUpgradeWorker spy = mockNodeAtVersion("v1.28.9"); + Assert.assertFalse(spy.isNodeUpgraded(mockVm(), "node-1")); + Mockito.verify(spy, Mockito.never()).executeSshCommand(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()); + } + + @Test + public void testIsNodeUpgradedComparesWholeVersion() throws Exception { + KubernetesClusterUpgradeWorker spy = mockNodeAtVersion("v1.29.20"); + Assert.assertFalse(spy.isNodeUpgraded(mockVm(), "node-1")); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java index 329f9b0e42ad..bccf47f0bf35 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java @@ -46,6 +46,14 @@ public void testClusterNodeVersionMatches() { executeAndTestVersionMatch(true, v1233WorkerNodeOutput, false); + executeAndTestVersionMatch(true, "v1.24.0\n", true); + + executeAndTestVersionMatch(true, "v1.24.01", false); + + executeAndTestVersionMatch(true, "sudo: unable to resolve host node-1\nv1.24.0\n", true); + + executeAndTestVersionMatch(true, "v1.24.0+vendor.1", true); + executeAndTestVersionMatch(false, v1240WorkerNodeOutput, false); executeAndTestVersionMatch(false, v1233WorkerNodeOutput, false); diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 7c2c9f09eae4..b3beeca4309c 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -842,7 +842,7 @@ export default { message: 'message.kubernetes.cluster.upgrade', docHelp: 'plugins/cloudstack-kubernetes-service.html#upgrading-kubernetes-cluster', dataView: true, - show: (record) => { return ['Created', 'Running'].includes(record.state) && record.clustertype === 'CloudManaged' }, + show: (record) => { return ['Created', 'Running', 'Alert'].includes(record.state) && record.clustertype === 'CloudManaged' }, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/compute/UpgradeKubernetesCluster.vue'))) }, From b682b1cd846f94bf130bd33c7283b1156c833343 Mon Sep 17 00:00:00 2001 From: Brad House Date: Sun, 8 Nov 2026 02:55:17 +0000 Subject: [PATCH 3/3] CKS: restart the nodes of a cluster with new node settings, a few at a time A CloudManaged Kubernetes cluster could get a new service offering only through a scale, which changes every node at once with no drain, its affinity groups only while stopped, and its root disk size not at all. updateKubernetesCluster sets, for each node type (control, worker, etcd), the service offering, the root disk size and the affinity groups. The nodes created from then on get them. A cluster created before its node types had settings of their own first gets the service offering of the cluster copied to them; templates are left as they are. With rollout=true, the nodes that differ from the settings of their type are restarted with them: the external etcd nodes, then the control nodes, one at a time, then the worker nodes, maxunavailable at a time (1 by default, and always leaving a worker running). For each batch: - every other node must be running and Ready, and a control or etcd node is only taken down if the others keep a quorum - each Kubernetes node is annotated with its boot ID, unless someone else cordoned it, then cordoned and drained - its VM is stopped, given the offering, a larger root disk and the affinity groups, and started, which places it by its new groups; cloud-init grows the root partition and file system at boot - once it is Ready with another boot ID, and etcd is healthy, it is uncordoned and the annotation removed A control or etcd node type with too few nodes to lose one (a single control node, a 2-node stacked control plane, 2 etcd nodes) can not be rolled: a rollout=true that would make one of its nodes differ from the settings is refused, and its nodes that differ anyway are left out and reported in lasterror, while the other types are still rolled. Nodes are restarted rather than changed live, which CloudStack can not do reliably for a KVM VM, and which would leave the guest and kubelet unaware of the new resources. Nothing else is recorded: the nodes that still differ, are stopped or carry the annotation are the work left, and go first when the rollout is run again, which is how one that failed is resumed. However a rollout ends, it uncordons the nodes that are back, leaving the others cordoned, counts the CPU cores and memory of the cluster from its VMs, and moves the cluster from the new Updating state to Running, or to Alert if it is not healthy. Why it stopped is shown as lasterror. cancelrollout=true stops a rollout between two batches. Other operations are refused while a cluster is Updating, and the state scanner moves a cluster left Updating by a management server that stopped to Alert. The response also shows the number of nodes of each type that differ from the settings of their type. Signed-off-by: Brad House --- .../kubernetes/cluster/KubernetesCluster.java | 11 + .../apache/cloudstack/api/ApiConstants.java | 10 + .../META-INF/db/schema-42300to2400.sql | 5 + .../cluster/KubernetesClusterEventTypes.java | 1 + .../cluster/KubernetesClusterManagerImpl.java | 439 +++++++++- .../cluster/KubernetesClusterService.java | 10 + .../cluster/KubernetesClusterVO.java | 36 + .../KubernetesClusterActionWorker.java | 7 + ...esClusterResourceModifierActionWorker.java | 19 +- .../KubernetesClusterRollingUpdateWorker.java | 790 ++++++++++++++++++ .../KubernetesClusterStartWorker.java | 32 +- .../KubernetesClusterAffinityGroupMapDao.java | 2 + ...ernetesClusterAffinityGroupMapDaoImpl.java | 8 + .../cluster/utils/KubernetesClusterUtil.java | 35 + .../cluster/UpdateKubernetesClusterCmd.java | 204 +++++ .../response/KubernetesClusterResponse.java | 56 ++ ...ubernetesClusterManagerImplUpdateTest.java | 573 +++++++++++++ ...ernetesClusterRollingUpdateWorkerTest.java | 737 ++++++++++++++++ .../UpdateKubernetesClusterCmdTest.java | 59 ++ 19 files changed, 3009 insertions(+), 25 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRollingUpdateWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpdateKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplUpdateTest.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRollingUpdateWorkerTest.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpdateKubernetesClusterCmdTest.java diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 2be99f3df589..304296af4123 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -47,6 +47,7 @@ enum Event { AddNodeRequested, RemoveNodeRequested, UpgradeRequested, + UpdateRequested, OperationSucceeded, OperationFailed, CreateFailed, @@ -62,6 +63,7 @@ enum State { Scaling("Transient state in which resources are either getting scaled up/down"), ScalingStoppedCluster("Transient state in which the service offerings of stopped clusters are getting scaled"), Upgrading("Transient state in which cluster is getting upgraded"), + Updating("Transient state in which the nodes of the cluster are restarted with their stored settings, a few at a time"), Importing("Transient state in which additional nodes are added as worker nodes to a cluster"), RemovingNodes("Transient state in which additional nodes are removed from a cluster"), Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."), @@ -108,6 +110,12 @@ enum State { s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert); + // a rollout ends in Running if the cluster is healthy, and in Alert otherwise + s_fsm.addTransition(State.Running, Event.UpdateRequested, State.Updating); + s_fsm.addTransition(State.Alert, Event.UpdateRequested, State.Updating); + s_fsm.addTransition(State.Updating, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.Updating, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing); s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing); s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running); @@ -155,6 +163,9 @@ enum State { long getCores(); long getMemory(); long getNodeRootDiskSize(); + Long getControlNodeRootDiskSize(); + Long getWorkerNodeRootDiskSize(); + Long getEtcdNodeRootDiskSize(); String getEndpoint(); boolean isCheckForGc(); @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 7485029f22cb..11c22708de5c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1324,6 +1324,16 @@ public class ApiConstants { public static final String NODE_TYPE_OFFERING_MAP = "nodeofferings"; public static final String NODE_TYPE_TEMPLATE_MAP = "nodetemplates"; public static final String NODE_TYPE_AFFINITY_GROUP_MAP = "nodeaffinitygroups"; + public static final String NODE_TYPE_ROOT_DISK_SIZE_MAP = "noderootdisksizes"; + public static final String CONTROL_ROOT_DISK_SIZE = "controlrootdisksize"; + public static final String WORKER_ROOT_DISK_SIZE = "workerrootdisksize"; + public static final String ETCD_ROOT_DISK_SIZE = "etcdrootdisksize"; + public static final String CONTROL_NODES_OUT_OF_DATE = "controlnodesoutofdate"; + public static final String WORKER_NODES_OUT_OF_DATE = "workernodesoutofdate"; + public static final String ETCD_NODES_OUT_OF_DATE = "etcdnodesoutofdate"; + public static final String ROLLOUT = "rollout"; + public static final String CANCEL_ROLLOUT = "cancelrollout"; + public static final String MAX_UNAVAILABLE = "maxunavailable"; public static final String CONTROL_AFFINITY_GROUP_IDS = "controlaffinitygroupids"; public static final String CONTROL_AFFINITY_GROUP_NAMES = "controlaffinitygroupnames"; public static final String WORKER_AFFINITY_GROUP_IDS = "workeraffinitygroupids"; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 3365f1f855ba..f1d60e42e2f3 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -239,3 +239,8 @@ CREATE TABLE IF NOT EXISTS `cloud`.`instance_boot_group_details` ( PRIMARY KEY (`id`), CONSTRAINT `fk_instance_boot_group_details__group_id` FOREIGN KEY (`boot_group_id`) REFERENCES `instance_boot_group` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- CKS: root disk size per node type. When not set, the cluster wide node_root_disk_size is used, as before. +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'control_node_root_disk_size', 'bigint unsigned DEFAULT NULL COMMENT "root disk size in GB of the control nodes, node_root_disk_size when not set"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'worker_node_root_disk_size', 'bigint unsigned DEFAULT NULL COMMENT "root disk size in GB of the worker nodes, node_root_disk_size when not set"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'etcd_node_root_disk_size', 'bigint unsigned DEFAULT NULL COMMENT "root disk size in GB of the etcd nodes, the size of the template when not set"'); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java index d11a0fc3dea5..6358f0ea5265 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java @@ -26,4 +26,5 @@ public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD"; public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE"; public static final String EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE = "KUBERNETES.CLUSTER.AFFINITY.UPDATE"; + public static final String EVENT_KUBERNETES_CLUSTER_UPDATE = "KUBERNETES.CLUSTER.UPDATE"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index b6e2ccfda628..8c66cde8dd43 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -32,6 +32,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -89,6 +90,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpdateKubernetesClusterAffinityGroupCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpdateKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.loadbalancer.AssignToLoadBalancerRuleCmd; import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd; @@ -171,6 +173,7 @@ import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterDestroyWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRollingUpdateWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterScaleWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStartWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStopWorker; @@ -223,8 +226,13 @@ import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; import com.cloud.template.TemplateApiService; import com.cloud.user.Account; import com.cloud.user.AccountManager; @@ -363,6 +371,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected VMInstanceDao vmInstanceDao; @Inject + protected VolumeDao volumeDao; + @Inject + protected DiskOfferingDao diskOfferingDao; + @Inject protected AsyncJobDao asyncJobDao; @Inject protected UserVmJoinDao userVmJoinDao; @@ -876,6 +888,17 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes setNodeTypeServiceOfferingResponse(response, WORKER, kubernetesCluster.getWorkerNodeServiceOfferingId()); setNodeTypeServiceOfferingResponse(response, CONTROL, kubernetesCluster.getControlNodeServiceOfferingId()); setNodeTypeServiceOfferingResponse(response, ETCD, kubernetesCluster.getEtcdNodeServiceOfferingId()); + response.setControlNodeRootDiskSize(kubernetesCluster.getControlNodeRootDiskSize()); + response.setWorkerNodeRootDiskSize(kubernetesCluster.getWorkerNodeRootDiskSize()); + response.setEtcdNodeRootDiskSize(kubernetesCluster.getEtcdNodeRootDiskSize()); + if (KubernetesCluster.ClusterType.CloudManaged.equals(kubernetesCluster.getClusterType())) { + Map outOfDate = countOutOfDateNodes(kubernetesCluster); + response.setControlNodesOutOfDate(outOfDate.get(CONTROL)); + response.setWorkerNodesOutOfDate(outOfDate.get(WORKER)); + response.setEtcdNodesOutOfDate(outOfDate.get(ETCD)); + KubernetesClusterDetailsVO lastError = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ROLLOUT_LAST_ERROR_DETAIL); + response.setLastError(lastError == null ? null : lastError.getValue()); + } if (kubernetesCluster.getEtcdNodeCount() != null) { response.setEtcdNodes(kubernetesCluster.getEtcdNodeCount()); @@ -1084,7 +1107,8 @@ public boolean isCommandSupported(KubernetesCluster cluster, String cmdName) { BaseCmd.getCommandNameByClass(StopKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(AddNodesToKubernetesClusterCmd.class), - BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class) + BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(UpdateKubernetesClusterCmd.class) ).contains(cmdName); case ExternalManaged: return Arrays.asList( @@ -2485,6 +2509,410 @@ private String getNodeType(KubernetesClusterVmMapVO clusterVmMapping) { return WORKER.name(); } + public static final String ROLLOUT_CANCEL_DETAIL = "rollout.cancel"; + public static final String ROLLOUT_LAST_ERROR_DETAIL = "rollout.lasterror"; + protected static final long MIN_NODE_ROOT_DISK_SIZE = 1L; + /** The largest size in GB whose size in bytes fits in a long */ + protected static final long MAX_NODE_ROOT_DISK_SIZE = Long.MAX_VALUE >> 30; + protected static final String EXPLICIT_DEDICATION_AFFINITY_TYPE = "ExplicitDedication"; + + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_UPDATE, + eventDescription = "updating Kubernetes cluster", async = true) + public boolean updateKubernetesCluster(UpdateKubernetesClusterCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + if (!isCommandSupported(kubernetesCluster, cmd.getActualCommandName())) { + throw new InvalidParameterValueException(String.format("Update of Kubernetes cluster is not supported for an externally managed cluster (%s)", kubernetesCluster.getName())); + } + if (cmd.isCancelRollout()) { + return cancelRollout(cmd, kubernetesCluster); + } + validateKubernetesClusterUpdateParameters(cmd, kubernetesCluster); + if (cmd.isRollout() && !KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) && !KubernetesCluster.State.Alert.equals(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("The nodes of Kubernetes cluster %s can only be restarted while it is Running or Alert, it is %s. " + + "Use rollout=false to only store the new settings", kubernetesCluster.getName(), kubernetesCluster.getState())); + } + // a rollout first takes the cluster to Updating, so that no other operation works from the settings it stores + final KubernetesClusterRollingUpdateWorker worker = cmd.isRollout() + ? createRollingUpdateWorker(kubernetesCluster, cmd.getMaxUnavailable() == null ? 1 : cmd.getMaxUnavailable()) : null; + if (worker != null) { + worker.begin(); + } + try { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + KubernetesClusterVO cluster = kubernetesClusterDao.lockRow(kubernetesCluster.getId(), true); + // other operations create nodes from the settings + if (worker == null && (OPERATION_STATES.contains(cluster.getState()) || KubernetesCluster.State.Destroying.equals(cluster.getState()) + || KubernetesCluster.State.Destroyed.equals(cluster.getState()))) { + throw new InvalidParameterValueException(String.format("The node settings of Kubernetes cluster %s can not be changed while it is %s", + cluster.getName(), cluster.getState())); + } + Set unrollable = worker == null ? Set.of() : getUnrollableNodeTypes(cluster); + Set outOfDateBefore = unrollable.isEmpty() ? Set.of() : getOutOfDateNodeIds(cluster, unrollable); + convertToNodeTypeSettings(cluster); + storeNodeTypeSettings(cluster, cmd); + if (!unrollable.isEmpty()) { + checkUnrollableNodeTypes(cluster, unrollable, outOfDateBefore); + } + } + }); + } catch (RuntimeException e) { + if (worker != null) { + try { + worker.abort(); + } catch (RuntimeException releaseError) { + e.addSuppressed(releaseError); + } + } + throw e; + } + if (worker == null) { + logger.info("Stored the node settings of Kubernetes cluster {}, without restarting its nodes", kubernetesCluster); + return true; + } + return worker.rollout(); + } + + protected KubernetesClusterRollingUpdateWorker createRollingUpdateWorker(KubernetesCluster cluster, int maxUnavailable) { + KubernetesClusterRollingUpdateWorker worker = new KubernetesClusterRollingUpdateWorker(cluster, maxUnavailable, this); + return ComponentContext.inject(worker); + } + + /** Asks the rollout in progress to stop between two batches of nodes */ + protected boolean cancelRollout(UpdateKubernetesClusterCmd cmd, KubernetesClusterVO cluster) { + if (cmd.isRollout() || cmd.getMaxUnavailable() != null || MapUtils.isNotEmpty(cmd.getServiceOfferingNodeTypeMap()) + || MapUtils.isNotEmpty(cmd.getRootDiskSizeNodeTypeMap()) || MapUtils.isNotEmpty(cmd.getAffinityGroupNodeTypeMap())) { + throw new InvalidParameterValueException(String.format("%s takes no other parameter", ApiConstants.CANCEL_ROLLOUT)); + } + if (!KubernetesCluster.State.Updating.equals(cluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s has no rollout in progress, it is %s", cluster.getName(), cluster.getState())); + } + kubernetesClusterDetailsDao.addDetail(cluster.getId(), ROLLOUT_CANCEL_DETAIL, "true", false); + logger.info("Asked the rollout of Kubernetes cluster {} to stop", cluster); + return true; + } + + protected void validateKubernetesClusterUpdateParameters(UpdateKubernetesClusterCmd cmd, KubernetesClusterVO kubernetesCluster) { + final Map offerings = cmd.getServiceOfferingNodeTypeMap(); + final Map rootDiskSizes = cmd.getRootDiskSizeNodeTypeMap(); + final Map> affinityGroups = cmd.getAffinityGroupNodeTypeMap(); + if (!cmd.isRollout() && MapUtils.isEmpty(offerings) && MapUtils.isEmpty(rootDiskSizes) && MapUtils.isEmpty(affinityGroups)) { + throw new InvalidParameterValueException(String.format("Nothing to update for Kubernetes cluster %s: pass %s, %s, %s or %s=true", kubernetesCluster.getName(), + ApiConstants.NODE_TYPE_OFFERING_MAP, ApiConstants.NODE_TYPE_ROOT_DISK_SIZE_MAP, ApiConstants.NODE_TYPE_AFFINITY_GROUP_MAP, ApiConstants.ROLLOUT)); + } + final List vmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (cmd.getMaxUnavailable() != null) { + if (!cmd.isRollout()) { + throw new InvalidParameterValueException(String.format("%s can only be passed with %s=true", ApiConstants.MAX_UNAVAILABLE, ApiConstants.ROLLOUT)); + } + long workers = KubernetesClusterUtil.countManagedNodes(vmMaps, WORKER); + if (cmd.getMaxUnavailable() < 1 || cmd.getMaxUnavailable() > 1 && cmd.getMaxUnavailable() >= workers) { + throw new InvalidParameterValueException(String.format("%s must be 1, or more and less than the %d worker nodes of Kubernetes cluster %s, " + + "for some to keep running the workloads", ApiConstants.MAX_UNAVAILABLE, workers, kubernetesCluster.getName())); + } + } + final boolean hasEtcdNodes = kubernetesCluster.getEtcdNodeCount() != null && kubernetesCluster.getEtcdNodeCount() > 0; + if (MapUtils.isNotEmpty(offerings)) { + final KubernetesSupportedVersion clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (clusterVersion == null) { + throw new CloudRuntimeException(String.format("Invalid Kubernetes version associated with Kubernetes cluster : %s", kubernetesCluster.getName())); + } + for (Map.Entry entry : offerings.entrySet()) { + checkUpdatableNodeType(entry.getKey(), hasEtcdNodes, ApiConstants.NODE_TYPE_OFFERING_MAP); + ServiceOffering offering = serviceOfferingDao.findById(entry.getValue()); + if (offering == null) { + throw new InvalidParameterValueException("Unable to find the service offering with ID " + entry.getValue()); + } + checkServiceOfferingForNodesScale(offering, kubernetesCluster, clusterVersion); + } + } + for (Map.Entry entry : Objects.requireNonNullElse(rootDiskSizes, Map.of()).entrySet()) { + KubernetesClusterNodeType nodeType = checkUpdatableNodeType(entry.getKey(), hasEtcdNodes, ApiConstants.NODE_TYPE_ROOT_DISK_SIZE_MAP); + if (entry.getValue() == null || entry.getValue() < MIN_NODE_ROOT_DISK_SIZE || entry.getValue() > MAX_NODE_ROOT_DISK_SIZE) { + throw new InvalidParameterValueException(String.format("The root disk size of the %s nodes must be from %d to %d GB", nodeType.name().toLowerCase(), + MIN_NODE_ROOT_DISK_SIZE, MAX_NODE_ROOT_DISK_SIZE)); + } + if (cmd.isRollout()) { + checkRootDiskSizeRollout(vmMaps, nodeType, entry.getValue()); + } + } + if (MapUtils.isNotEmpty(affinityGroups)) { + validateNodeAffinityGroups(affinityGroups, kubernetesCluster.getAccountId()); + for (Map.Entry> entry : affinityGroups.entrySet()) { + checkUpdatableNodeType(entry.getKey(), hasEtcdNodes, ApiConstants.NODE_TYPE_AFFINITY_GROUP_MAP); + if (!filterAffinityGroupIds(entry.getValue(), true).isEmpty()) { + throw new InvalidParameterValueException(String.format("%s takes no explicit dedication group: CKS gives the nodes of a dedicated domain " + + "or account theirs", ApiConstants.NODE_TYPE_AFFINITY_GROUP_MAP)); + } + } + } + } + + /** The control and etcd node types with too few nodes to restart one at a time */ + protected Set getUnrollableNodeTypes(KubernetesClusterVO cluster) { + List vmMaps = kubernetesClusterVmMapDao.listByClusterId(cluster.getId()); + boolean hasEtcdNodes = cluster.getEtcdNodeCount() != null && cluster.getEtcdNodeCount() > 0; + Set types = new HashSet<>(); + for (KubernetesClusterNodeType nodeType : List.of(CONTROL, ETCD)) { + int members = KubernetesClusterUtil.countManagedNodes(vmMaps, nodeType); + if (members > 0 && !KubernetesClusterRollingUpdateWorker.canRestartOneAtATime(nodeType, members, hasEtcdNodes)) { + types.add(nodeType); + } + } + return types; + } + + /** + * Refuses settings that make a node of a type that can not be restarted one at a time differ from them, as a + * rollout would have to restart it for them. Run once the settings are stored, before they are committed. A node + * that differed already, or whose VM is not running, is left to the rollout, which skips its type and reports it. + */ + protected void checkUnrollableNodeTypes(KubernetesClusterVO cluster, Set unrollable, Set outOfDateBefore) { + List vmMaps = kubernetesClusterVmMapDao.listByClusterId(cluster.getId()); + for (KubernetesClusterNodeType nodeType : unrollable) { + int members = KubernetesClusterUtil.countManagedNodes(vmMaps, nodeType); + for (KubernetesClusterVmMapVO vmMap : vmMaps) { + VMInstanceVO vm = vmMap.isExternalNode() || KubernetesClusterUtil.getNodeType(vmMap) != nodeType ? null : vmInstanceDao.findById(vmMap.getVmId()); + if (vm != null && VirtualMachine.State.Running.equals(vm.getState()) && !outOfDateBefore.contains(vm.getId()) + && getNodeOutOfDateReason(cluster, vmMap, vm) != null) { + throw new InvalidParameterValueException(String.format("The %d %s node(s) of Kubernetes cluster %s can not be restarted one at a time, as the " + + "cluster can not keep running without one, so a rollout can not change their settings. Use rollout=false to store them " + + "without restarting the nodes", members, nodeType.name().toLowerCase(), cluster.getName())); + } + } + } + } + + protected Set getOutOfDateNodeIds(KubernetesClusterVO cluster, Set nodeTypes) { + Set ids = new HashSet<>(); + for (KubernetesClusterVmMapVO vmMap : kubernetesClusterVmMapDao.listByClusterId(cluster.getId())) { + VMInstanceVO vm = vmMap.isExternalNode() || !nodeTypes.contains(KubernetesClusterUtil.getNodeType(vmMap)) ? null : vmInstanceDao.findById(vmMap.getVmId()); + if (vm != null && getNodeOutOfDateReason(cluster, vmMap, vm) != null) { + ids.add(vm.getId()); + } + } + return ids; + } + + protected KubernetesClusterNodeType checkUpdatableNodeType(String nodeTypeName, boolean hasEtcdNodes, String parameter) { + KubernetesClusterNodeType nodeType = KubernetesClusterNodeType.valueOf(nodeTypeName.toUpperCase()); + if (DEFAULT.equals(nodeType)) { + throw new InvalidParameterValueException(String.format("%s takes the node types control, worker and etcd", parameter)); + } + if (ETCD.equals(nodeType) && !hasEtcdNodes) { + throw new InvalidParameterValueException(String.format("%s: the cluster has no etcd nodes", parameter)); + } + return nodeType; + } + + /** Refuses a rollout of a root disk size that would shrink the root disk of a node, or that its disk offering does not allow */ + protected void checkRootDiskSizeRollout(List vmMaps, KubernetesClusterNodeType nodeType, long sizeGb) { + for (KubernetesClusterVmMapVO vmMap : vmMaps) { + if (vmMap.isExternalNode() || KubernetesClusterUtil.getNodeType(vmMap) != nodeType) { + continue; + } + List roots = volumeDao.findByInstanceAndType(vmMap.getVmId(), Volume.Type.ROOT); + if (CollectionUtils.isEmpty(roots) || roots.get(0).getSize() == null) { + continue; + } + VMInstanceVO vm = vmInstanceDao.findById(vmMap.getVmId()); + String name = vm == null ? String.valueOf(vmMap.getVmId()) : vm.getHostName(); + if (roots.get(0).getSize() > (sizeGb << 30)) { + throw new InvalidParameterValueException(String.format("The root disk of node %s is larger than %d GB, and a rollout only grows root disks. " + + "Use rollout=false to only store the size for new nodes", name, sizeGb)); + } + DiskOfferingVO diskOffering = roots.get(0).getSize() < (sizeGb << 30) ? diskOfferingDao.findById(roots.get(0).getDiskOfferingId()) : null; + if (diskOffering != null && diskOffering.getDiskSizeStrictness()) { + throw new InvalidParameterValueException(String.format("The root disk of node %s can not be grown, as the size of its disk offering %s is strictly fixed", + name, diskOffering.getName())); + } + } + } + + protected void storeNodeTypeSettings(KubernetesClusterVO cluster, UpdateKubernetesClusterCmd cmd) { + for (Map.Entry entry : Objects.requireNonNullElse(cmd.getServiceOfferingNodeTypeMap(), Map.of()).entrySet()) { + setNodeTypeServiceOfferingId(cluster, KubernetesClusterNodeType.valueOf(entry.getKey().toUpperCase()), entry.getValue()); + } + for (Map.Entry entry : Objects.requireNonNullElse(cmd.getRootDiskSizeNodeTypeMap(), Map.of()).entrySet()) { + setNodeTypeRootDiskSize(cluster, KubernetesClusterNodeType.valueOf(entry.getKey().toUpperCase()), entry.getValue()); + } + for (Map.Entry> entry : Objects.requireNonNullElse(cmd.getAffinityGroupNodeTypeMap(), Map.>of()).entrySet()) { + String nodeType = entry.getKey().toUpperCase(); + kubernetesClusterAffinityGroupMapDao.removeByClusterIdAndNodeType(cluster.getId(), nodeType); + persistAffinityGroupMappings(cluster.getId(), Map.of(nodeType, new ArrayList<>(new LinkedHashSet<>(entry.getValue())))); + } + kubernetesClusterDao.update(cluster.getId(), cluster); + } + + protected void setNodeTypeServiceOfferingId(KubernetesClusterVO cluster, KubernetesClusterNodeType nodeType, Long serviceOfferingId) { + if (CONTROL == nodeType) { + cluster.setControlNodeServiceOfferingId(serviceOfferingId); + } else if (WORKER == nodeType) { + cluster.setWorkerNodeServiceOfferingId(serviceOfferingId); + } else if (ETCD == nodeType) { + cluster.setEtcdNodeServiceOfferingId(serviceOfferingId); + } + } + + protected void setNodeTypeRootDiskSize(KubernetesClusterVO cluster, KubernetesClusterNodeType nodeType, Long rootDiskSize) { + if (CONTROL == nodeType) { + cluster.setControlNodeRootDiskSize(rootDiskSize); + } else if (WORKER == nodeType) { + cluster.setWorkerNodeRootDiskSize(rootDiskSize); + } else if (ETCD == nodeType) { + cluster.setEtcdNodeRootDiskSize(rootDiskSize); + } + } + + /** + * A cluster created before its node types had settings of their own has a service offering only for the whole + * cluster. It is copied to the node types, so that a change to one node type keeps the others as they are. The + * template is left as it is, as this API does not change templates. A node type with no affinity groups of its + * own gets those its nodes all have, if any, so that a rollout does not take them away. Nothing changes for node + * types that have their settings already. + */ + protected void convertToNodeTypeSettings(KubernetesClusterVO cluster) { + boolean hasEtcdNodes = cluster.getEtcdNodeCount() != null && cluster.getEtcdNodeCount() > 0; + for (KubernetesClusterNodeType nodeType : hasEtcdNodes ? List.of(CONTROL, WORKER, ETCD) : List.of(CONTROL, WORKER)) { + if (getNodeTypeServiceOfferingIdIfSet(cluster, nodeType) == null) { + setNodeTypeServiceOfferingId(cluster, nodeType, cluster.getServiceOfferingId()); + logger.info("Copied the service offering of Kubernetes cluster {} to its {} nodes", cluster, nodeType.name().toLowerCase()); + } + if (CollectionUtils.isNotEmpty(getNodeTypeAffinityGroupIds(cluster.getId(), nodeType))) { + continue; + } + Set common = null; + for (KubernetesClusterVmMapVO vmMap : kubernetesClusterVmMapDao.listByClusterId(cluster.getId())) { + if (vmMap.isExternalNode() || KubernetesClusterUtil.getNodeType(vmMap) != nodeType) { + continue; + } + Set groups = getVmAffinityGroupIds(vmMap.getVmId(), false); + if (common != null && !common.equals(groups)) { + common = null; + break; + } + common = groups; + } + if (CollectionUtils.isNotEmpty(common)) { + persistAffinityGroupMappings(cluster.getId(), Map.of(nodeType.name(), new ArrayList<>(common))); + logger.info("Recorded the affinity groups {} of the {} nodes of Kubernetes cluster {} as those of their type", common, nodeType.name().toLowerCase(), cluster); + } + } + kubernetesClusterDao.update(cluster.getId(), cluster); + } + + protected Long getNodeTypeServiceOfferingIdIfSet(KubernetesCluster cluster, KubernetesClusterNodeType nodeType) { + if (CONTROL == nodeType) { + return cluster.getControlNodeServiceOfferingId(); + } + return WORKER == nodeType ? cluster.getWorkerNodeServiceOfferingId() : cluster.getEtcdNodeServiceOfferingId(); + } + + /** The service offering the nodes of a type get: their own, or else the one of the cluster */ + public Long getNodeTypeServiceOfferingId(KubernetesCluster cluster, KubernetesClusterNodeType nodeType) { + Long offeringId = getNodeTypeServiceOfferingIdIfSet(cluster, nodeType); + return offeringId != null ? offeringId : cluster.getServiceOfferingId(); + } + + public List getNodeTypeAffinityGroupIds(long clusterId, KubernetesClusterNodeType nodeType) { + return new ArrayList<>(Objects.requireNonNullElse(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, nodeType.name()), + List.of())); + } + + /** + * The affinity groups, other than explicit dedication, that the nodes of a type are to have, or null if the type + * has none of those recorded, in which case the nodes keep theirs: its nodes may have differed when it got its + * settings. CKS gives the nodes their explicit dedication group on its own. + */ + public Set getNodeTypeWantedAffinityGroupIds(long clusterId, KubernetesClusterNodeType nodeType) { + Set groupIds = filterAffinityGroupIds(getNodeTypeAffinityGroupIds(clusterId, nodeType), false); + return groupIds.isEmpty() ? null : groupIds; + } + + /** + * The affinity groups of a VM: either the explicit dedication groups, which CKS gives the nodes of a dedicated + * domain or account on its own, or the others + */ + public Set getVmAffinityGroupIds(long vmId, boolean explicitDedication) { + return filterAffinityGroupIds(affinityGroupVMMapDao.listAffinityGroupIdsByVmId(vmId), explicitDedication); + } + + protected Set filterAffinityGroupIds(List ids, boolean explicitDedication) { + Set groupIds = new LinkedHashSet<>(); + for (Long groupId : Objects.requireNonNullElse(ids, List.of())) { + AffinityGroupVO group = affinityGroupDao.findById(groupId); + if (group != null && EXPLICIT_DEDICATION_AFFINITY_TYPE.equals(group.getType()) == explicitDedication) { + groupIds.add(groupId); + } + } + return groupIds; + } + + /** + * Why a node differs from the settings of its node type, or null if it does not: its service offering, a root disk + * smaller than the size of the type (a rollout only grows disks), or its affinity groups + */ + public String getNodeOutOfDateReason(KubernetesCluster cluster, KubernetesClusterVmMapVO vmMap, VMInstanceVO vm) { + KubernetesClusterNodeType nodeType = KubernetesClusterUtil.getNodeType(vmMap); + Long offeringId = getNodeTypeServiceOfferingId(cluster, nodeType); + if (offeringId != null && offeringId != vm.getServiceOfferingId()) { + return "its service offering differs"; + } + long sizeGb = KubernetesClusterUtil.getNodeRootDiskSize(cluster, nodeType); + if (sizeGb > 0) { + List roots = volumeDao.findByInstanceAndType(vm.getId(), Volume.Type.ROOT); + if (CollectionUtils.isNotEmpty(roots) && roots.get(0).getSize() != null && roots.get(0).getSize() < (sizeGb << 30)) { + return String.format("its root disk is smaller than %d GB", sizeGb); + } + } + Set wanted = getNodeTypeWantedAffinityGroupIds(cluster.getId(), nodeType); + if (wanted != null && !wanted.equals(getVmAffinityGroupIds(vm.getId(), false))) { + return "its affinity groups differ"; + } + return null; + } + + protected Map countOutOfDateNodes(KubernetesCluster cluster) { + Map counts = new HashMap<>(Map.of(CONTROL, 0, WORKER, 0)); + if (cluster.getEtcdNodeCount() != null && cluster.getEtcdNodeCount() > 0) { + counts.put(ETCD, 0); + } + for (KubernetesClusterVmMapVO vmMap : kubernetesClusterVmMapDao.listByClusterId(cluster.getId())) { + VMInstanceVO vm = vmMap.isExternalNode() ? null : vmInstanceDao.findById(vmMap.getVmId()); + if (vm != null && getNodeOutOfDateReason(cluster, vmMap, vm) != null) { + counts.merge(KubernetesClusterUtil.getNodeType(vmMap), 1, Integer::sum); + } + } + return counts; + } + + /** Counts the CPU cores and memory of a cluster from the service offerings its VMs have */ + public void updateKubernetesClusterCapacity(long clusterId) { + long cores = 0; + long memory = 0; + for (KubernetesClusterVmMapVO vmMap : kubernetesClusterVmMapDao.listByClusterId(clusterId)) { + VMInstanceVO vm = vmInstanceDao.findById(vmMap.getVmId()); + ServiceOffering offering = vm == null ? null : serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()); + if (offering != null) { + cores += Objects.requireNonNullElse(offering.getCpu(), 0); + memory += Objects.requireNonNullElse(offering.getRamSize(), 0); + } + } + KubernetesClusterVO update = kubernetesClusterDao.createForUpdate(clusterId); + update.setCores(cores); + update.setMemory(memory); + kubernetesClusterDao.update(clusterId, update); + } + private void updateNodeCount(KubernetesClusterVO kubernetesCluster) { List nodeList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); kubernetesCluster.setControlNodeCount(nodeList.stream().filter(KubernetesClusterVmMapVO::isControlNode).count()); @@ -2841,6 +3269,7 @@ public List> getCommands() { cmdList.add(AddNodesToKubernetesClusterCmd.class); cmdList.add(RemoveNodesFromKubernetesClusterCmd.class); cmdList.add(UpdateKubernetesClusterAffinityGroupCmd.class); + cmdList.add(UpdateKubernetesClusterCmd.class); return cmdList; } @@ -3011,7 +3440,7 @@ public void reallyRun() { protected static final List OPERATION_STATES = List.of(KubernetesCluster.State.Starting, KubernetesCluster.State.Stopping, KubernetesCluster.State.Scaling, KubernetesCluster.State.ScalingStoppedCluster, KubernetesCluster.State.Upgrading, KubernetesCluster.State.Importing, KubernetesCluster.State.RemovingNodes, - KubernetesCluster.State.Recovering); + KubernetesCluster.State.Recovering, KubernetesCluster.State.Updating); /** * Fails the operation of each cluster in an operation's state which has no job in progress on any management @@ -3061,6 +3490,11 @@ protected void recoverOrphanedOperationState(long clusterId) throws NoTransition final KubernetesCluster.State state = cluster.getState(); if (_stateMachine.transitTo(cluster, KubernetesCluster.Event.OperationFailed, null, kubernetesClusterDao)) { logger.warn("Kubernetes cluster {} was in state {} with no operation in progress, as its operation ended without moving it on, for example because the management server running it stopped; marked the operation as failed", cluster, state); + if (KubernetesCluster.State.Updating.equals(state)) { + // a rollout may have changed the offering of some nodes. Its nodes are left as they are: the next + // rollout takes up the ones it stopped or cordoned first + updateKubernetesClusterCapacity(clusterId); + } } } @@ -3191,6 +3625,7 @@ public ConfigKey[] getConfigKeys() { KubernetesClusterNodeDrainTimeout, KubernetesClusterAddNodeTimeout, KubernetesClusterRemoveNodeTimeout, + KubernetesClusterUpdateNodeTimeout, KubernetesClusterExperimentalFeaturesEnabled, KubernetesMaxClusterSize, KubernetesControlNodeInstallAttemptWait, diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index d0a5c6bc5f39..e9cf8a4c5fc2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpdateKubernetesClusterAffinityGroupCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpdateKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; @@ -107,6 +108,13 @@ public interface KubernetesClusterService extends PluggableService, Configurable "Timeout interval (in seconds) in which an external node (VM / baremetal host) removal from a cluster should be completed", true, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesClusterUpdateNodeTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.update.node.timeout", + "1800", + "Timeout interval (in seconds) in which a rollout of updateKubernetesCluster should be done with a node, or a batch of worker nodes, " + + "from the drain to the nodes being Ready again. Nodes that are not done in time fail the rollout. Must be at least 1, smaller values use the default", + true, + KubernetesServiceEnabled.key()); static final ConfigKey KubernetesClusterExperimentalFeaturesEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.cluster.experimental.features.enabled", "false", @@ -181,6 +189,8 @@ public interface KubernetesClusterService extends PluggableService, Configurable boolean updateKubernetesClusterAffinityGroups(UpdateKubernetesClusterAffinityGroupCmd cmd) throws CloudRuntimeException; + boolean updateKubernetesCluster(UpdateKubernetesClusterCmd cmd) throws CloudRuntimeException; + boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd); boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 7dfd0043e320..e677350fb500 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -148,6 +148,15 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "csi_enabled") private boolean csiEnabled; + @Column(name = "control_node_root_disk_size") + private Long controlNodeRootDiskSize; + + @Column(name = "worker_node_root_disk_size") + private Long workerNodeRootDiskSize; + + @Column(name = "etcd_node_root_disk_size") + private Long etcdNodeRootDiskSize; + @Override public long getId() { return id; @@ -525,4 +534,31 @@ public void setCniConfigDetails(String cniConfigDetails) { this.cniConfigDetails = cniConfigDetails; } + @Override + public Long getControlNodeRootDiskSize() { + return controlNodeRootDiskSize; + } + + public void setControlNodeRootDiskSize(Long controlNodeRootDiskSize) { + this.controlNodeRootDiskSize = controlNodeRootDiskSize; + } + + @Override + public Long getWorkerNodeRootDiskSize() { + return workerNodeRootDiskSize; + } + + public void setWorkerNodeRootDiskSize(Long workerNodeRootDiskSize) { + this.workerNodeRootDiskSize = workerNodeRootDiskSize; + } + + @Override + public Long getEtcdNodeRootDiskSize() { + return etcdNodeRootDiskSize; + } + + public void setEtcdNodeRootDiskSize(Long etcdNodeRootDiskSize) { + this.etcdNodeRootDiskSize = etcdNodeRootDiskSize; + } + } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 1722d37d45b9..85d0bec3a90f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -175,6 +175,8 @@ public class KubernetesClusterActionWorker { // the cluster to and has not moved it out of yet private boolean operationStarted = false; private KubernetesCluster.State ownedState; + // the state transitToOperationState moved the cluster from + private KubernetesCluster.State operationStartState; @Inject protected CAManager caManager; @@ -995,9 +997,14 @@ protected boolean transitToOperationState(final KubernetesCluster.Event event, f throw operationRaced(state, current == null ? null : current.getState(), event); } ownedState = cluster.getState(); + operationStartState = state; return true; } + protected KubernetesCluster.State getOperationStartState() { + return operationStartState; + } + /** * Makes the operation hold a state it went on in without moving the cluster, so that it moves the cluster out of * it when it ends: a retried destroy completes the Destroying state. diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index c56c87d33f1b..ea4eb07ca6f0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -80,6 +80,7 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.dao.FirewallRulesDao; @@ -341,15 +342,23 @@ protected Map planKubernetesCluster(Long domainId, Lo return destinationMap; } - protected void resizeNodeVolume(final UserVm vm) throws ManagementServerException { + protected void resizeNodeVolume(final UserVm vm, final KubernetesClusterNodeType nodeType) throws ManagementServerException { try { if (vm.getHypervisorType() == Hypervisor.HypervisorType.VMware && templateDao.findById(vm.getTemplateId()).isDeployAsIs()) { List vmVols = volumeDao.findByInstance(vm.getId()); for (VolumeVO volumeVO : vmVols) { if (volumeVO.getVolumeType() == Volume.Type.ROOT) { + // etcd nodes without a size of their own are resized to the cluster wide size here, as before + long size = KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, nodeType); + size = size > 0 ? size : kubernetesCluster.getNodeRootDiskSize(); + if (volumeVO.getSize() != null && volumeVO.getSize() > size * 1024L * 1024L * 1024L) { + // the size of the node type may have been set lower than that of an existing node, which is not shrunk + logger.debug("Not resizing the root volume {} of VM {} down to {} GB", volumeVO, vm, size); + continue; + } ResizeVolumeCmd resizeVolumeCmd = new ResizeVolumeCmd(); resizeVolumeCmd = ComponentContext.inject(resizeVolumeCmd); - resizeVolumeCmd.setSize(kubernetesCluster.getNodeRootDiskSize()); + resizeVolumeCmd.setSize(size); resizeVolumeCmd.setId(volumeVO.getId()); volumeService.resizeVolume(resizeVolumeCmd); @@ -412,8 +421,8 @@ protected List provisionKubernetesClusterNodeVms(final long nodeCount, f UserVm vm = createKubernetesNode(controlIpAddress, domainId, accountId); vmContext.setEventResourceId(vm.getId()); addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, false, false); - if (kubernetesCluster.getNodeRootDiskSize() > 0) { - resizeNodeVolume(vm); + if (KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, WORKER) > 0) { + resizeNodeVolume(vm, WORKER); } startKubernetesVM(vm, domainId, accountId, WORKER); vm = userVmDao.findById(vm.getId()); @@ -443,7 +452,7 @@ protected UserVm createKubernetesNode(String joinIp, Long domainId, Long account networkIds.add(kubernetesCluster.getNetworkId()); Account owner = accountDao.findById(kubernetesCluster.getAccountId()); Network.IpAddresses addrs = new Network.IpAddresses(null, null); - long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + long rootDiskSize = KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, WORKER); Map customParameterMap = new HashMap(); if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRollingUpdateWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRollingUpdateWorker.java new file mode 100644 index 000000000000..756c8d698d5f --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRollingUpdateWorker.java @@ -0,0 +1,790 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.BooleanSupplier; +import java.util.stream.Collectors; + +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VirtualMachine; + +/** + * Restarts the nodes of a cluster whose service offering, root disk size or affinity groups differ from the settings + * of their node type, with those settings: the external etcd nodes, then the control nodes, one at a time, then the + * worker nodes, a batch at a time. + *

+ * Each Kubernetes node is annotated with its boot ID before it is drained, unless it was cordoned by someone else. It is + * then stopped, changed and started, and only taken to be back once it is Ready with another boot ID, as Kubernetes + * keeps reporting a node Ready for a while after it went down. Nothing else is recorded: the nodes that still differ, + * are stopped or carry the annotation are the work left, so a rollout that stopped is resumed by running it again. + * However it ends, {@link #release} puts back in service what can be. + */ +public class KubernetesClusterRollingUpdateWorker extends KubernetesClusterResourceModifierActionWorker { + + public static final String BOOT_ID_ANNOTATION = "cloudstack.apache.org/rollout-boot-id"; + protected static final String KUBECTL = "sudo /opt/bin/kubectl "; + /** One line per node: its name, Ready condition, boot ID, rollout annotation and whether it is cordoned, tab separated */ + protected static final String NODE_STATUSES_COMMAND = KUBECTL + "get nodes -o jsonpath='{range .items[*]}{.metadata.name}{\"\\t\"}" + + "{.status.conditions[?(@.type==\"Ready\")].status}{\"\\t\"}{.status.nodeInfo.bootID}{\"\\t\"}" + + "{.metadata.annotations.cloudstack\\.apache\\.org/rollout-boot-id}{\"\\t\"}{.spec.unschedulable}{\"\\n\"}{end}'"; + protected static final String ANNOTATE_COMMAND = KUBECTL + "annotate node %s --overwrite " + BOOT_ID_ANNOTATION + "=%s"; + protected static final String CORDON_COMMAND = KUBECTL + "cordon %s"; + protected static final String RELEASE_COMMAND = KUBECTL + "uncordon %s && " + KUBECTL + "annotate node %s " + BOOT_ID_ANNOTATION + "-"; + protected static final String ETCD_READY_COMMAND = KUBECTL + "get --raw=/readyz/etcd"; + protected static final String ETCD_MEMBER_HEALTH_COMMAND = "sudo /opt/bin/etcdctl --endpoints=http://127.0.0.1:2379 endpoint health"; + /** The health of the etcd member a control node runs, from the metrics address kubeadm gives it */ + protected static final String STACKED_ETCD_MEMBER_HEALTH_COMMAND = "curl -s http://127.0.0.1:2381/health"; + protected static final List NODE_TYPE_ORDER = List.of(ETCD, CONTROL, WORKER); + protected static final int DRAIN_ATTEMPTS = 3; + protected static final long POLL_WAIT = 15000; + protected static final int MAX_ERROR_LENGTH = 1024; + + /** A Kubernetes node as kubectl reports it */ + protected static class NodeStatus { + final boolean ready; + final String bootId; + final String annotation; + final boolean unschedulable; + + NodeStatus(boolean ready, String bootId, String annotation, boolean unschedulable) { + this.ready = ready; + this.bootId = bootId; + this.annotation = annotation; + this.unschedulable = unschedulable; + } + } + + /** A node of a batch, with what the batch did to it */ + protected static class Node { + final KubernetesClusterVmMapVO vmMap; + final KubernetesClusterNodeType type; + UserVmVO vm; + /** The boot ID the Kubernetes node has to change from to be back, "" if any will do */ + String bootIdBefore = ""; + /** Whether the node carries the annotation, so that it is uncordoned once it is back */ + boolean annotated; + /** Whether this batch annotated the node */ + boolean annotatedHere; + /** Whether its VM may have gone down in this batch */ + boolean stopped; + + Node(KubernetesClusterVmMapVO vmMap, UserVmVO vm) { + this.vmMap = vmMap; + this.vm = vm; + this.type = KubernetesClusterUtil.getNodeType(vmMap); + } + + boolean isKubernetesNode() { + return type != ETCD; + } + } + + protected final int maxUnavailable; + private boolean begun; + /** The SSH address of the control node kubectl runs on */ + protected Pair kubectlAddress; + + public KubernetesClusterRollingUpdateWorker(final KubernetesCluster kubernetesCluster, final int maxUnavailable, + final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + this.maxUnavailable = maxUnavailable; + } + + // pure decisions + + /** The worker batch size: at most maxunavailable, and leaving a worker node running, but at least 1 */ + public static int clampMaxUnavailable(final int requested, final int workers) { + return Math.max(1, Math.min(requested, workers - 1)); + } + + /** The number of members of an etcd cluster that have to be up for it to work */ + public static int quorum(final int members) { + return members / 2 + 1; + } + + /** + * The number of other healthy nodes of its type a control or etcd node needs to be taken down: a quorum of the + * etcd members, which the control nodes are unless etcd runs on its own nodes, in which case one other control + * node to serve the Kubernetes API. + */ + public static int healthyOthersNeeded(final KubernetesClusterNodeType type, final int members, final boolean etcdOnOwnNodes) { + return type == CONTROL && etcdOnOwnNodes ? 1 : quorum(members); + } + + /** Whether the nodes of a type can be restarted one at a time, with the others keeping the cluster running */ + public static boolean canRestartOneAtATime(final KubernetesClusterNodeType type, final int members, final boolean etcdOnOwnNodes) { + return type == WORKER || members - 1 >= healthyOthersNeeded(type, members, etcdOnOwnNodes); + } + + /** Whether a node is Ready since a boot other than the one it had before its restart */ + public static boolean isBack(final NodeStatus status, final String bootIdBefore) { + return status != null && status.ready && status.bootId != null && !status.bootId.equals(bootIdBefore); + } + + public static List> toBatches(final List nodes, final int size) { + List> batches = new ArrayList<>(); + for (int i = 0; i < nodes.size(); i += size) { + batches.add(new ArrayList<>(nodes.subList(i, Math.min(nodes.size(), i + size)))); + } + return batches; + } + + /** Reads the output of {@link #NODE_STATUSES_COMMAND}; lines that are not about a node, such as a warning from sudo, are left out */ + public static Map parseNodeStatuses(final String output) { + Map statuses = new HashMap<>(); + if (output == null) { + return statuses; + } + for (String line : output.split("\\R")) { + String[] fields = line.split("\t", -1); + if (fields.length != 5 || StringUtils.isBlank(fields[0])) { + continue; + } + statuses.put(fields[0].trim(), new NodeStatus("True".equals(fields[1].trim()), StringUtils.trimToNull(fields[2]), + StringUtils.trimToNull(fields[3]), "true".equals(fields[4].trim()))); + } + return statuses; + } + + // the rollout + + /** + * Moves the cluster to Updating, which only one operation can do at a time, so that the caller can then change + * the settings of the node types without another operation working from them + */ + public void begin() { + if (begun) { + return; + } + init(); + transitToOperationState(KubernetesCluster.Event.UpdateRequested); + begun = true; + // a cancel left by a rollout that did not end; one asked for from now on is for this rollout + try { + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterManagerImpl.ROLLOUT_CANCEL_DETAIL); + } catch (RuntimeException e) { + try { + abort(); + } catch (RuntimeException releaseError) { + e.addSuppressed(releaseError); + } + throw e; + } + } + + /** + * Runs the rollout. Throws if it does not complete, after {@link #release} has put back in service what it could. + * Node types that can not lose a node are left out and reported, without failing the rollout. + */ + public boolean rollout() { + begin(); + String error; + List skipped = new ArrayList<>(); + try { + // the settings to roll out, which the caller stored after begin() + kubernetesCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + // nodes a rollout that stopped restarted, but did not get to uncordon, are not restarted again + releaseNodesThatAreBack(); + error = rollNodes(skipped); + } catch (Exception e) { + logger.warn("Rollout of Kubernetes cluster {} failed", kubernetesCluster, e); + error = StringUtils.defaultIfEmpty(e.getMessage(), e.getClass().getSimpleName()); + } + List problems = new ArrayList<>(skipped); + if (error != null) { + problems.add(0, error); + } + final String summary = problems.isEmpty() ? null : String.join("; ", problems); + release(summary); + if (error != null) { + throw new CloudRuntimeException(String.format("The rollout of Kubernetes cluster %s stopped: %s. Call updateKubernetesCluster with rollout=true " + + "to resume it", kubernetesCluster.getName(), summary)); + } + logger.info("Rollout of Kubernetes cluster {} completed{}", kubernetesCluster, summary == null ? "" : ", but " + summary); + return true; + } + + /** + * Works through the node types in order; returns why it stopped, or null once no node is left but those of the + * types added to skipped + */ + protected String rollNodes(final List skipped) { + for (KubernetesClusterNodeType type : NODE_TYPE_ORDER) { + resolveKubectlAddress(Set.of()); + List nodes = selectNodes(type, getNodeStatuses()); + int members = countManagedNodes(type); + if (!nodes.isEmpty() && !canRestartOneAtATime(type, members, isEtcdOnOwnNodes())) { + // the others are left to roll; this one can never be, so it is reported rather than retried + String reason = String.format("the %d %s node(s) were not restarted, as a cluster with %d of them can not keep running without one", + nodes.size(), type.name().toLowerCase(), members); + logger.warn("Not restarting the {} nodes of Kubernetes cluster {}: {}", type.name().toLowerCase(), kubernetesCluster, reason); + skipped.add(reason); + continue; + } + int batchSize = type == WORKER ? clampMaxUnavailable(maxUnavailable, members) : 1; + for (List batch : toBatches(nodes, batchSize)) { + if (isCancelRequested()) { + return "it was cancelled"; + } + rollBatch(batch); + } + } + return null; + } + + protected boolean isCancelRequested() { + return kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), KubernetesClusterManagerImpl.ROLLOUT_CANCEL_DETAIL) != null; + } + + /** + * The nodes of the type to restart: those that differ from the settings of their type, are stopped, or carry the + * annotation of a rollout that did not finish with them, which go first. External nodes are left out. + */ + protected List selectNodes(final KubernetesClusterNodeType type, final Map statuses) { + List nodes = new ArrayList<>(); + Set unfinished = new HashSet<>(); + for (KubernetesClusterVmMapVO vmMap : getKubernetesClusterVMMaps()) { + if (vmMap.isExternalNode() || KubernetesClusterUtil.getNodeType(vmMap) != type) { + continue; + } + UserVmVO vm = userVmDao.findById(vmMap.getVmId()); + if (vm == null) { + continue; + } + NodeStatus status = type == ETCD ? null : statuses.get(getNodeName(vm)); + boolean stopped = !VirtualMachine.State.Running.equals(vm.getState()); + if (stopped || status != null && status.annotation != null) { + unfinished.add(vm.getId()); + } else if (manager.getNodeOutOfDateReason(kubernetesCluster, vmMap, vm) == null) { + continue; + } + nodes.add(new Node(vmMap, vm)); + } + nodes.sort(Comparator.comparing(node -> !unfinished.contains(node.vm.getId()))); + return nodes; + } + + protected int countManagedNodes(final KubernetesClusterNodeType type) { + return KubernetesClusterUtil.countManagedNodes(getKubernetesClusterVMMaps(), type); + } + + /** + * Restarts a batch of nodes with the settings of their type. A node is only uncordoned once it is back; on a + * failure, the nodes this batch annotated but did not stop are uncordoned at once, as they never went down. + */ + protected void rollBatch(final List batch) { + final KubernetesClusterNodeType type = batch.get(0).type; + // the VMs may have changed while the batches before ran + for (Node node : batch) { + UserVmVO vm = userVmDao.findById(node.vm.getId()); + if (vm == null) { + throw new CloudRuntimeException(String.format("the VM of node %s is gone", getNodeName(node.vm))); + } + node.vm = vm; + } + final long deadline = System.currentTimeMillis() + getNodeTimeout() * 1000; + final Set batchVmIds = batch.stream().map(node -> node.vm.getId()).collect(Collectors.toSet()); + logger.info("Restarting {} node(s) {} of Kubernetes cluster {} with the settings of their type", type.name().toLowerCase(), + batch.stream().map(node -> getNodeName(node.vm)).collect(Collectors.toList()), kubernetesCluster); + try { + resolveKubectlAddress(type == CONTROL ? batchVmIds : Set.of()); + Map statuses = getNodeStatuses(); + String unhealthy = getUnhealthyReason(batchVmIds, statuses); + if (unhealthy != null) { + throw new CloudRuntimeException(String.format("the cluster is not healthy: %s. Start it, or pass a maxunavailable that covers it", unhealthy)); + } + if (type != WORKER) { + checkQuorum(batch.get(0), statuses); + } + for (Node node : batch) { + annotateAndCordon(node, statuses.get(getNodeName(node.vm))); + } + for (Node node : batch) { + drain(node, deadline); + } + for (Node node : batch) { + stopVm(node); + } + for (Node node : batch) { + applySettings(node); + } + for (Node node : batch) { + startVm(node); + } + for (Node node : batch) { + waitUntilBack(node, deadline); + } + for (Node node : batch) { + String reason = manager.getNodeOutOfDateReason(kubernetesCluster, node.vmMap, userVmDao.findById(node.vm.getId())); + if (reason != null) { + throw new CloudRuntimeException(String.format("node %s still differs from the settings of its type after its restart: %s", getNodeName(node.vm), reason)); + } + if (node.annotated && !releaseNode(node.vm)) { + throw new CloudRuntimeException(String.format("unable to uncordon node %s", getNodeName(node.vm))); + } + } + } catch (RuntimeException e) { + for (Node node : batch) { + if (node.annotatedHere && !node.stopped && !releaseNode(node.vm)) { + logger.warn("Unable to uncordon node {} of Kubernetes cluster {}, which was not restarted", getNodeName(node.vm), kubernetesCluster); + } + } + throw e; + } + } + + /** + * Records the boot ID of a Kubernetes node on it, unless it is cordoned by someone else, and cordons it. A node + * that carries the annotation already keeps it: its rollout did not finish, and the boot ID is from before it. + */ + protected void annotateAndCordon(final Node node, final NodeStatus status) { + if (!node.isKubernetesNode()) { + return; + } + final String name = getNodeName(node.vm); + if (status == null) { + if (VirtualMachine.State.Running.equals(node.vm.getState())) { + throw new CloudRuntimeException(String.format("node %s is not in the Kubernetes cluster", name)); + } + // a stopped VM whose node was deleted: it registers again once started, with any boot ID + return; + } + if (status.annotation != null) { + node.annotated = true; + node.bootIdBefore = status.annotation; + } else { + if (status.bootId == null) { + throw new CloudRuntimeException(String.format("node %s reports no boot ID", name)); + } + node.bootIdBefore = status.bootId; + if (!status.unschedulable) { + // recorded before the cordon, so that any cordon of ours can be told apart from someone else's + if (runKubectl(String.format(ANNOTATE_COMMAND, name, status.bootId)) == null) { + throw new CloudRuntimeException(String.format("unable to annotate node %s", name)); + } + node.annotated = true; + node.annotatedHere = true; + } + } + if (runKubectl(String.format(CORDON_COMMAND, name)) == null) { + throw new CloudRuntimeException(String.format("unable to cordon node %s", name)); + } + } + + protected void drain(final Node node, final long deadline) { + if (!node.isKubernetesNode() || !VirtualMachine.State.Running.equals(node.vm.getState())) { + return; + } + Pair drained = drainKubernetesClusterNode(kubectlAddress.first(), kubectlAddress.second(), node.vm, DRAIN_ATTEMPTS, DRAIN_RETRY_WAIT, deadline); + if (!Boolean.TRUE.equals(drained.first())) { + throw new CloudRuntimeException(String.format("unable to drain node %s: %s", getNodeName(node.vm), drained.second())); + } + } + + protected void stopVm(final Node node) { + node.stopped = true; + if (!VirtualMachine.State.Running.equals(node.vm.getState())) { + return; + } + final String name = getNodeName(node.vm); + registerVmContext(node.vm); + try { + userVmService.stopVirtualMachine(node.vm.getId(), false); + } catch (Exception e) { + logger.warn("Failed to stop VM {} of Kubernetes cluster {}", node.vm, kubernetesCluster, e); + } finally { + CallContext.unregister(); + } + UserVmVO vm = userVmDao.findById(node.vm.getId()); + if (vm == null || !VirtualMachine.State.Stopped.equals(vm.getState())) { + // a VM that is still running never went down + node.stopped = vm == null || !VirtualMachine.State.Running.equals(vm.getState()); + throw new CloudRuntimeException(String.format("unable to stop the VM of node %s", name)); + } + node.vm = vm; + } + + /** Gives the stopped VM the service offering, root disk size and affinity groups of its node type, as needed */ + protected void applySettings(final Node node) { + final String name = getNodeName(node.vm); + try { + Long offeringId = manager.getNodeTypeServiceOfferingId(kubernetesCluster, node.type); + if (offeringId != null && offeringId != node.vm.getServiceOfferingId()) { + registerVmContext(node.vm); + try { + if (!userVmManager.upgradeVirtualMachine(node.vm.getId(), offeringId, new HashMap<>())) { + throw new CloudRuntimeException(String.format("unable to change the service offering of the VM of node %s", name)); + } + } finally { + CallContext.unregister(); + } + } + long sizeGb = KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, node.type); + List roots = volumeDao.findByInstanceAndType(node.vm.getId(), Volume.Type.ROOT); + if (sizeGb > 0 && !roots.isEmpty() && roots.get(0).getSize() != null && roots.get(0).getSize() < (sizeGb << 30)) { + ResizeVolumeCmd resizeVolumeCmd = ComponentContext.inject(new ResizeVolumeCmd()); + resizeVolumeCmd.setId(roots.get(0).getId()); + resizeVolumeCmd.setSize(sizeGb); + volumeService.resizeVolume(resizeVolumeCmd); + } + Set wanted = manager.getNodeTypeWantedAffinityGroupIds(kubernetesCluster.getId(), node.type); + Set groupIds = wanted == null ? null : new LinkedHashSet<>(wanted); + if (groupIds != null && !groupIds.equals(manager.getVmAffinityGroupIds(node.vm.getId(), false))) { + // the explicit dedication group CKS gives the nodes of a dedicated domain or account stays + groupIds.addAll(manager.getVmAffinityGroupIds(node.vm.getId(), true)); + affinityGroupVMMapDao.updateMap(node.vm.getId(), new ArrayList<>(groupIds)); + } + } catch (CloudRuntimeException e) { + throw e; + } catch (Exception e) { + logger.warn("Failed to change the VM of node {} of Kubernetes cluster {}", name, kubernetesCluster, e); + throw new CloudRuntimeException(String.format("unable to change the VM of node %s: %s", name, e.getMessage()), e); + } + } + + /** Starts the VM, which the deployment planner places by its new affinity groups */ + protected void startVm(final Node node) { + registerVmContext(node.vm); + try { + userVmManager.startVirtualMachine(userVmDao.findById(node.vm.getId()), null); + } catch (Exception e) { + logger.warn("Failed to start VM {} of Kubernetes cluster {}", node.vm, kubernetesCluster, e); + throw new CloudRuntimeException(String.format("unable to start the VM of node %s: %s", getNodeName(node.vm), + StringUtils.defaultIfEmpty(e.getMessage(), e.getClass().getSimpleName())), e); + } finally { + CallContext.unregister(); + } + node.vm = userVmDao.findById(node.vm.getId()); + } + + /** + * Waits for a Kubernetes node to be Ready with a new boot ID, and for the etcd member of an etcd or control node + * and the etcd cluster to be healthy again + */ + protected void waitUntilBack(final Node node, final long deadline) { + final String name = getNodeName(node.vm); + if (node.isKubernetesNode() && !pollUntil(deadline, () -> isBack(getNodeStatusesOrEmpty().get(name), node.bootIdBefore))) { + throw new CloudRuntimeException(String.format("node %s is not Ready with a new boot ID", name)); + } + if (isEtcdMember(node.type) && !pollUntil(deadline, () -> isEtcdMemberHealthy(node.vm))) { + throw new CloudRuntimeException(String.format("the etcd member of node %s does not report healthy", name)); + } + if (node.type != WORKER && !pollUntil(deadline, this::isEtcdReady)) { + throw new CloudRuntimeException("etcd does not report ready"); + } + } + + /** + * Refuses to take a control or etcd node down if the others of its type could not keep a quorum without it, + * or, for a control node when etcd runs on its own nodes, if no other control node serves the Kubernetes API. + */ + protected void checkQuorum(final Node node, final Map statuses) { + final boolean etcdOnOwnNodes = isEtcdOnOwnNodes(); + int members = 0; + int healthyOthers = 0; + for (KubernetesClusterVmMapVO vmMap : getKubernetesClusterVMMaps()) { + if (KubernetesClusterUtil.getNodeType(vmMap) != node.type) { + continue; + } + members++; + UserVmVO vm = vmMap.getVmId() == node.vm.getId() ? null : userVmDao.findById(vmMap.getVmId()); + if (vm == null || !VirtualMachine.State.Running.equals(vm.getState())) { + continue; + } + NodeStatus status = statuses.get(getNodeName(vm)); + boolean ready = node.type == ETCD || status != null && status.ready; + if (ready && (!isEtcdMember(node.type) || isEtcdMemberHealthy(vm))) { + healthyOthers++; + } + } + int needed = healthyOthersNeeded(node.type, members, etcdOnOwnNodes); + if (healthyOthers < needed) { + throw new CloudRuntimeException(String.format("%s node %s can not be restarted: only %d of the other %d are healthy, and %d are needed to keep the cluster available", + node.type.name().toLowerCase(), getNodeName(node.vm), healthyOthers, members - 1, needed)); + } + if (!isEtcdReady()) { + throw new CloudRuntimeException("etcd does not report ready"); + } + } + + /** + * Why the cluster is not healthy, apart from the given VMs, or null if it is: a VM that is not running, or a + * Kubernetes node that is not Ready + */ + protected String getUnhealthyReason(final Set exemptVmIds, final Map statuses) { + for (KubernetesClusterVmMapVO vmMap : getKubernetesClusterVMMaps()) { + if (exemptVmIds.contains(vmMap.getVmId())) { + continue; + } + UserVmVO vm = userVmDao.findById(vmMap.getVmId()); + if (vm == null || !VirtualMachine.State.Running.equals(vm.getState())) { + return String.format("the VM of node %s is %s", vm == null ? vmMap.getVmId() : getNodeName(vm), vm == null ? "missing" : vm.getState()); + } + NodeStatus status = vmMap.isEtcdNode() ? null : statuses.get(getNodeName(vm)); + if (!vmMap.isEtcdNode() && (status == null || !status.ready)) { + return String.format("node %s is not Ready", getNodeName(vm)); + } + } + return null; + } + + // release + + /** + * Puts back in service what a rollout leaves behind, however it ends; running it again changes nothing more. + * It uncordons the annotated nodes that are Ready with a new boot ID, and leaves the others cordoned, counts the + * CPU cores and memory of the cluster from its VMs, moves the cluster from Updating to Running, or to Alert if it is + * not healthy, and records why the rollout stopped, if it did. + */ + public void release(final String error) { + release(error, true); + } + + /** + * Ends a rollout that did no node work, such as one whose settings could not be stored or were refused: the + * cluster goes back to the state it was in, without looking at its nodes, and the last error, which may tell + * of a node left down, stays as it is + */ + public void abort() { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster == null || !KubernetesCluster.State.Updating.equals(cluster.getState())) { + return; + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.State.Running.equals(getOperationStartState()) + ? KubernetesCluster.Event.OperationSucceeded : KubernetesCluster.Event.OperationFailed); + } + + protected void release(final String error, final boolean recordError) { + Map statuses = null; + try { + statuses = releaseNodesThatAreBack(); + } catch (Exception e) { + logger.warn("Unable to uncordon the nodes of Kubernetes cluster {} the rollout is done with", kubernetesCluster, e); + } + try { + manager.updateKubernetesClusterCapacity(kubernetesCluster.getId()); + } catch (Exception e) { + logger.warn("Unable to count the CPU cores and memory of Kubernetes cluster {}", kubernetesCluster, e); + } + String unhealthy = statuses == null ? "its nodes can not be reached" : getUnhealthyReason(Set.of(), statuses); + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster == null || !KubernetesCluster.State.Updating.equals(cluster.getState())) { + // released already: the details may be those of a rollout that started since + return; + } + // before the cluster leaves Updating, after which another rollout may start and be asked to cancel + if (!recordError) { + logger.debug("Kept the last rollout error of Kubernetes cluster {}", kubernetesCluster); + } else if (error == null) { + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL); + } else { + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL, + StringUtils.abbreviate(error, MAX_ERROR_LENGTH), true); + } + kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), KubernetesClusterManagerImpl.ROLLOUT_CANCEL_DETAIL); + if (unhealthy != null) { + logger.warn("Kubernetes cluster {} is not healthy after its rollout: {}", kubernetesCluster, unhealthy); + } + stateTransitTo(kubernetesCluster.getId(), unhealthy == null ? KubernetesCluster.Event.OperationSucceeded : KubernetesCluster.Event.OperationFailed); + } + + /** + * Uncordons the annotated nodes that are Ready with a new boot ID, and removes their annotation; returns the + * status of the nodes after that + */ + protected Map releaseNodesThatAreBack() { + resolveKubectlAddress(Set.of()); + Map statuses = getNodeStatuses(); + boolean released = false; + for (KubernetesClusterVmMapVO vmMap : getKubernetesClusterVMMaps()) { + UserVmVO vm = userVmDao.findById(vmMap.getVmId()); + NodeStatus status = vm == null ? null : statuses.get(getNodeName(vm)); + if (status == null || status.annotation == null || !isBack(status, status.annotation)) { + continue; + } + if (releaseNode(vm)) { + released = true; + } else { + logger.warn("Unable to uncordon node {} of Kubernetes cluster {}", getNodeName(vm), kubernetesCluster); + } + } + return released ? getNodeStatuses() : statuses; + } + + // access to the nodes + + /** + * Runs kubectl on the first running control node that is not one of the given VMs: the first control node, which + * the address of the cluster reaches, unless it is being restarted + */ + protected void resolveKubectlAddress(final Set excludedVmIds) { + Pair address = getKubernetesClusterServerIpSshPort(null); + if (StringUtils.isEmpty(address.first())) { + throw new CloudRuntimeException(String.format("unable to find the address of Kubernetes cluster %s", kubernetesCluster.getName())); + } + publicIpAddress = address.first(); + sshPort = address.second(); + for (KubernetesClusterVmMapVO vmMap : getKubernetesClusterVMMaps()) { + if (!vmMap.isControlNode() || excludedVmIds.contains(vmMap.getVmId())) { + continue; + } + UserVmVO vm = userVmDao.findById(vmMap.getVmId()); + if (vm != null && VirtualMachine.State.Running.equals(vm.getState())) { + kubectlAddress = getSshAddress(vm); + return; + } + } + throw new CloudRuntimeException(String.format("no control node of Kubernetes cluster %s is running", kubernetesCluster.getName())); + } + + /** + * The address to reach a node over SSH: without port forwarding, its own address; with it, the port of its rule, + * or else of its position among the VMs of the cluster + */ + protected Pair getSshAddress(final UserVm vm) { + if (sshPort == DEFAULT_SSH_PORT && StringUtils.isNotEmpty(vm.getPrivateIpAddress())) { + return new Pair<>(vm.getPrivateIpAddress(), DEFAULT_SSH_PORT); + } + List vmMaps = getKubernetesClusterVMMaps(); + int index = 0; + for (int i = 0; i < vmMaps.size(); i++) { + if (vmMaps.get(i).getVmId() == vm.getId()) { + index = i; + break; + } + } + return getNodeSshAddress(vm, index); + } + + /** The output of a kubectl command, or null if it failed */ + protected String runKubectl(final String command) { + try { + Pair result = executeSshCommand(kubectlAddress.first(), kubectlAddress.second(), command, 10000, 10000, 60000); + return Boolean.TRUE.equals(result.first()) && result.second() != null ? result.second() : null; + } catch (Exception e) { + logger.debug("Unable to run {} on Kubernetes cluster {}", command, kubernetesCluster, e); + return null; + } + } + + protected Map getNodeStatuses() { + String output = runKubectl(NODE_STATUSES_COMMAND); + if (output == null) { + throw new CloudRuntimeException(String.format("unable to get the status of the nodes of Kubernetes cluster %s", kubernetesCluster.getName())); + } + return parseNodeStatuses(output); + } + + /** The node statuses, or none while the Kubernetes API can not be reached, as while a control node restarts */ + protected Map getNodeStatusesOrEmpty() { + try { + return getNodeStatuses(); + } catch (CloudRuntimeException e) { + logger.debug("Unable to get the status of the nodes of Kubernetes cluster {}: {}", kubernetesCluster, e.getMessage()); + return Map.of(); + } + } + + /** Uncordons a node and removes its annotation */ + protected boolean releaseNode(final UserVm vm) { + String name = getNodeName(vm); + return runKubectl(String.format(RELEASE_COMMAND, name, name)) != null; + } + + protected boolean isEtcdReady() { + String output = runKubectl(ETCD_READY_COMMAND); + return output != null && output.lines().anyMatch(line -> "ok".equals(line.trim())); + } + + protected boolean isEtcdOnOwnNodes() { + return kubernetesCluster.getEtcdNodeCount() != null && kubernetesCluster.getEtcdNodeCount() > 0; + } + + /** Whether nodes of the type run an etcd member: etcd nodes, or control nodes when etcd runs on them */ + protected boolean isEtcdMember(final KubernetesClusterNodeType type) { + return isEtcdOnOwnNodes() ? type == ETCD : type == CONTROL; + } + + /** Whether the etcd member on a VM reports itself healthy, which it only does once it is part of the etcd cluster */ + protected boolean isEtcdMemberHealthy(final UserVm vm) { + boolean ownNode = isEtcdOnOwnNodes(); + Pair address = getSshAddress(vm); + try { + Pair result = executeSshCommand(address.first(), address.second(), + ownNode ? ETCD_MEMBER_HEALTH_COMMAND : STACKED_ETCD_MEMBER_HEALTH_COMMAND, 10000, 10000, 30000); + if (!Boolean.TRUE.equals(result.first()) || result.second() == null) { + return false; + } + return ownNode ? result.second().contains("is healthy") : result.second().replace(" ", "").contains("\"health\":\"true\""); + } catch (Exception e) { + logger.debug("Unable to check the etcd member on {} of Kubernetes cluster {}", vm, kubernetesCluster, e); + return false; + } + } + + /** Checks the condition until it holds or the deadline passes; returns whether it held */ + protected boolean pollUntil(final long deadline, final BooleanSupplier condition) { + do { + if (condition.getAsBoolean()) { + return true; + } + } while (System.currentTimeMillis() < deadline && sleepBeforeRetry(POLL_WAIT)); + return false; + } + + protected long getNodeTimeout() { + Long timeout = KubernetesClusterService.KubernetesClusterUpdateNodeTimeout.value(); + return timeout != null && timeout > 0 ? timeout : Long.parseLong(KubernetesClusterService.KubernetesClusterUpdateNodeTimeout.defaultValue()); + } + + /** Registers a call context for the events of a VM action; the caller unregisters it */ + protected void registerVmContext(final UserVm vm) { + CallContext vmContext = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine); + vmContext.setEventResourceId(Objects.requireNonNull(vm).getId()); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 3c1508c11766..735b36ab65ef 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -260,7 +260,7 @@ private Pair createKubernetesControlNode(final Network network, S serverIp = controlNodeIp; } Network.IpAddresses addrs = new Network.IpAddresses(controlNodeIp, null); - long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + long rootDiskSize = KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, CONTROL); Map customParameterMap = new HashMap(); if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); @@ -442,7 +442,7 @@ private UserVm createKubernetesAdditionalControlNode(final String joinIp, final List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); Network.IpAddresses addrs = new Network.IpAddresses(null, null); - long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + long rootDiskSize = KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, CONTROL); Map customParameterMap = new HashMap(); if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); @@ -512,6 +512,10 @@ private UserVm createEtcdNode(List requestedIps, List affinityGroupIds = getMergedAffinityGroupIds(ETCD, domainId, accountId); String hostName = etcdNodeHostnames.get(etcdNodeIndex); Map customParameterMap = new HashMap(); + long rootDiskSize = KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, ETCD); + if (rootDiskSize > 0) { + customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); + } if (zone.isSecurityGroupEnabled()) { List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); @@ -542,8 +546,8 @@ private Pair provisionKubernetesClusterControlVm(final Network n k8sControlVMAndControlIP = createKubernetesControlNode(network, publicIpAddress, etcdIps, domainId, accountId, asNumber); k8sControlVM = k8sControlVMAndControlIP.first(); addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false, false, false); - if (kubernetesCluster.getNodeRootDiskSize() > 0) { - resizeNodeVolume(k8sControlVM); + if (KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, CONTROL) > 0) { + resizeNodeVolume(k8sControlVM, CONTROL); } startKubernetesVM(k8sControlVM, domainId, accountId, CONTROL); k8sControlVM = userVmDao.findById(k8sControlVM.getId()); @@ -565,8 +569,8 @@ private List provisionKubernetesClusterAdditionalControlVms(final String UserVm vm = null; vm = createKubernetesAdditionalControlNode(controlIpAddress, i, domainId, accountId); addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false, false, false); - if (kubernetesCluster.getNodeRootDiskSize() > 0) { - resizeNodeVolume(vm); + if (KubernetesClusterUtil.getNodeRootDiskSize(kubernetesCluster, CONTROL) > 0) { + resizeNodeVolume(vm, CONTROL); } startKubernetesVM(vm, domainId, accountId, CONTROL); vm = userVmDao.findById(vm.getId()); @@ -701,9 +705,11 @@ private void startKubernetesClusterVMs(Long domainId, Long accountId) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start all VMs in Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } try { - resizeNodeVolume(vm); KubernetesClusterVmMapVO map = kubernetesClusterVmMapDao.findByVmId(vm.getId()); - KubernetesServiceHelper.KubernetesClusterNodeType nodeType = getNodeTypeFromClusterVMMapRecord(map); + KubernetesServiceHelper.KubernetesClusterNodeType nodeType = KubernetesClusterUtil.getNodeType(map); + // an existing node keeps the cluster wide size, as before: the size of its node type is only + // brought to it by a rollout of updateKubernetesCluster + resizeNodeVolume(vm, null); startKubernetesVM(vm, domainId, accountId, nodeType); } catch (ManagementServerException ex) { logger.warn("Failed to start VM: {} in Kubernetes cluster: {} due to {}", vm, kubernetesCluster, ex); @@ -718,16 +724,6 @@ private void startKubernetesClusterVMs(Long domainId, Long accountId) { } } - private KubernetesServiceHelper.KubernetesClusterNodeType getNodeTypeFromClusterVMMapRecord(KubernetesClusterVmMapVO map) { - if (map.isControlNode()) { - return CONTROL; - } else if (map.isEtcdNode()) { - return ETCD; - } else { - return WORKER; - } - } - private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) { if (StringUtils.isEmpty(publicIpAddress)) { KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDao.java index 8c152153a2cd..4ad5bc5c3689 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDao.java @@ -28,4 +28,6 @@ public interface KubernetesClusterAffinityGroupMapDao extends GenericDao listAffinityGroupIdsByClusterIdAndNodeType(long clusterId, String nodeType); int removeByClusterId(long clusterId); + + int removeByClusterIdAndNodeType(long clusterId, String nodeType); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDaoImpl.java index fcd4d6f7c97f..61abd4bdfbc8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDaoImpl.java @@ -69,4 +69,12 @@ public int removeByClusterId(long clusterId) { sc.setParameters("clusterId", clusterId); return remove(sc); } + + @Override + public int removeByClusterIdAndNodeType(long clusterId, String nodeType) { + SearchCriteria sc = clusterIdAndNodeTypeSearch.create(); + sc.setParameters("clusterId", clusterId); + sc.setParameters("nodeType", nodeType); + return remove(sc); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 46a3f71c136a..5916fbe97a41 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -25,6 +25,7 @@ import java.net.Socket; import java.net.URL; import java.security.SecureRandom; +import java.util.List; import java.util.stream.Collectors; import javax.net.ssl.HttpsURLConnection; @@ -39,6 +40,7 @@ import org.apache.logging.log4j.LogManager; import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.nio.TrustAllManager; @@ -48,6 +50,39 @@ public class KubernetesClusterUtil { protected static Logger LOGGER = LogManager.getLogger(KubernetesClusterUtil.class); + /** + * Returns the root disk size, in GB, that new nodes of a type get: the size stored for the type if there is one, + * otherwise the cluster wide size for control and worker nodes. Etcd nodes never got the cluster wide size, so + * without a size of their own they keep the size of the template. 0 stands for the size of the template. + */ + public static long getNodeRootDiskSize(final KubernetesCluster kubernetesCluster, final KubernetesClusterNodeType nodeType) { + Long size = null; + if (KubernetesClusterNodeType.CONTROL == nodeType) { + size = kubernetesCluster.getControlNodeRootDiskSize(); + } else if (KubernetesClusterNodeType.WORKER == nodeType) { + size = kubernetesCluster.getWorkerNodeRootDiskSize(); + } else if (KubernetesClusterNodeType.ETCD == nodeType) { + size = kubernetesCluster.getEtcdNodeRootDiskSize(); + } + if (size != null && size > 0) { + return size; + } + return KubernetesClusterNodeType.ETCD == nodeType ? 0 : kubernetesCluster.getNodeRootDiskSize(); + } + + /** The node type of a VM of a cluster */ + public static KubernetesClusterNodeType getNodeType(final KubernetesClusterVmMapVO vmMap) { + if (vmMap.isControlNode()) { + return KubernetesClusterNodeType.CONTROL; + } + return vmMap.isEtcdNode() ? KubernetesClusterNodeType.ETCD : KubernetesClusterNodeType.WORKER; + } + + /** The number of nodes of a type that CKS manages, external nodes left out */ + public static int countManagedNodes(final List vmMaps, final KubernetesClusterNodeType type) { + return (int) vmMaps.stream().filter(vmMap -> !vmMap.isExternalNode() && getNodeType(vmMap) == type).count(); + } + public static final String CLUSTER_NODE_READY_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'"; public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl get nodes | awk '{if ($1 == \"%s\") print $5}'"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpdateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpdateKubernetesClusterCmd.java new file mode 100644 index 000000000000..107fcb54cc0c --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpdateKubernetesClusterCmd.java @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VmDetailConstants; + +@APICommand(name = "updateKubernetesCluster", + description = "Updates the service offering, root disk size and affinity groups of the node types of a CloudManaged Kubernetes cluster. " + + "The new settings are used for the nodes created from then on. With rollout=true, the existing nodes that differ from them are " + + "drained, stopped, changed and started again, one at a time, or maxunavailable worker nodes at a time. A rollout that fails " + + "is resumed by calling this API again with rollout=true", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + since = "24.0.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class UpdateKubernetesClusterCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + @Inject + protected KubernetesServiceHelper kubernetesServiceHelper; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, + entityType = KubernetesClusterResponse.class, + description = "The ID of the Kubernetes cluster") + private Long id; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_OFFERING_MAP, type = CommandType.MAP, + description = "Node type to service offering ID mapping, for the node types control, worker and etcd. " + + "Example: nodeofferings[0].node=worker&nodeofferings[0].offering=") + private Map> serviceOfferingNodeTypeMap; + + @Parameter(name = ApiConstants.NODE_TYPE_ROOT_DISK_SIZE_MAP, type = CommandType.MAP, + description = "Node type to root disk size in GB mapping, for the node types control, worker and etcd. " + + "Example: noderootdisksizes[0].node=worker&noderootdisksizes[0].size=50. A rollout only grows root disks") + private Map> rootDiskSizeNodeTypeMap; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_AFFINITY_GROUP_MAP, type = CommandType.MAP, + description = "Node type to affinity group IDs mapping, for the node types control, worker and etcd. " + + "The groups replace those of the node type. " + + "Example: nodeaffinitygroups[0].node=worker&nodeaffinitygroups[0].affinitygroup=,") + private Map> affinityGroupNodeTypeMap; + + @Parameter(name = ApiConstants.ROLLOUT, type = CommandType.BOOLEAN, + description = "Whether to restart the existing nodes that differ from the settings of their node type with them. " + + "The cluster must be Running or Alert. If false, the default, only the settings are stored") + private Boolean rollout; + + @Parameter(name = ApiConstants.CANCEL_ROLLOUT, type = CommandType.BOOLEAN, + description = "Asks the rollout in progress to stop once it is done with the nodes it is working on. Takes no other parameter") + private Boolean cancelRollout; + + @Parameter(name = ApiConstants.MAX_UNAVAILABLE, type = CommandType.INTEGER, + description = "With rollout=true, the number of worker nodes restarted at a time, 1 by default; " + + "above 1, it must be less than the number of worker nodes. " + + "A cluster with a single worker node has no worker left running while it restarts. " + + "Control and etcd nodes are always restarted one at a time, and only if the others keep the cluster running") + private Integer maxUnavailable; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Map getServiceOfferingNodeTypeMap() { + return kubernetesServiceHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + } + + /** The root disk sizes in GB, by node type */ + public Map getRootDiskSizeNodeTypeMap() { + Map mapping = new HashMap<>(); + if (MapUtils.isEmpty(rootDiskSizeNodeTypeMap)) { + return mapping; + } + for (Map entry : rootDiskSizeNodeTypeMap.values()) { + String nodeType = entry.get(VmDetailConstants.CKS_NODE_TYPE); + String size = entry.get(ApiConstants.SIZE); + if (StringUtils.isAnyEmpty(nodeType, size)) { + throw new InvalidParameterValueException(String.format("Incomplete node type to root disk size mapping: '%s' -> '%s'", nodeType, size)); + } + if (!kubernetesServiceHelper.isValidNodeType(nodeType)) { + throw new InvalidParameterValueException(String.format("The provided value '%s' for node type is invalid", nodeType)); + } + long sizeGb = NumberUtils.toLong(size, -1); + if (sizeGb < 0) { + throw new InvalidParameterValueException(String.format("The root disk size '%s' for node type %s is not a number of GB", size, nodeType)); + } + mapping.put(nodeType.toUpperCase(), sizeGb); + } + return mapping; + } + + public Map> getAffinityGroupNodeTypeMap() { + return kubernetesServiceHelper.getAffinityGroupNodeTypeMap(affinityGroupNodeTypeMap); + } + + public boolean isRollout() { + return Boolean.TRUE.equals(rollout); + } + + public boolean isCancelRollout() { + return Boolean.TRUE.equals(cancelRollout); + } + + public Integer getMaxUnavailable() { + return maxUnavailable; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_UPDATE; + } + + @Override + public String getEventDescription() { + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); + return String.format("Updating Kubernetes cluster ID: %s", cluster != null ? cluster.getUuid() : getId()); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + kubernetesClusterService.updateKubernetesCluster(this); + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 932d722de354..51f6e77b4c65 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -220,6 +220,34 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "The date when this Kubernetes cluster was created") private Date created; + @SerializedName(ApiConstants.CONTROL_ROOT_DISK_SIZE) + @Param(description = "The root disk size in GB of the control nodes, if set for the node type", since = "24.0.0") + private Long controlNodeRootDiskSize; + + @SerializedName(ApiConstants.WORKER_ROOT_DISK_SIZE) + @Param(description = "The root disk size in GB of the worker nodes, if set for the node type", since = "24.0.0") + private Long workerNodeRootDiskSize; + + @SerializedName(ApiConstants.ETCD_ROOT_DISK_SIZE) + @Param(description = "The root disk size in GB of the etcd nodes, if set for the node type", since = "24.0.0") + private Long etcdNodeRootDiskSize; + + @SerializedName(ApiConstants.CONTROL_NODES_OUT_OF_DATE) + @Param(description = "The number of control nodes whose service offering, root disk size or affinity groups differ from those set for the node type", since = "24.0.0") + private Integer controlNodesOutOfDate; + + @SerializedName(ApiConstants.WORKER_NODES_OUT_OF_DATE) + @Param(description = "The number of worker nodes whose service offering, root disk size or affinity groups differ from those set for the node type", since = "24.0.0") + private Integer workerNodesOutOfDate; + + @SerializedName(ApiConstants.ETCD_NODES_OUT_OF_DATE) + @Param(description = "The number of etcd nodes whose service offering, root disk size or affinity groups differ from those set for the node type", since = "24.0.0") + private Integer etcdNodesOutOfDate; + + @SerializedName(ApiConstants.LAST_ERROR) + @Param(description = "Why the last rollout of updateKubernetesCluster failed or was cancelled, or how many control or etcd nodes it could not restart, as their cluster can not keep running without one", since = "24.0.0") + private String lastError; + @SerializedName(ApiConstants.CONTROL_AFFINITY_GROUP_IDS) @Param(description = "The IDs of affinity groups associated with control nodes", since = "4.23.0") private String controlAffinityGroupIds; @@ -583,4 +611,32 @@ public void setEtcdAffinityGroupIds(String etcdAffinityGroupIds) { public void setEtcdAffinityGroupNames(String etcdAffinityGroupNames) { this.etcdAffinityGroupNames = etcdAffinityGroupNames; } + + public void setControlNodeRootDiskSize(Long controlNodeRootDiskSize) { + this.controlNodeRootDiskSize = controlNodeRootDiskSize; + } + + public void setWorkerNodeRootDiskSize(Long workerNodeRootDiskSize) { + this.workerNodeRootDiskSize = workerNodeRootDiskSize; + } + + public void setEtcdNodeRootDiskSize(Long etcdNodeRootDiskSize) { + this.etcdNodeRootDiskSize = etcdNodeRootDiskSize; + } + + public void setControlNodesOutOfDate(Integer controlNodesOutOfDate) { + this.controlNodesOutOfDate = controlNodesOutOfDate; + } + + public void setWorkerNodesOutOfDate(Integer workerNodesOutOfDate) { + this.workerNodesOutOfDate = workerNodesOutOfDate; + } + + public void setEtcdNodesOutOfDate(Integer etcdNodesOutOfDate) { + this.etcdNodesOutOfDate = etcdNodesOutOfDate; + } + + public void setLastError(String lastError) { + this.lastError = lastError; + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplUpdateTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplUpdateTest.java new file mode 100644 index 000000000000..ae83c71609bd --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplUpdateTest.java @@ -0,0 +1,573 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.cloudstack.affinity.AffinityGroupVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpdateKubernetesClusterCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRollingUpdateWorker; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; + +/** Tests of updateKubernetesCluster: validation, the stored settings, cancel, and the Updating state */ +@RunWith(MockitoJUnitRunner.Silent.class) +public class KubernetesClusterManagerImplUpdateTest { + + private static final long CLUSTER_ID = 1L; + private static final long CLUSTER_OFFERING = 3L; + + @Mock + KubernetesClusterDao kubernetesClusterDao; + @Mock + KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Mock + KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + @Mock + KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; + @Mock + KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Mock + ServiceOfferingDao serviceOfferingDao; + @Mock + VolumeDao volumeDao; + @Mock + DiskOfferingDao diskOfferingDao; + @Mock + VMInstanceDao vmInstanceDao; + @Mock + AffinityGroupDao affinityGroupDao; + @Mock + AffinityGroupVMMapDao affinityGroupVMMapDao; + @Mock + AsyncJobDao asyncJobDao; + @Mock + AccountManager accountManager; + @Mock + AccountDao accountDao; + + @Spy + @InjectMocks + KubernetesClusterManagerImpl manager; + + MockedStatic transactionMock; + KubernetesClusterVO cluster; + final List vmMaps = new ArrayList<>(); + + @Before + public void setUp() throws Exception { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + Field f = ConfigKey.class.getDeclaredField("_defaultValue"); + f.setAccessible(true); + f.set(KubernetesClusterService.KubernetesServiceEnabled, "true"); + transactionMock = Mockito.mockStatic(Transaction.class); + transactionMock.when(() -> Transaction.execute(Mockito.any(TransactionCallback.class))) + .thenAnswer(invocation -> ((TransactionCallback) invocation.getArgument(0)).doInTransaction(null)); + cluster = Mockito.spy(new KubernetesClusterVO()); + Mockito.doReturn(CLUSTER_ID).when(cluster).getId(); + Mockito.doReturn("cluster").when(cluster).getName(); + cluster.setState(KubernetesCluster.State.Running); + cluster.setClusterType(KubernetesCluster.ClusterType.CloudManaged); + cluster.setServiceOfferingId(CLUSTER_OFFERING); + cluster.setTemplateId(4L); + cluster.setKubernetesVersionId(5L); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(cluster); + Mockito.when(kubernetesClusterDao.lockRow(CLUSTER_ID, true)).thenReturn(cluster); + Mockito.when(kubernetesSupportedVersionDao.findById(5L)).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); + Mockito.when(kubernetesClusterVmMapDao.listByClusterId(CLUSTER_ID)).thenReturn(vmMaps); + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(Mockito.anyLong(), Mockito.anyString())).thenReturn(List.of()); + Mockito.when(affinityGroupVMMapDao.listAffinityGroupIdsByVmId(Mockito.anyLong())).thenReturn(List.of()); + Mockito.when(volumeDao.findByInstanceAndType(Mockito.anyLong(), Mockito.eq(Volume.Type.ROOT))).thenReturn(List.of()); + } + + @After + public void tearDown() { + transactionMock.close(); + CallContext.unregister(); + } + + private UpdateKubernetesClusterCmd cmd(boolean rollout, Integer maxUnavailable, Map offerings, Map rootDiskSizes, + Map> affinityGroups) { + UpdateKubernetesClusterCmd cmd = Mockito.mock(UpdateKubernetesClusterCmd.class); + Mockito.when(cmd.getId()).thenReturn(CLUSTER_ID); + Mockito.when(cmd.getActualCommandName()).thenReturn(BaseCmd.getCommandNameByClass(UpdateKubernetesClusterCmd.class)); + Mockito.when(cmd.isRollout()).thenReturn(rollout); + Mockito.when(cmd.getMaxUnavailable()).thenReturn(maxUnavailable); + Mockito.when(cmd.getServiceOfferingNodeTypeMap()).thenReturn(offerings); + Mockito.when(cmd.getRootDiskSizeNodeTypeMap()).thenReturn(rootDiskSizes); + Mockito.when(cmd.getAffinityGroupNodeTypeMap()).thenReturn(affinityGroups); + return cmd; + } + + private UpdateKubernetesClusterCmd cancelCmd() { + UpdateKubernetesClusterCmd cmd = cmd(false, null, null, null, null); + Mockito.when(cmd.isCancelRollout()).thenReturn(true); + return cmd; + } + + private void mockOffering(long id) { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.getId()).thenReturn(id); + Mockito.when(serviceOfferingDao.findById(id)).thenReturn(offering); + Mockito.doNothing().when(manager).checkServiceOfferingForNodesScale(Mockito.eq(offering), Mockito.any(), Mockito.any()); + } + + private VMInstanceVO addVm(long vmId, KubernetesServiceHelper.KubernetesClusterNodeType type, long offeringId, Long rootSizeGb) { + KubernetesClusterVmMapVO vmMap = new KubernetesClusterVmMapVO(CLUSTER_ID, vmId, type == CONTROL); + vmMap.setEtcdNode(type == ETCD); + vmMaps.add(vmMap); + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getHostName()).thenReturn("vm" + vmId); + Mockito.when(vm.getServiceOfferingId()).thenReturn(offeringId); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vmInstanceDao.findById(vmId)).thenReturn(vm); + if (rootSizeGb != null) { + VolumeVO root = Mockito.mock(VolumeVO.class); + Mockito.when(root.getSize()).thenReturn(rootSizeGb << 30); + Mockito.when(root.getDiskOfferingId()).thenReturn(50L); + Mockito.when(volumeDao.findByInstanceAndType(vmId, Volume.Type.ROOT)).thenReturn(List.of(root)); + } + return vm; + } + + private KubernetesClusterRollingUpdateWorker mockWorker() { + KubernetesClusterRollingUpdateWorker worker = Mockito.mock(KubernetesClusterRollingUpdateWorker.class); + Mockito.doReturn(worker).when(manager).createRollingUpdateWorker(Mockito.any(), Mockito.anyInt()); + Mockito.when(worker.rollout()).thenReturn(true); + return worker; + } + + // rollout=false + + @Test + public void updateWithoutRolloutOnlyStoresTheSettings() { + cluster.setState(KubernetesCluster.State.Stopped); + mockOffering(7L); + Mockito.when(accountDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(com.cloud.user.AccountVO.class)); + Mockito.when(affinityGroupDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(AffinityGroupVO.class)); + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + + Assert.assertTrue(manager.updateKubernetesCluster(cmd(false, null, Map.of(WORKER.name(), 7L), Map.of(CONTROL.name(), 40L), + Map.of(WORKER.name(), List.of(8L, 8L, 9L))))); + + Assert.assertEquals(Long.valueOf(7L), cluster.getWorkerNodeServiceOfferingId()); + // gen 1 to gen 2: the other node types keep the offering of the cluster + Assert.assertEquals(Long.valueOf(CLUSTER_OFFERING), cluster.getControlNodeServiceOfferingId()); + Assert.assertNull(cluster.getEtcdNodeServiceOfferingId()); + Assert.assertEquals(Long.valueOf(40L), cluster.getControlNodeRootDiskSize()); + Assert.assertNull(cluster.getWorkerNodeRootDiskSize()); + Mockito.verify(kubernetesClusterAffinityGroupMapDao).removeByClusterIdAndNodeType(CLUSTER_ID, WORKER.name()); + Mockito.verify(kubernetesClusterAffinityGroupMapDao, Mockito.times(2)).persist(Mockito.any()); + Mockito.verify(worker, Mockito.never()).rollout(); + } + + @Test + public void updateRefusesWhileAnotherOperationRuns() { + for (KubernetesCluster.State state : List.of(KubernetesCluster.State.Updating, KubernetesCluster.State.Scaling, KubernetesCluster.State.Destroying)) { + cluster.setState(state); + Assert.assertThrows(state.name(), InvalidParameterValueException.class, + () -> manager.updateKubernetesCluster(cmd(false, null, null, Map.of(WORKER.name(), 40L), null))); + } + Assert.assertNull(cluster.getWorkerNodeRootDiskSize()); + } + + @Test + public void rolloutNeedsRunningOrAlert() { + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + cluster.setState(KubernetesCluster.State.Stopped); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.updateKubernetesCluster(cmd(true, null, null, null, null))); + + cluster.setState(KubernetesCluster.State.Alert); + Assert.assertTrue(manager.updateKubernetesCluster(cmd(true, null, null, null, null))); + Mockito.verify(manager).createRollingUpdateWorker(cluster, 1); + Mockito.verify(worker).rollout(); + } + + @Test + public void rolloutTakesTheClusterToUpdatingBeforeStoringTheSettings() { + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + mockOffering(7L); + Mockito.doAnswer(i -> { + Assert.assertNull("stored before the cluster was Updating", cluster.getWorkerNodeServiceOfferingId()); + return null; + }).when(worker).begin(); + + Assert.assertTrue(manager.updateKubernetesCluster(cmd(true, null, Map.of(WORKER.name(), 7L), null, null))); + + InOrder order = Mockito.inOrder(worker); + order.verify(worker).begin(); + order.verify(worker).rollout(); + Assert.assertEquals(Long.valueOf(7L), cluster.getWorkerNodeServiceOfferingId()); + } + + @Test + public void rolloutReleasesTheClusterIfTheSettingsCanNotBeStored() { + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + Mockito.doThrow(new CloudRuntimeException("db down")).when(manager).convertToNodeTypeSettings(Mockito.any()); + + Assert.assertThrows(CloudRuntimeException.class, () -> manager.updateKubernetesCluster(cmd(true, null, null, null, null))); + + // the last error, of a rollout that did node work, stays + Mockito.verify(worker).abort(); + Mockito.verify(worker, Mockito.never()).release(Mockito.any()); + Mockito.verify(worker, Mockito.never()).rollout(); + } + + @Test + public void rolloutWithoutSettingsResumes() { + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + + Assert.assertTrue(manager.updateKubernetesCluster(cmd(true, null, Map.of(), Map.of(), Map.of()))); + + Mockito.verify(worker).rollout(); + } + + // gen 1 to gen 2 + + @Test + public void convertCopiesTheClusterOfferingAndCommonAffinityGroupsButNotTemplates() { + cluster.setEtcdNodeCount(3L); + cluster.setWorkerNodeServiceOfferingId(9L); + addVm(10L, WORKER, 9L, null); + addVm(11L, WORKER, 9L, null); + addVm(12L, CONTROL, CLUSTER_OFFERING, null); + addVm(13L, CONTROL, CLUSTER_OFFERING, null); + Mockito.when(affinityGroupVMMapDao.listAffinityGroupIdsByVmId(10L)).thenReturn(List.of(20L, 30L)); + Mockito.when(affinityGroupVMMapDao.listAffinityGroupIdsByVmId(11L)).thenReturn(List.of(20L, 30L)); + Mockito.when(affinityGroupVMMapDao.listAffinityGroupIdsByVmId(12L)).thenReturn(List.of(20L)); + Mockito.when(affinityGroupVMMapDao.listAffinityGroupIdsByVmId(13L)).thenReturn(List.of(21L)); + AffinityGroupVO antiAffinity = Mockito.mock(AffinityGroupVO.class); + Mockito.when(antiAffinity.getType()).thenReturn("host anti-affinity"); + AffinityGroupVO dedication = Mockito.mock(AffinityGroupVO.class); + Mockito.when(dedication.getType()).thenReturn("ExplicitDedication"); + Mockito.when(affinityGroupDao.findById(20L)).thenReturn(antiAffinity); + Mockito.when(affinityGroupDao.findById(21L)).thenReturn(antiAffinity); + Mockito.when(affinityGroupDao.findById(30L)).thenReturn(dedication); + + manager.convertToNodeTypeSettings(cluster); + + Assert.assertEquals(Long.valueOf(CLUSTER_OFFERING), cluster.getControlNodeServiceOfferingId()); + Assert.assertEquals(Long.valueOf(9L), cluster.getWorkerNodeServiceOfferingId()); + Assert.assertEquals(Long.valueOf(CLUSTER_OFFERING), cluster.getEtcdNodeServiceOfferingId()); + Assert.assertNull(cluster.getControlNodeTemplateId()); + Assert.assertNull(cluster.getWorkerNodeTemplateId()); + // the workers share group 20, the explicit dedication group left out; the control nodes share none + Mockito.verify(kubernetesClusterAffinityGroupMapDao).persist(Mockito.argThat(map -> WORKER.name().equals(map.getNodeType()) && map.getAffinityGroupId() == 20L)); + Mockito.verify(kubernetesClusterAffinityGroupMapDao, Mockito.times(1)).persist(Mockito.any()); + } + + // validation + + @Test + public void validationRefusesBadParameters() { + addVm(10L, WORKER, CLUSTER_OFFERING, 20L); + addVm(11L, WORKER, CLUSTER_OFFERING, 20L); + addVm(12L, WORKER, CLUSTER_OFFERING, 20L); + List refused = List.of( + cmd(false, null, null, null, null), + cmd(false, 1, null, Map.of(WORKER.name(), 30L), null), + cmd(true, 0, null, null, null), + cmd(true, 3, null, null, null), + cmd(false, null, null, Map.of(ETCD.name(), 30L), null), + cmd(false, null, null, Map.of(WORKER.name(), 0L), null), + // a rollout only grows disks + cmd(true, null, null, Map.of(WORKER.name(), 10L), null)); + for (UpdateKubernetesClusterCmd cmd : refused) { + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateKubernetesClusterUpdateParameters(cmd, cluster)); + } + manager.validateKubernetesClusterUpdateParameters(cmd(true, 2, null, Map.of(WORKER.name(), 30L), null), cluster); + manager.validateKubernetesClusterUpdateParameters(cmd(true, 1, null, null, null), cluster); + // stored for new nodes only + manager.validateKubernetesClusterUpdateParameters(cmd(false, null, null, Map.of(WORKER.name(), 10L), null), cluster); + } + + @Test + public void rolloutRefusesSettingsOfAControlNodeThatCanNotBeRestarted() { + addVm(10L, CONTROL, CLUSTER_OFFERING, null); + addVm(11L, WORKER, CLUSTER_OFFERING, null); + mockOffering(7L); + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + + InvalidParameterValueException e = Assert.assertThrows(InvalidParameterValueException.class, + () -> manager.updateKubernetesCluster(cmd(true, null, Map.of(CONTROL.name(), 7L), null, null))); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("The 1 control node(s)")); + Mockito.verify(worker).abort(); + Mockito.verify(worker, Mockito.never()).rollout(); + } + + @Test + public void rolloutRefusesSettingsOfTwoEtcdNodes() { + cluster.setEtcdNodeCount(2L); + addVm(10L, CONTROL, CLUSTER_OFFERING, null); + addVm(11L, CONTROL, CLUSTER_OFFERING, null); + addVm(12L, ETCD, CLUSTER_OFFERING, null); + addVm(13L, ETCD, CLUSTER_OFFERING, null); + mockOffering(7L); + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + + InvalidParameterValueException e = Assert.assertThrows(InvalidParameterValueException.class, + () -> manager.updateKubernetesCluster(cmd(true, null, Map.of(ETCD.name(), 7L), null, null))); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("The 2 etcd node(s)")); + // two control nodes with external etcd: one serves the API while the other restarts + Assert.assertTrue(manager.updateKubernetesCluster(cmd(true, null, Map.of(CONTROL.name(), 7L), null, null))); + Mockito.verify(worker).rollout(); + } + + @Test + public void rolloutLeavesAnUnrollableNodeThatDifferedAlreadyOrIsStoppedToTheRollout() { + VMInstanceVO control = addVm(10L, CONTROL, 9L, null); + addVm(11L, WORKER, CLUSTER_OFFERING, null); + mockOffering(7L); + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + + // it differed from the settings before the call: the rollout skips and reports it + Assert.assertTrue(manager.updateKubernetesCluster(cmd(true, null, Map.of(CONTROL.name(), 7L), null, null))); + + Mockito.when(control.getServiceOfferingId()).thenReturn(7L); + Mockito.when(control.getState()).thenReturn(VirtualMachine.State.Stopped); + mockOffering(8L); + Assert.assertTrue(manager.updateKubernetesCluster(cmd(true, null, Map.of(CONTROL.name(), 8L), null, null))); + Mockito.verify(worker, Mockito.times(2)).rollout(); + } + + @Test + public void rolloutAcceptsSettingsThatRestartNoUnrollableNode() { + addVm(10L, CONTROL, CLUSTER_OFFERING, null); + addVm(11L, WORKER, CLUSTER_OFFERING, null); + mockOffering(7L); + mockOffering(CLUSTER_OFFERING); + KubernetesClusterRollingUpdateWorker worker = mockWorker(); + + // the offering the control node has already, with the template size of its disk: nothing to restart it for + Assert.assertTrue(manager.updateKubernetesCluster(cmd(true, null, Map.of(CONTROL.name(), CLUSTER_OFFERING, WORKER.name(), 7L), null, null))); + Mockito.verify(worker).rollout(); + } + + @Test + public void explicitDedicationGroupIsRefusedForANodeType() { + Mockito.when(accountDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(com.cloud.user.AccountVO.class)); + AffinityGroupVO dedication = Mockito.mock(AffinityGroupVO.class); + Mockito.when(dedication.getType()).thenReturn("ExplicitDedication"); + Mockito.when(affinityGroupDao.findById(30L)).thenReturn(dedication); + + InvalidParameterValueException e = Assert.assertThrows(InvalidParameterValueException.class, + () -> manager.validateKubernetesClusterUpdateParameters(cmd(false, null, null, null, Map.of(WORKER.name(), List.of(30L))), cluster)); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("explicit dedication")); + } + + @Test + public void rolloutRefusesToGrowADiskOfAStrictOffering() { + addVm(10L, WORKER, CLUSTER_OFFERING, 20L); + DiskOfferingVO strict = Mockito.mock(DiskOfferingVO.class); + Mockito.when(strict.getDiskSizeStrictness()).thenReturn(true); + Mockito.when(diskOfferingDao.findById(50L)).thenReturn(strict); + + Assert.assertThrows(InvalidParameterValueException.class, + () -> manager.validateKubernetesClusterUpdateParameters(cmd(true, null, null, Map.of(WORKER.name(), 30L), null), cluster)); + } + + // cancel + + @Test + public void cancelMarksTheRolloutInProgress() { + cluster.setState(KubernetesCluster.State.Updating); + + Assert.assertTrue(manager.updateKubernetesCluster(cancelCmd())); + + Mockito.verify(kubernetesClusterDetailsDao).addDetail(CLUSTER_ID, KubernetesClusterManagerImpl.ROLLOUT_CANCEL_DETAIL, "true", false); + } + + @Test + public void cancelRefusesWithoutARolloutOrWithOtherParameters() { + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.updateKubernetesCluster(cancelCmd())); + cluster.setState(KubernetesCluster.State.Updating); + UpdateKubernetesClusterCmd cmd = cancelCmd(); + Mockito.when(cmd.isRollout()).thenReturn(true); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.updateKubernetesCluster(cmd)); + Mockito.verify(kubernetesClusterDetailsDao, Mockito.never()).addDetail(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean()); + } + + // the Updating state + + @Test + public void noOtherOperationCanStartWhileUpdating() { + for (KubernetesCluster.Event event : List.of(KubernetesCluster.Event.StartRequested, KubernetesCluster.Event.StopRequested, + KubernetesCluster.Event.DestroyRequested, KubernetesCluster.Event.RecoveryRequested, KubernetesCluster.Event.AutoscaleRequested, + KubernetesCluster.Event.ScaleUpRequested, KubernetesCluster.Event.ScaleDownRequested, KubernetesCluster.Event.AddNodeRequested, + KubernetesCluster.Event.RemoveNodeRequested, KubernetesCluster.Event.UpgradeRequested, KubernetesCluster.Event.UpdateRequested, + KubernetesCluster.Event.FaultsDetected)) { + Assert.assertThrows(event.name(), NoTransitionException.class, + () -> KubernetesCluster.State.getStateMachine().getNextState(KubernetesCluster.State.Updating, event)); + } + } + + @Test + public void updatingEndsInRunningOrAlert() throws NoTransitionException { + Assert.assertEquals(KubernetesCluster.State.Updating, KubernetesCluster.State.getStateMachine().getNextState(KubernetesCluster.State.Running, + KubernetesCluster.Event.UpdateRequested)); + Assert.assertEquals(KubernetesCluster.State.Updating, KubernetesCluster.State.getStateMachine().getNextState(KubernetesCluster.State.Alert, + KubernetesCluster.Event.UpdateRequested)); + Assert.assertEquals(KubernetesCluster.State.Running, KubernetesCluster.State.getStateMachine().getNextState(KubernetesCluster.State.Updating, + KubernetesCluster.Event.OperationSucceeded)); + Assert.assertEquals(KubernetesCluster.State.Alert, KubernetesCluster.State.getStateMachine().getNextState(KubernetesCluster.State.Updating, + KubernetesCluster.Event.OperationFailed)); + Assert.assertTrue(KubernetesClusterManagerImpl.OPERATION_STATES.contains(KubernetesCluster.State.Updating)); + } + + @Test + public void orphanedRolloutGoesToAlertAndRecountsCapacity() throws Exception { + cluster.setState(KubernetesCluster.State.Updating); + Mockito.when(asyncJobDao.findInstancePendingAsyncJob(ApiCommandResourceType.KubernetesCluster.toString(), CLUSTER_ID)).thenReturn(null); + Mockito.when(kubernetesClusterDao.updateState(KubernetesCluster.State.Updating, KubernetesCluster.Event.OperationFailed, + KubernetesCluster.State.Alert, cluster, null)).thenReturn(true); + Mockito.doNothing().when(manager).updateKubernetesClusterCapacity(CLUSTER_ID); + + manager.recoverOrphanedOperationState(CLUSTER_ID); + + Mockito.verify(manager).updateKubernetesClusterCapacity(CLUSTER_ID); + } + + // out of date nodes and capacity + + @Test + public void nodeOutOfDateReason() { + cluster.setWorkerNodeServiceOfferingId(7L); + cluster.setWorkerNodeRootDiskSize(30L); + VMInstanceVO upToDate = addVm(10L, WORKER, 7L, 30L); + VMInstanceVO oldOffering = addVm(11L, WORKER, CLUSTER_OFFERING, 30L); + VMInstanceVO smallDisk = addVm(12L, WORKER, 7L, 20L); + VMInstanceVO largeDisk = addVm(13L, WORKER, 7L, 50L); + VMInstanceVO otherGroups = addVm(14L, WORKER, 7L, 30L); + Mockito.when(affinityGroupVMMapDao.listAffinityGroupIdsByVmId(14L)).thenReturn(List.of(20L)); + Mockito.when(affinityGroupDao.findById(20L)).thenReturn(Mockito.mock(AffinityGroupVO.class)); + + Assert.assertNull(manager.getNodeOutOfDateReason(cluster, vmMaps.get(0), upToDate)); + Assert.assertEquals("its service offering differs", manager.getNodeOutOfDateReason(cluster, vmMaps.get(1), oldOffering)); + Assert.assertEquals("its root disk is smaller than 30 GB", manager.getNodeOutOfDateReason(cluster, vmMaps.get(2), smallDisk)); + Assert.assertNull(manager.getNodeOutOfDateReason(cluster, vmMaps.get(3), largeDisk)); + // with no groups recorded for the type, the nodes keep theirs + Assert.assertNull(manager.getNodeOutOfDateReason(cluster, vmMaps.get(4), otherGroups)); + Assert.assertEquals(Map.of(CONTROL, 0, WORKER, 2), manager.countOutOfDateNodes(cluster)); + + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(CLUSTER_ID, WORKER.name())).thenReturn(List.of(21L)); + Mockito.when(affinityGroupDao.findById(21L)).thenReturn(Mockito.mock(AffinityGroupVO.class)); + Assert.assertEquals("its affinity groups differ", manager.getNodeOutOfDateReason(cluster, vmMaps.get(4), otherGroups)); + Assert.assertEquals("its affinity groups differ", manager.getNodeOutOfDateReason(cluster, vmMaps.get(0), upToDate)); + } + + @Test + public void explicitDedicationGroupOfTheTypeIsNotADifference() { + VMInstanceVO vm = addVm(10L, WORKER, CLUSTER_OFFERING, null); + AffinityGroupVO dedication = Mockito.mock(AffinityGroupVO.class); + Mockito.when(dedication.getType()).thenReturn("ExplicitDedication"); + Mockito.when(affinityGroupDao.findById(30L)).thenReturn(dedication); + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(CLUSTER_ID, WORKER.name())).thenReturn(List.of(30L)); + + // only a dedication group recorded: the nodes keep their groups + Assert.assertNull(manager.getNodeTypeWantedAffinityGroupIds(CLUSTER_ID, WORKER)); + Assert.assertNull(manager.getNodeOutOfDateReason(cluster, vmMaps.get(0), vm)); + } + + @Test + public void explicitDedicationGroupIsNotADifference() { + VMInstanceVO vm = addVm(10L, WORKER, CLUSTER_OFFERING, null); + AffinityGroupVO dedication = Mockito.mock(AffinityGroupVO.class); + Mockito.when(dedication.getType()).thenReturn("ExplicitDedication"); + Mockito.when(affinityGroupDao.findById(30L)).thenReturn(dedication); + Mockito.when(affinityGroupVMMapDao.listAffinityGroupIdsByVmId(10L)).thenReturn(List.of(30L)); + + Assert.assertNull(manager.getNodeOutOfDateReason(cluster, vmMaps.get(0), vm)); + Assert.assertEquals(Set.of(30L), manager.getVmAffinityGroupIds(10L, true)); + } + + @Test + public void capacityIsCountedFromTheOfferingsOfAllVms() { + addVm(10L, CONTROL, 1L, null); + addVm(11L, WORKER, 2L, null); + addVm(12L, WORKER, 2L, null); + vmMaps.get(2).setExternalNode(true); + Map offerings = new HashMap<>(); + for (long id : List.of(1L, 2L)) { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.getCpu()).thenReturn((int) id * 2); + Mockito.when(offering.getRamSize()).thenReturn((int) id * 1024); + offerings.put(id, offering); + } + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenAnswer(i -> offerings.get((Long) i.getArgument(1))); + KubernetesClusterVO update = new KubernetesClusterVO(); + Mockito.when(kubernetesClusterDao.createForUpdate(CLUSTER_ID)).thenReturn(update); + + manager.updateKubernetesClusterCapacity(CLUSTER_ID); + + Assert.assertEquals(10L, update.getCores()); + Assert.assertEquals(5120L, update.getMemory()); + Mockito.verify(kubernetesClusterDao).update(CLUSTER_ID, update); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRollingUpdateWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRollingUpdateWorkerTest.java new file mode 100644 index 000000000000..f7355c3f3a80 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRollingUpdateWorkerTest.java @@ -0,0 +1,737 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; +import org.apache.cloudstack.context.CallContext; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.User; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.UserVmManager; +import com.cloud.vm.UserVmService; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.UserVmDao; + +/** + * Runs the rollout against a simulated cluster: the VMs, their offerings, and the Kubernetes nodes kubectl reports, + * with the calls made to them recorded in order. + */ +@RunWith(MockitoJUnitRunner.Silent.class) +public class KubernetesClusterRollingUpdateWorkerTest { + + private static final long CLUSTER_ID = 1L; + private static final long OLD_OFFERING = 10L; + private static final long NEW_OFFERING = 20L; + + @Mock + KubernetesClusterManagerImpl manager; + @Mock + KubernetesClusterDao kubernetesClusterDao; + @Mock + KubernetesClusterDetailsDao detailsDao; + @Mock + KubernetesClusterVmMapDao vmMapDao; + @Mock + UserVmDao userVmDao; + @Mock + UserVmService userVmService; + @Mock + UserVmManager userVmManager; + @Mock + VolumeDao volumeDao; + @Mock + AffinityGroupVMMapDao affinityGroupVMMapDao; + @Mock + KubernetesCluster cluster; + @Mock + KubernetesClusterVO clusterVO; + + /** A Kubernetes node as the simulated kubectl reports it */ + static class Kube { + boolean ready = true; + String bootId = UUID.randomUUID().toString(); + String annotation; + boolean unschedulable; + } + + final List maps = new ArrayList<>(); + final Map vms = new HashMap<>(); + final Map vmStates = new HashMap<>(); + final Map vmOfferings = new HashMap<>(); + final Map kube = new LinkedHashMap<>(); + final Map desiredOfferings = new HashMap<>(); + final List calls = new ArrayList<>(); + final Set staleAfterStart = new HashSet<>(); + final Set failingDrains = new HashSet<>(); + final Set failingStarts = new HashSet<>(); + final Set details = new HashSet<>(); + KubernetesCluster.State clusterState; + long nextVmId = 100; + + @Before + public void setUp() throws Exception { + CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); + manager.kubernetesClusterDao = kubernetesClusterDao; + manager.kubernetesClusterDetailsDao = detailsDao; + manager.kubernetesClusterVmMapDao = vmMapDao; + clusterState = KubernetesCluster.State.Updating; + Mockito.when(cluster.getId()).thenReturn(CLUSTER_ID); + Mockito.when(cluster.getName()).thenReturn("cluster"); + Mockito.when(clusterVO.getId()).thenReturn(CLUSTER_ID); + Mockito.when(clusterVO.getName()).thenReturn("cluster"); + Mockito.when(kubernetesClusterDao.findById(CLUSTER_ID)).thenReturn(clusterVO); + Mockito.when(clusterVO.getState()).thenAnswer(i -> clusterState); + Mockito.when(vmMapDao.listByClusterId(CLUSTER_ID)).thenReturn(maps); + Mockito.when(userVmDao.findById(Mockito.anyLong())).thenAnswer(i -> vms.get((Long) i.getArgument(0))); + desiredOfferings.put(CONTROL, OLD_OFFERING); + desiredOfferings.put(WORKER, NEW_OFFERING); + desiredOfferings.put(ETCD, OLD_OFFERING); + Mockito.when(manager.getNodeTypeServiceOfferingId(Mockito.any(), Mockito.any())).thenAnswer(i -> desiredOfferings.get((KubernetesClusterNodeType) i.getArgument(1))); + Mockito.when(manager.getNodeOutOfDateReason(Mockito.any(), Mockito.any(), Mockito.any())).thenAnswer(i -> { + KubernetesClusterVmMapVO vmMap = i.getArgument(1); + UserVmVO vm = i.getArgument(2); + return desiredOfferings.get(KubernetesClusterUtil.getNodeType(vmMap)).equals(vmOfferings.get(vm.getId())) ? null : "its service offering differs"; + }); + Mockito.when(manager.getNodeTypeWantedAffinityGroupIds(Mockito.anyLong(), Mockito.any())).thenReturn(null); + Mockito.when(manager.getVmAffinityGroupIds(Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(Set.of()); + Mockito.doAnswer(i -> details.add(i.getArgument(1)) ? null : null).when(detailsDao).addDetail(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean()); + Mockito.doAnswer(i -> { + details.remove((String) i.getArgument(1)); + return null; + }).when(detailsDao).removeDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.when(detailsDao.findDetail(Mockito.anyLong(), Mockito.anyString())).thenAnswer(i -> details.contains((String) i.getArgument(1)) ? + Mockito.mock(KubernetesClusterDetailsVO.class) : null); + Mockito.when(volumeDao.findByInstanceAndType(Mockito.anyLong(), Mockito.any())).thenReturn(List.of()); + Mockito.when(userVmService.stopVirtualMachine(Mockito.anyLong(), Mockito.eq(false))).thenAnswer(i -> { + long id = i.getArgument(0); + calls.add("stop:" + name(id)); + // Kubernetes keeps the node Ready for a while + vmStates.put(id, VirtualMachine.State.Stopped); + return null; + }); + Mockito.when(userVmManager.upgradeVirtualMachine(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyMap())).thenAnswer(i -> { + long id = i.getArgument(0); + calls.add("upgrade:" + name(id)); + vmOfferings.put(id, i.getArgument(1)); + return true; + }); + Mockito.doAnswer(i -> { + UserVmVO vm = i.getArgument(0); + String name = name(vm.getId()); + calls.add("start:" + name); + if (failingStarts.contains(name)) { + throw new CloudRuntimeException("insufficient capacity"); + } + vmStates.put(vm.getId(), VirtualMachine.State.Running); + if (!staleAfterStart.contains(name) && kube.containsKey(name)) { + kube.get(name).bootId = UUID.randomUUID().toString(); + kube.get(name).ready = true; + } + return null; + }).when(userVmManager).startVirtualMachine(Mockito.any(UserVmVO.class), Mockito.isNull()); + } + + @After + public void tearDown() { + CallContext.unregister(); + } + + private String name(long vmId) { + return vms.get(vmId).getHostName(); + } + + private UserVmVO addNode(String name, KubernetesClusterNodeType type, long offeringId) { + long id = nextVmId++; + KubernetesClusterVmMapVO vmMap = new KubernetesClusterVmMapVO(CLUSTER_ID, id, type == CONTROL); + vmMap.setEtcdNode(type == ETCD); + maps.add(vmMap); + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.when(vm.getId()).thenReturn(id); + Mockito.when(vm.getHostName()).thenReturn(name); + Mockito.when(vm.getState()).thenAnswer(i -> vmStates.get(id)); + Mockito.when(vm.getServiceOfferingId()).thenAnswer(i -> vmOfferings.get(id)); + vms.put(id, vm); + vmStates.put(id, VirtualMachine.State.Running); + vmOfferings.put(id, offeringId); + if (type != ETCD) { + kube.put(name, new Kube()); + } + return vm; + } + + private String kubectl(String command) { + if (KubernetesClusterRollingUpdateWorker.NODE_STATUSES_COMMAND.equals(command)) { + return "sudo: unable to resolve host\n" + kube.entrySet().stream().map(e -> String.join("\t", e.getKey(), e.getValue().ready ? "True" : "Unknown", + e.getValue().bootId, e.getValue().annotation == null ? "" : e.getValue().annotation, e.getValue().unschedulable ? "true" : "")) + .collect(Collectors.joining("\n")); + } + if (KubernetesClusterRollingUpdateWorker.ETCD_READY_COMMAND.equals(command)) { + return "ok"; + } + String[] tokens = command.split(" "); + String name = tokens[3].equals("node") ? tokens[4] : tokens[3]; + switch (tokens[2]) { + case "annotate": + kube.get(name).annotation = tokens[6].split("=")[1]; + calls.add("annotate:" + name); + break; + case "cordon": + kube.get(name).unschedulable = true; + calls.add("cordon:" + name); + break; + case "uncordon": + kube.get(name).unschedulable = false; + kube.get(name).annotation = null; + calls.add("release:" + name); + break; + default: + return null; + } + return ""; + } + + private KubernetesClusterRollingUpdateWorker newWorker(int maxUnavailable) { + KubernetesClusterRollingUpdateWorker worker = Mockito.spy(new KubernetesClusterRollingUpdateWorker(cluster, maxUnavailable, manager)); + worker.userVmDao = userVmDao; + worker.userVmService = userVmService; + worker.userVmManager = userVmManager; + worker.volumeDao = volumeDao; + worker.affinityGroupVMMapDao = affinityGroupVMMapDao; + worker.kubectlAddress = new Pair<>("10.0.0.1", 2222); + Mockito.doNothing().when(worker).init(); + Mockito.doReturn(true).when(worker).transitToOperationState(KubernetesCluster.Event.UpdateRequested); + Mockito.doNothing().when(worker).resolveKubectlAddress(Mockito.anySet()); + Mockito.doReturn(false).when(worker).sleepBeforeRetry(Mockito.anyLong()); + Mockito.doReturn(true).when(worker).isEtcdMemberHealthy(Mockito.any()); + // the worker unregisters it after each VM action + Mockito.doAnswer(i -> CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class))).when(worker).registerVmContext(Mockito.any()); + Mockito.doAnswer(i -> kubectl(i.getArgument(0))).when(worker).runKubectl(Mockito.anyString()); + Mockito.doAnswer(i -> { + String name = KubernetesClusterActionWorker.getNodeName(i.getArgument(2)); + calls.add("drain:" + name); + return failingDrains.contains(name) ? new Pair<>(false, "Cannot evict pod as it would violate the pod's disruption budget") : new Pair<>(true, ""); + }).when(worker).drainKubernetesClusterNode(Mockito.anyString(), Mockito.anyInt(), Mockito.any(), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doAnswer(i -> { + KubernetesCluster.Event event = i.getArgument(1); + calls.add("event:" + event); + clusterState = event == KubernetesCluster.Event.OperationSucceeded ? KubernetesCluster.State.Running : KubernetesCluster.State.Alert; + return true; + }).when(worker).stateTransitTo(Mockito.eq(CLUSTER_ID), Mockito.any()); + return worker; + } + + private void addStackedControlPlane(int controls) { + for (int i = 1; i <= controls; i++) { + addNode("c" + i, CONTROL, OLD_OFFERING); + } + } + + // pure decisions + + @Test + public void clampMaxUnavailableLeavesAWorkerRunning() { + Assert.assertEquals(3, KubernetesClusterRollingUpdateWorker.clampMaxUnavailable(3, 4)); + Assert.assertEquals(3, KubernetesClusterRollingUpdateWorker.clampMaxUnavailable(5, 4)); + Assert.assertEquals(1, KubernetesClusterRollingUpdateWorker.clampMaxUnavailable(2, 2)); + Assert.assertEquals(1, KubernetesClusterRollingUpdateWorker.clampMaxUnavailable(1, 1)); + Assert.assertEquals(1, KubernetesClusterRollingUpdateWorker.clampMaxUnavailable(3, 0)); + } + + @Test + public void quorumAndHealthyOthersNeeded() { + Assert.assertEquals(1, KubernetesClusterRollingUpdateWorker.quorum(1)); + Assert.assertEquals(2, KubernetesClusterRollingUpdateWorker.quorum(2)); + Assert.assertEquals(2, KubernetesClusterRollingUpdateWorker.quorum(3)); + Assert.assertEquals(3, KubernetesClusterRollingUpdateWorker.quorum(5)); + Assert.assertEquals(2, KubernetesClusterRollingUpdateWorker.healthyOthersNeeded(CONTROL, 3, false)); + Assert.assertEquals(1, KubernetesClusterRollingUpdateWorker.healthyOthersNeeded(CONTROL, 3, true)); + Assert.assertEquals(2, KubernetesClusterRollingUpdateWorker.healthyOthersNeeded(ETCD, 3, true)); + } + + @Test + public void toBatchesSplitsInOrder() { + Assert.assertEquals(List.of(List.of(1, 2), List.of(3, 4), List.of(5)), KubernetesClusterRollingUpdateWorker.toBatches(List.of(1, 2, 3, 4, 5), 2)); + Assert.assertEquals(List.of(), KubernetesClusterRollingUpdateWorker.toBatches(List.of(), 1)); + } + + @Test + public void parseNodeStatusesSkipsOtherLines() { + Map statuses = KubernetesClusterRollingUpdateWorker.parseNodeStatuses( + "sudo: unable to resolve host x\nw1\tTrue\tb1\t\t\nw2\tFalse\tb2\tb0\ttrue\n"); + Assert.assertEquals(2, statuses.size()); + Assert.assertTrue(statuses.get("w1").ready); + Assert.assertNull(statuses.get("w1").annotation); + Assert.assertFalse(statuses.get("w1").unschedulable); + Assert.assertFalse(statuses.get("w2").ready); + Assert.assertEquals("b0", statuses.get("w2").annotation); + Assert.assertTrue(statuses.get("w2").unschedulable); + } + + @Test + public void isBackNeedsReadyAndANewBootId() { + Assert.assertFalse(KubernetesClusterRollingUpdateWorker.isBack(null, "b1")); + Assert.assertFalse(KubernetesClusterRollingUpdateWorker.isBack(new KubernetesClusterRollingUpdateWorker.NodeStatus(true, "b1", "b1", true), "b1")); + Assert.assertFalse(KubernetesClusterRollingUpdateWorker.isBack(new KubernetesClusterRollingUpdateWorker.NodeStatus(false, "b2", "b1", true), "b1")); + Assert.assertFalse(KubernetesClusterRollingUpdateWorker.isBack(new KubernetesClusterRollingUpdateWorker.NodeStatus(true, null, "b1", true), "b1")); + Assert.assertTrue(KubernetesClusterRollingUpdateWorker.isBack(new KubernetesClusterRollingUpdateWorker.NodeStatus(true, "b2", "b1", true), "b1")); + } + + // selection + + @Test + public void selectNodesTakesOutOfDateStoppedAndAnnotatedNodesFirst() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + addNode("w2", WORKER, NEW_OFFERING); + addNode("w3", WORKER, NEW_OFFERING); + UserVmVO w4 = addNode("w4", WORKER, NEW_OFFERING); + UserVmVO external = addNode("x1", WORKER, OLD_OFFERING); + maps.get(maps.size() - 1).setExternalNode(true); + kube.get("w3").annotation = "b0"; + vmStates.put(w4.getId(), VirtualMachine.State.Stopped); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + List selected = worker.selectNodes(WORKER, worker.getNodeStatuses()).stream() + .map(node -> node.vm.getHostName()).collect(Collectors.toList()); + + Assert.assertEquals(List.of("w3", "w4", "w1"), selected); + Assert.assertTrue(worker.selectNodes(CONTROL, worker.getNodeStatuses()).isEmpty()); + Assert.assertNotNull(external); + } + + // the sequence + + @Test + public void rolloutAnnotatesDrainsStopsChangesStartsAndReleases() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + addNode("w2", WORKER, NEW_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + Assert.assertTrue(worker.rollout()); + + Assert.assertEquals(List.of("annotate:w1", "cordon:w1", "drain:w1", "stop:w1", "upgrade:w1", "start:w1", "release:w1", + "event:OperationSucceeded"), calls); + Assert.assertEquals(NEW_OFFERING, (long) vmOfferings.get(vms.keySet().stream().filter(id -> name(id).equals("w1")).findFirst().get())); + Assert.assertNull(kube.get("w1").annotation); + Assert.assertFalse(kube.get("w1").unschedulable); + Assert.assertEquals(KubernetesCluster.State.Running, clusterState); + Mockito.verify(manager).updateKubernetesClusterCapacity(CLUSTER_ID); + Assert.assertTrue(details.isEmpty()); + } + + @Test + public void rolloutRejectsAStaleReadyWithTheSameBootId() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + staleAfterStart.add("w1"); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(e.getMessage(), e.getMessage().contains("not Ready with a new boot ID")); + Assert.assertFalse(calls.contains("release:w1")); + // left cordoned and annotated, for the next rollout to take up first + Assert.assertTrue(kube.get("w1").unschedulable); + Assert.assertNotNull(kube.get("w1").annotation); + Assert.assertTrue(details.contains(KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL)); + } + + @Test + public void nodeCordonedBySomeoneElseIsNotAnnotatedNorUncordoned() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + kube.get("w1").unschedulable = true; + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + worker.rollout(); + + Assert.assertEquals(List.of("cordon:w1", "drain:w1", "stop:w1", "upgrade:w1", "start:w1", "event:OperationSucceeded"), calls); + Assert.assertTrue(kube.get("w1").unschedulable); + } + + @Test + public void workerBatchIsClampedToLeaveAWorkerRunning() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + addNode("w2", WORKER, OLD_OFFERING); + addNode("w3", WORKER, OLD_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(5); + + worker.rollout(); + + List stops = calls.stream().filter(call -> call.startsWith("stop:") || call.startsWith("release:")).collect(Collectors.toList()); + Assert.assertEquals(List.of("stop:w1", "stop:w2", "release:w1", "release:w2", "stop:w3", "release:w3"), stops); + } + + @Test + public void preflightRefusesWhileAnotherNodeIsDown() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + addNode("w2", WORKER, NEW_OFFERING); + kube.get("w2").ready = false; + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(e.getMessage(), e.getMessage().contains("node w2 is not Ready")); + Assert.assertFalse(calls.stream().anyMatch(call -> call.startsWith("stop:"))); + Assert.assertEquals(KubernetesCluster.State.Alert, clusterState); + } + + // quorum + + @Test + public void quorumRefusesTheControlNodeOfATwoNodeStackedControlPlane() { + addStackedControlPlane(2); + desiredOfferings.put(CONTROL, NEW_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + Assert.assertTrue(worker.rollout()); + + Mockito.verify(detailsDao).addDetail(Mockito.eq(CLUSTER_ID), Mockito.eq(KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL), + Mockito.contains("the 2 control node(s) were not restarted, as a cluster with 2 of them can not keep running without one"), Mockito.anyBoolean()); + Assert.assertTrue(calls.stream().noneMatch(call -> call.startsWith("annotate:") || call.startsWith("stop:"))); + } + + @Test + public void workersOfASingleControlNodeClusterRollWhileItsControlNodeIsReported() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + addNode("w2", WORKER, OLD_OFFERING); + desiredOfferings.put(CONTROL, NEW_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + // the control node can never be restarted, so a rerun would fail the same way: it is reported, not failed + Assert.assertTrue(worker.rollout()); + + Mockito.verify(detailsDao).addDetail(Mockito.eq(CLUSTER_ID), Mockito.eq(KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL), + Mockito.contains("the 1 control node(s) were not restarted"), Mockito.anyBoolean()); + Assert.assertEquals(List.of("stop:w1", "release:w1", "stop:w2", "release:w2"), + calls.stream().filter(call -> call.startsWith("stop:") || call.startsWith("release:")).collect(Collectors.toList())); + // the cluster is healthy, the error is recorded + Assert.assertEquals(KubernetesCluster.State.Running, clusterState); + Assert.assertTrue(details.contains(KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL)); + } + + @Test + public void etcdAndControlNodesThatCanNotBeRestartedAreBothReported() { + addStackedControlPlane(1); + addNode("e1", ETCD, OLD_OFFERING); + addNode("e2", ETCD, OLD_OFFERING); + desiredOfferings.put(CONTROL, NEW_OFFERING); + desiredOfferings.put(ETCD, NEW_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + Mockito.doReturn(true).when(worker).isEtcdOnOwnNodes(); + + Assert.assertTrue(worker.rollout()); + + Mockito.verify(detailsDao).addDetail(Mockito.eq(CLUSTER_ID), Mockito.eq(KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL), + Mockito.argThat(error -> error.contains("2 etcd node(s)") && error.contains("1 control node(s)")), Mockito.anyBoolean()); + Assert.assertTrue(calls.stream().noneMatch(call -> call.startsWith("stop:"))); + } + + @Test + public void aFailureIsReportedWithTheNodesThatWereSkipped() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + addNode("w2", WORKER, OLD_OFFERING); + desiredOfferings.put(CONTROL, NEW_OFFERING); + failingDrains.add("w1"); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(e.getMessage(), e.getMessage().contains("w1") && e.getMessage().contains("the 1 control node(s) were not restarted")); + } + + @Test + public void abortPutsTheClusterBackWithoutLookingAtItsNodes() { + addStackedControlPlane(1); + kube.get("c1").ready = false; + for (KubernetesCluster.State state : List.of(KubernetesCluster.State.Running, KubernetesCluster.State.Alert)) { + calls.clear(); + clusterState = state; + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + worker.begin(); + Mockito.doReturn(state).when(worker).getOperationStartState(); + clusterState = KubernetesCluster.State.Updating; + + worker.abort(); + + Assert.assertEquals(List.of("event:" + (state == KubernetesCluster.State.Running ? "OperationSucceeded" : "OperationFailed")), calls); + Assert.assertEquals(state, clusterState); + Mockito.verify(worker, Mockito.never()).runKubectl(Mockito.anyString()); + } + } + + @Test + public void aCancelAskedForAfterBeginIsKept() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + details.add(KubernetesClusterManagerImpl.ROLLOUT_CANCEL_DETAIL); + worker.begin(); + // asked for while the settings are stored + details.add(KubernetesClusterManagerImpl.ROLLOUT_CANCEL_DETAIL); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(e.getMessage(), e.getMessage().contains("cancelled")); + Assert.assertTrue(calls.stream().noneMatch(call -> call.startsWith("stop:"))); + } + + @Test + public void affinityGroupsOfTheTypeReplaceTheNodesAndKeepItsDedicationGroup() { + addStackedControlPlane(1); + UserVmVO w1 = addNode("w1", WORKER, OLD_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + Mockito.when(manager.getNodeTypeWantedAffinityGroupIds(CLUSTER_ID, WORKER)).thenReturn(Set.of(5L)); + Mockito.when(manager.getVmAffinityGroupIds(w1.getId(), false)).thenReturn(Set.of(4L)); + Mockito.when(manager.getVmAffinityGroupIds(w1.getId(), true)).thenReturn(Set.of(30L)); + + Assert.assertTrue(worker.rollout()); + + Mockito.verify(affinityGroupVMMapDao).updateMap(Mockito.eq(w1.getId()), Mockito.argThat(ids -> new HashSet<>(ids).equals(Set.of(5L, 30L)))); + } + + @Test + public void quorumRefusesWhenAnotherEtcdMemberIsUnhealthy() { + addStackedControlPlane(3); + desiredOfferings.put(CONTROL, NEW_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + UserVmVO c2 = vms.values().stream().filter(vm -> vm.getHostName().equals("c2")).findFirst().get(); + Mockito.doReturn(false).when(worker).isEtcdMemberHealthy(c2); + UserVmVO c3 = vms.values().stream().filter(vm -> vm.getHostName().equals("c3")).findFirst().get(); + Mockito.doReturn(false).when(worker).isEtcdMemberHealthy(c3); + + Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(calls.stream().noneMatch(call -> call.startsWith("stop:"))); + } + + @Test + public void controlNodesOfAThreeNodeStackedControlPlaneGoOneAtATime() { + addStackedControlPlane(3); + desiredOfferings.put(CONTROL, NEW_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(3); + + worker.rollout(); + + List steps = calls.stream().filter(call -> call.startsWith("stop:") || call.startsWith("release:")).collect(Collectors.toList()); + Assert.assertEquals(List.of("stop:c1", "release:c1", "stop:c2", "release:c2", "stop:c3", "release:c3"), steps); + Mockito.verify(worker).resolveKubectlAddress(Set.of(maps.get(0).getVmId())); + } + + // every exit releases, idempotently + + private List releaseAgain(KubernetesClusterRollingUpdateWorker worker, String error) { + Set detailsBefore = new HashSet<>(details); + KubernetesCluster.State stateBefore = clusterState; + int callsBefore = calls.size(); + worker.release(error); + Assert.assertEquals(stateBefore, clusterState); + Assert.assertEquals(detailsBefore, details); + return calls.subList(callsBefore, calls.size()); + } + + @Test + public void releaseAfterSuccessIsIdempotent() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + worker.rollout(); + + Assert.assertEquals(List.of(), releaseAgain(worker, null)); + Mockito.verify(manager, Mockito.times(2)).updateKubernetesClusterCapacity(CLUSTER_ID); + } + + @Test + public void releaseAfterADrainFailureUncordonsTheNodeThatNeverWentDown() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + failingDrains.add("w1"); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(calls.contains("release:w1")); + Assert.assertFalse(calls.stream().anyMatch(call -> call.startsWith("stop:"))); + Assert.assertFalse(kube.get("w1").unschedulable); + Assert.assertEquals(KubernetesCluster.State.Running, clusterState); + Assert.assertTrue(details.contains(KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL)); + Assert.assertEquals(List.of(), releaseAgain(worker, "unable to drain")); + } + + @Test + public void releaseAfterAStartFailureLeavesTheNodeCordonedAndTheClusterInAlert() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + failingStarts.add("w1"); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(e.getMessage(), e.getMessage().contains("unable to start the VM of node w1")); + Assert.assertTrue(kube.get("w1").unschedulable); + Assert.assertEquals(KubernetesCluster.State.Alert, clusterState); + Assert.assertEquals(List.of(), releaseAgain(worker, e.getMessage())); + } + + @Test + public void releaseAfterACancelStopsBetweenBatches() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + addNode("w2", WORKER, OLD_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + Mockito.doAnswer(i -> { + Object result = i.callRealMethod(); + // asked to stop while the first batch runs + details.add(KubernetesClusterManagerImpl.ROLLOUT_CANCEL_DETAIL); + return result; + }).when(worker).rollBatch(Mockito.anyList()); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(e.getMessage(), e.getMessage().contains("cancelled")); + Assert.assertTrue(calls.contains("release:w1")); + Assert.assertFalse(calls.contains("stop:w2")); + Assert.assertFalse(details.contains(KubernetesClusterManagerImpl.ROLLOUT_CANCEL_DETAIL)); + Assert.assertEquals(KubernetesCluster.State.Running, clusterState); + Assert.assertEquals(List.of(), releaseAgain(worker, "it was cancelled")); + } + + @Test + public void releaseAfterAnExceptionBeforeTheFirstBatch() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + Mockito.doThrow(new NullPointerException()).when(worker).rollNodes(Mockito.anyList()); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, worker::rollout); + + Assert.assertTrue(e.getMessage(), e.getMessage().contains("NullPointerException")); + Assert.assertEquals(KubernetesCluster.State.Running, clusterState); + Assert.assertTrue(details.contains(KubernetesClusterManagerImpl.ROLLOUT_LAST_ERROR_DETAIL)); + Assert.assertEquals(List.of(), releaseAgain(worker, e.getMessage())); + } + + @Test + public void releaseDoesNotMoveAClusterItDoesNotHold() { + addStackedControlPlane(1); + clusterState = KubernetesCluster.State.Running; + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + worker.release(null); + + Assert.assertFalse(calls.stream().anyMatch(call -> call.startsWith("event:"))); + } + + @Test + public void releaseUncordonsOnlyAnnotatedNodesThatAreBack() { + addStackedControlPlane(1); + addNode("w1", WORKER, NEW_OFFERING); + addNode("w2", WORKER, NEW_OFFERING); + addNode("w3", WORKER, NEW_OFFERING); + kube.get("w1").annotation = "old-boot"; + kube.get("w1").unschedulable = true; + kube.get("w2").annotation = kube.get("w2").bootId; + kube.get("w2").unschedulable = true; + kube.get("w3").unschedulable = true; + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + worker.release("failed"); + + Assert.assertEquals(List.of("release:w1", "event:OperationSucceeded"), calls); + Assert.assertTrue(kube.get("w2").unschedulable); + Assert.assertTrue(kube.get("w3").unschedulable); + } + + // a rerun after a failure + + @Test + public void rerunAfterAFailureTakesUpTheStoppedNodeFirst() { + addStackedControlPlane(1); + addNode("w1", WORKER, OLD_OFFERING); + addNode("w2", WORKER, OLD_OFFERING); + addNode("w3", WORKER, OLD_OFFERING); + failingStarts.add("w2"); + KubernetesClusterRollingUpdateWorker first = newWorker(1); + Assert.assertThrows(CloudRuntimeException.class, first::rollout); + Assert.assertEquals(KubernetesCluster.State.Alert, clusterState); + + failingStarts.clear(); + calls.clear(); + clusterState = KubernetesCluster.State.Updating; + KubernetesClusterRollingUpdateWorker second = newWorker(1); + second.rollout(); + + // w2 is stopped and annotated: started first, without a drain or a stop; w3 after it + Assert.assertEquals(List.of("cordon:w2", "start:w2", "release:w2", "annotate:w3", "cordon:w3", "drain:w3", "stop:w3", "upgrade:w3", "start:w3", + "release:w3", "event:OperationSucceeded"), calls); + Assert.assertEquals(KubernetesCluster.State.Running, clusterState); + } + + @Test + public void rerunReleasesANodeLeftBackButCordonedWithoutRestartingIt() { + addStackedControlPlane(1); + addNode("w1", WORKER, NEW_OFFERING); + kube.get("w1").annotation = "boot-before-crash"; + kube.get("w1").unschedulable = true; + KubernetesClusterRollingUpdateWorker worker = newWorker(1); + + worker.rollout(); + + Assert.assertEquals(List.of("release:w1", "event:OperationSucceeded"), calls); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpdateKubernetesClusterCmdTest.java b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpdateKubernetesClusterCmdTest.java new file mode 100644 index 000000000000..651c668a11bb --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpdateKubernetesClusterCmdTest.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import java.util.Map; + +import org.apache.cloudstack.api.ApiConstants; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.KubernetesServiceHelperImpl; +import com.cloud.vm.VmDetailConstants; + +public class UpdateKubernetesClusterCmdTest { + + private UpdateKubernetesClusterCmd cmd(String nodeType, String size) { + UpdateKubernetesClusterCmd cmd = new UpdateKubernetesClusterCmd(); + cmd.kubernetesServiceHelper = new KubernetesServiceHelperImpl(); + ReflectionTestUtils.setField(cmd, "rootDiskSizeNodeTypeMap", + Map.of("0", Map.of(VmDetailConstants.CKS_NODE_TYPE, nodeType, ApiConstants.SIZE, size))); + return cmd; + } + + @Test + public void rootDiskSizesAreReadByNodeType() { + Assert.assertEquals(Map.of("WORKER", 50L), cmd("worker", "50").getRootDiskSizeNodeTypeMap()); + Assert.assertTrue(new UpdateKubernetesClusterCmd().getRootDiskSizeNodeTypeMap().isEmpty()); + } + + @Test + public void badRootDiskSizesAreRefused() { + for (String[] entry : new String[][] {{"worker", "-1"}, {"worker", "big"}, {"master", "50"}, {"worker", ""}}) { + Assert.assertThrows(entry[0] + "=" + entry[1], InvalidParameterValueException.class, () -> cmd(entry[0], entry[1]).getRootDiskSizeNodeTypeMap()); + } + } + + @Test + public void rolloutAndCancelDefaultToFalse() { + UpdateKubernetesClusterCmd cmd = new UpdateKubernetesClusterCmd(); + Assert.assertFalse(cmd.isRollout()); + Assert.assertFalse(cmd.isCancelRollout()); + } +}