diff --git a/PendingReleaseNotes b/PendingReleaseNotes index c9afb25f7ed7..2b9baae6f755 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -91,3 +91,16 @@ example.ver.1 > example.ver.2: no device-mapper multipath configuration is required. A new Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase / NVMeTCPAdapter on the KVM side back the new pool type. + + * CKS: CloudManaged Kubernetes clusters now have node groups, pools of + worker nodes with their own service offering, template, Kubernetes + labels and taints and autoscaling settings, managed with the new + create, list, update, scale and deleteKubernetesClusterNodeGroup APIs. + Every cluster has a default node group holding the worker nodes of the + cluster size, created on upgrade for the existing clusters; the + scaleKubernetesCluster API applies to it. Nodes can have their own + labels and taints on top of the ones of their node group through the + new updateKubernetesClusterNode API. Autoscaling node groups other than + the default one requires a cluster autoscaler image supporting node + groups, set by the new cloud.kubernetes.cluster.autoscaler.image global + setting. diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 7485029f22cb..9a5cf4445507 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1330,6 +1330,14 @@ public class ApiConstants { public static final String WORKER_AFFINITY_GROUP_NAMES = "workeraffinitygroupnames"; public static final String ETCD_AFFINITY_GROUP_IDS = "etcdaffinitygroupids"; public static final String ETCD_AFFINITY_GROUP_NAMES = "etcdaffinitygroupnames"; + public static final String WORKER_NODES = "workernodes"; + public static final String NODE_GROUP_ID = "nodegroupid"; + public static final String NODE_GROUP_NAME = "nodegroupname"; + public static final String NODE_GROUPS = "nodegroups"; + public static final String NODE_LABELS = "nodelabels"; + public static final String NODE_TAINTS = "nodetaints"; + public static final String CLEANUP_NODE_LABELS = "cleanupnodelabels"; + public static final String CLEANUP_NODE_TAINTS = "cleanupnodetaints"; public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java index cef5cdae2f45..0055a2efa7c0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java @@ -38,6 +38,13 @@ public class KubernetesUserVmResponse extends UserVmResponse { @Param(description = "Kubernetes version of the node") private String nodeVersion; + @SerializedName(ApiConstants.NODE_GROUP_ID) + @Param(description = "The ID of the node group of the node, if any", since = "24.0") + private String nodeGroupId; + + @SerializedName(ApiConstants.NODE_GROUP_NAME) + @Param(description = "The name of the node group of the node, if any", since = "24.0") + private String nodeGroupName; public void setExternalNode(boolean externalNode) { isExternalNode = externalNode; @@ -48,4 +55,12 @@ public void setEtcdNode(boolean etcdNode) { } public void setNodeVersion(String nodeVersion) { this.nodeVersion = nodeVersion;} + + public void setNodeGroupId(String nodeGroupId) { + this.nodeGroupId = nodeGroupId; + } + + public void setNodeGroupName(String nodeGroupName) { + this.nodeGroupName = nodeGroupName; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql index 3365f1f855ba..e313074c52aa 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql @@ -239,3 +239,38 @@ CREATE TABLE IF NOT EXISTS `cloud`.`instance_boot_group_details` ( PRIMARY KEY (`id`), CONSTRAINT `fk_instance_boot_group_details__group_id` FOREIGN KEY (`boot_group_id`) REFERENCES `instance_boot_group` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- CKS node groups: pools of worker nodes with their own service offering, template, labels, taints and autoscaling settings. +-- Every CloudManaged cluster has a default node group (is_default = 1) holding the worker nodes of the cluster size, +-- the node groups of existing clusters are created by the management server on startup. +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_node_group` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(40) NOT NULL, + `cluster_id` bigint unsigned NOT NULL COMMENT 'kubernetes cluster id', + `name` varchar(255) NOT NULL, + `is_default` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'whether the node group is the default node group of the cluster', + `state` varchar(32) NOT NULL DEFAULT 'Ready' COMMENT 'state of the node group', + `service_offering_id` bigint unsigned NOT NULL COMMENT 'service offering of the node group nodes', + `template_id` bigint unsigned NOT NULL COMMENT 'template of the node group nodes', + `node_count` bigint unsigned NOT NULL DEFAULT 0 COMMENT 'number of nodes in the node group', + `labels` varchar(4096) DEFAULT NULL COMMENT 'comma separated key=value Kubernetes node labels', + `taints` varchar(4096) DEFAULT NULL COMMENT 'comma separated key[=value]:effect Kubernetes node taints', + `autoscaling_enabled` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'whether the cluster autoscaler manages the size of the node group', + `min_size` bigint unsigned DEFAULT NULL COMMENT 'minimum size of the node group when autoscaling is enabled', + `max_size` bigint unsigned DEFAULT NULL COMMENT 'maximum size of the node group when autoscaling is enabled', + `created` datetime NOT NULL, + `updated` datetime DEFAULT NULL COMMENT 'date of the last state change of the node group', + `removed` datetime DEFAULT NULL COMMENT 'date the node group was removed', + PRIMARY KEY (`id`), + CONSTRAINT `uc_kubernetes_cluster_node_group__uuid` UNIQUE (`uuid`), + CONSTRAINT `fk_kubernetes_cluster_node_group__cluster_id` FOREIGN KEY (`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_kubernetes_cluster_node_group__service_offering_id` FOREIGN KEY (`service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_kubernetes_cluster_node_group__template_id` FOREIGN KEY (`template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE, + INDEX `i_kubernetes_cluster_node_group__cluster_id`(`cluster_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map', 'node_group_id', 'bigint unsigned DEFAULT NULL COMMENT "node group of the worker node, NULL for the control, etcd and external nodes"'); +CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.kubernetes_cluster_vm_map', 'fk_kubernetes_cluster_vm_map__node_group_id', '(node_group_id)', '`kubernetes_cluster_node_group`(`id`)'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map', 'labels', 'varchar(4096) DEFAULT NULL COMMENT "comma separated key=value Kubernetes labels of the node, in addition to the labels of its node group"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map', 'taints', 'varchar(4096) DEFAULT NULL COMMENT "comma separated key[=value]:effect Kubernetes taints of the node, in addition to the taints of its node group"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'node_group_node_count', 'bigint unsigned NOT NULL DEFAULT 0 COMMENT "number of worker nodes in the node groups of the cluster other than the default one"'); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java index d11a0fc3dea5..457702a83771 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java @@ -26,4 +26,9 @@ public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD"; public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE"; public static final String EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE = "KUBERNETES.CLUSTER.AFFINITY.UPDATE"; + public static final String EVENT_KUBERNETES_CLUSTER_NODE_GROUP_CREATE = "KUBERNETES.CLUSTER.NODEGROUP.CREATE"; + public static final String EVENT_KUBERNETES_CLUSTER_NODE_GROUP_UPDATE = "KUBERNETES.CLUSTER.NODEGROUP.UPDATE"; + public static final String EVENT_KUBERNETES_CLUSTER_NODE_GROUP_SCALE = "KUBERNETES.CLUSTER.NODEGROUP.SCALE"; + public static final String EVENT_KUBERNETES_CLUSTER_NODE_GROUP_DELETE = "KUBERNETES.CLUSTER.NODEGROUP.DELETE"; + public static final String EVENT_KUBERNETES_CLUSTER_NODE_UPDATE = "KUBERNETES.CLUSTER.NODE.UPDATE"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index eb4ebb78ee48..c61b7fa29fd1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -42,6 +42,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import java.util.stream.Collectors; import javax.inject.Inject; @@ -91,6 +92,12 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpdateKubernetesClusterAffinityGroupCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.CreateKubernetesClusterNodeGroupCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.DeleteKubernetesClusterNodeGroupCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.ListKubernetesClusterNodeGroupsCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.ScaleKubernetesClusterNodeGroupCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.UpdateKubernetesClusterNodeCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.UpdateKubernetesClusterNodeGroupCmd; import org.apache.cloudstack.api.command.user.loadbalancer.AssignToLoadBalancerRuleCmd; import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd; import org.apache.cloudstack.api.command.user.loadbalancer.DeleteLoadBalancerRuleCmd; @@ -113,6 +120,8 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterNodeGroupResponse; +import org.apache.cloudstack.api.response.KubernetesClusterNodeResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.KubernetesUserVmResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -168,6 +177,7 @@ import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterDestroyWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterNodeGroupWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterScaleWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStartWorker; @@ -176,7 +186,9 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterNodeGroupDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeGroupUtil; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; import com.cloud.kubernetes.version.KubernetesVersionManagerImpl; @@ -220,7 +232,9 @@ import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.LaunchPermissionVO; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.template.TemplateApiService; import com.cloud.user.Account; @@ -312,6 +326,17 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected final static List CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER.name(), CONTROL.name(), ETCD.name()); + /** + * Locks of a Kubernetes cluster: the node groups lock serializes the start and completion of the operations on + * its node groups, the network rules lock the updates of its SSH rules and the autoscaler lock the deployments + * of its cluster autoscaler + */ + public static final String NODE_GROUPS_LOCK = "NodeGroups"; + public static final String NETWORK_RULES_LOCK = "NetworkRules"; + public static final String AUTOSCALER_LOCK = "Autoscaler"; + protected static final int KUBERNETES_CLUSTER_LOCK_TIMEOUT = 600; + protected static final String NODE_GROUP_TEMPLATE_LAUNCH_PERMISSION_DETAIL = "nodeGroupTemplateLaunchPermission"; + ScheduledExecutorService _gcExecutor; ScheduledExecutorService _stateScanner; @@ -324,6 +349,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject public KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; @Inject + public KubernetesClusterNodeGroupDao kubernetesClusterNodeGroupDao; + @Inject + protected LaunchPermissionDao launchPermissionDao; + @Inject public KubernetesSupportedVersionDao kubernetesSupportedVersionDao; @Inject protected SSHKeyPairDao sshKeyPairDao; @@ -851,6 +880,7 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes response.setMasterNodes(kubernetesCluster.getControlNodeCount()); response.setControlNodes(kubernetesCluster.getControlNodeCount()); response.setClusterSize(kubernetesCluster.getNodeCount()); + response.setWorkerNodes(kubernetesCluster.getNodeCount() + kubernetesCluster.getNodeGroupNodeCount()); VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesCluster.getTemplateId()); if (template != null) { response.setTemplateId(template.getUuid()); @@ -899,6 +929,11 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes } } + List nodeGroups = kubernetesClusterNodeGroupDao.listByClusterId(kubernetesCluster.getId()); + Map nodeGroupById = nodeGroups.stream() + .collect(Collectors.toMap(KubernetesClusterNodeGroupVO::getId, group -> group)); + response.setNodeGroups(nodeGroups.stream().map(group -> createKubernetesClusterNodeGroupResponse(group.getId())).collect(Collectors.toList())); + List vmResponses = new ArrayList<>(); List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); ResponseView userVmResponseView = ResponseView.Restricted; @@ -936,6 +971,13 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes kubernetesUserVmResponse.setExternalNode(vmMapVO.isExternalNode()); kubernetesUserVmResponse.setEtcdNode(vmMapVO.isEtcdNode()); kubernetesUserVmResponse.setNodeVersion(vmMapVO.getNodeVersion()); + if (vmMapVO.getNodeGroupId() != null) { + KubernetesClusterNodeGroupVO nodeGroup = nodeGroupById.get(vmMapVO.getNodeGroupId()); + if (nodeGroup != null) { + kubernetesUserVmResponse.setNodeGroupId(nodeGroup.getUuid()); + kubernetesUserVmResponse.setNodeGroupName(nodeGroup.getName()); + } + } vmResponses.add(kubernetesUserVmResponse); } } @@ -1079,7 +1121,13 @@ public boolean isCommandSupported(KubernetesCluster cluster, String cmdName) { BaseCmd.getCommandNameByClass(StopKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(AddNodesToKubernetesClusterCmd.class), - BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class) + BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(CreateKubernetesClusterNodeGroupCmd.class), + BaseCmd.getCommandNameByClass(ListKubernetesClusterNodeGroupsCmd.class), + BaseCmd.getCommandNameByClass(UpdateKubernetesClusterNodeGroupCmd.class), + BaseCmd.getCommandNameByClass(ScaleKubernetesClusterNodeGroupCmd.class), + BaseCmd.getCommandNameByClass(DeleteKubernetesClusterNodeGroupCmd.class), + BaseCmd.getCommandNameByClass(UpdateKubernetesClusterNodeCmd.class) ).contains(cmdName); case ExternalManaged: return Arrays.asList( @@ -1345,7 +1393,7 @@ protected void validateKubernetesClusterScaleSize(final KubernetesClusterVO kube if (clusterSize < 1) { throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled for size, %d", kubernetesCluster.getName(), clusterSize)); } - if (clusterSize + kubernetesCluster.getControlNodeCount() > maxClusterSize) { + if (clusterSize + kubernetesCluster.getControlNodeCount() + kubernetesCluster.getNodeGroupNodeCount() > maxClusterSize) { throw new InvalidParameterValueException( String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize)); } @@ -1426,10 +1474,14 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd if (maxSize <= minSize) { throw new InvalidParameterValueException("Maxsize must be greater than minsize"); } - if (maxSize + kubernetesCluster.getControlNodeCount() > maxClusterSize) { + if (maxSize + kubernetesCluster.getControlNodeCount() + kubernetesCluster.getNodeGroupNodeCount() > maxClusterSize) { throw new InvalidParameterValueException( String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize)); } + KubernetesClusterNodeGroupVO defaultNodeGroup = kubernetesClusterNodeGroupDao.findDefaultByClusterId(kubernetesCluster.getId()); + if (defaultNodeGroup != null) { + validateNodeGroupAutoscaling(kubernetesCluster, defaultNodeGroup, true, minSize, maxSize); + } } Long workerOfferingId = serviceOfferingNodeTypeMap != null ? serviceOfferingNodeTypeMap.getOrDefault(WORKER.name(), null) : null; @@ -1442,6 +1494,12 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd if (nodes == null || nodes.size() != nodeIds.size()) { throw new InvalidParameterValueException("Invalid node ids"); } + KubernetesClusterNodeGroupVO defaultNodeGroup = kubernetesClusterNodeGroupDao.findDefaultByClusterId(kubernetesCluster.getId()); + Long defaultNodeGroupId = defaultNodeGroup != null ? defaultNodeGroup.getId() : null; + if (nodes.stream().anyMatch(node -> node.isInNodeGroupOtherThan(defaultNodeGroupId))) { + throw new InvalidParameterValueException(String.format("Some of the nodes belong to a node group, use %s to remove them", + BaseCmd.getCommandNameByClass(ScaleKubernetesClusterNodeGroupCmd.class))); + } // Ensure there's always a control node long controleNodesToRemove = nodes.stream().filter(x -> x.isControlNode()).count(); if (controleNodesToRemove >= kubernetesCluster.getControlNodeCount()) { @@ -1745,6 +1803,12 @@ public KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterC final VMTemplateVO etcdNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, ETCD, clusterKubernetesVersion); final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId(), asNumber); + final String defaultNodeGroupLabels = KubernetesClusterNodeGroupUtil.labelsToString( + KubernetesClusterNodeGroupUtil.getLabelsFromApiMap(cmd.getNodeLabels())); + final List defaultNodeGroupTaintList = KubernetesClusterNodeGroupUtil.getTaintsFromApiMap(cmd.getNodeTaints()); + validateNodeGroupTaints(true, defaultNodeGroupTaintList); + final String defaultNodeGroupTaints = KubernetesClusterNodeGroupUtil.taintsToString(defaultNodeGroupTaintList); + final SecurityGroup finalSecurityGroup = securityGroup; final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { @Override @@ -1784,6 +1848,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } newCluster.setCsiEnabled(cmd.getEnableCsi()); kubernetesClusterDao.persist(newCluster); + createDefaultNodeGroup(newCluster, defaultNodeGroupLabels, defaultNodeGroupTaints, clusterSize); persistAffinityGroupMappings(newCluster.getId(), affinityGroupNodeTypeMap); addKubernetesClusterDetails(newCluster, defaultNetwork, cmd); return newCluster; @@ -2297,7 +2362,11 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Clou this); scaleWorker.setKeys(keys); scaleWorker = ComponentContext.inject(scaleWorker); - return scaleWorker.scaleCluster(); + try { + return scaleWorker.scaleCluster(); + } finally { + syncDefaultNodeGroupWithCluster(kubernetesCluster.getId()); + } } /** @@ -2493,7 +2562,11 @@ public boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd) { validateNodeAffinityGroups(validNodeIds, kubernetesCluster); KubernetesClusterAddWorker addWorker = new KubernetesClusterAddWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); addWorker = ComponentContext.inject(addWorker); - return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr(), cmd.isManualUpgrade()); + try { + return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr(), cmd.isManualUpgrade()); + } finally { + syncDefaultNodeGroupWithCluster(kubernetesCluster.getId()); + } } @Override @@ -2508,7 +2581,11 @@ public boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesCluster } KubernetesClusterRemoveWorker removeWorker = new KubernetesClusterRemoveWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); removeWorker = ComponentContext.inject(removeWorker); - return removeWorker.removeNodesFromCluster(validNodeIds); + try { + return removeWorker.removeNodesFromCluster(validNodeIds); + } finally { + syncDefaultNodeGroupWithCluster(kubernetesCluster.getId()); + } } private KubernetesClusterVO validateCluster(long clusterId) { @@ -2766,6 +2843,877 @@ public void cleanupForAccount(Account account) { deleteProjectKubernetesAccount(account); } + /** + * Runs the action holding the given lock of the Kubernetes cluster, shared by all the management servers + */ + public T executeWithKubernetesClusterLock(final long clusterId, final String purpose, final Supplier action) { + GlobalLock lock = GlobalLock.getInternLock(String.format("KubernetesCluster.%s.%d", purpose, clusterId)); + try { + if (!lock.lock(KUBERNETES_CLUSTER_LOCK_TIMEOUT)) { + throw new CloudRuntimeException(String.format("Unable to acquire the %s lock of the Kubernetes cluster with ID %d", purpose, clusterId)); + } + try { + return action.get(); + } finally { + lock.unlock(); + } + } finally { + lock.releaseRef(); + } + } + + protected long getExternalNodeCount(final long clusterId) { + return kubernetesClusterVmMapDao.listByClusterId(clusterId).stream().filter(KubernetesClusterVmMapVO::isExternalNode).count(); + } + + /** + * Creates the default node group of a CloudManaged cluster, which holds the worker nodes of the cluster size + * except the external nodes, with the worker service offering and template of the cluster + */ + protected KubernetesClusterNodeGroupVO createDefaultNodeGroup(final KubernetesClusterVO kubernetesCluster, final String labels, + final String taints, final long nodeCount) { + Long serviceOfferingId = kubernetesCluster.getWorkerNodeServiceOfferingId() != null ? + kubernetesCluster.getWorkerNodeServiceOfferingId() : kubernetesCluster.getServiceOfferingId(); + Long templateId = kubernetesCluster.getWorkerNodeTemplateId() != null ? + kubernetesCluster.getWorkerNodeTemplateId() : kubernetesCluster.getTemplateId(); + if (serviceOfferingId == null || templateId == null) { + throw new CloudRuntimeException(String.format("Unable to find the worker service offering and template of the Kubernetes cluster %s", + kubernetesCluster.getName())); + } + String name = KubernetesClusterNodeGroup.DEFAULT_NODE_GROUP_NAME; + if (kubernetesClusterNodeGroupDao.findByClusterIdAndName(kubernetesCluster.getId(), name) != null) { + name = String.format("%s-pool", KubernetesClusterNodeGroup.DEFAULT_NODE_GROUP_NAME); + } + KubernetesClusterNodeGroupVO nodeGroup = new KubernetesClusterNodeGroupVO(kubernetesCluster.getId(), name, + serviceOfferingId, templateId, labels, taints); + nodeGroup.setDefault(true); + nodeGroup.setNodeCount(nodeCount); + nodeGroup.setAutoscalingEnabled(kubernetesCluster.getAutoscalingEnabled()); + nodeGroup.setMinSize(kubernetesCluster.getMinSize()); + nodeGroup.setMaxSize(kubernetesCluster.getMaxSize()); + nodeGroup.setUpdated(new Date()); + return kubernetesClusterNodeGroupDao.persist(nodeGroup); + } + + /** + * Creates the default node group of the CloudManaged clusters created before node groups existed, with the + * existing worker nodes of the cluster size, and labels these nodes with the node group label + */ + protected void createMissingDefaultNodeGroups() { + List labelledClusterIds = new ArrayList<>(); + for (KubernetesClusterVO kubernetesCluster : kubernetesClusterDao.listAll()) { + if (!KubernetesCluster.ClusterType.CloudManaged.equals(kubernetesCluster.getClusterType()) || + Arrays.asList(KubernetesCluster.State.Destroying, KubernetesCluster.State.Destroyed).contains(kubernetesCluster.getState()) || + kubernetesClusterNodeGroupDao.findDefaultByClusterId(kubernetesCluster.getId()) != null) { + continue; + } + try { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + List vmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + List defaultNodes = vmMaps.stream().filter(vmMap -> !vmMap.isControlNode() && + !vmMap.isEtcdNode() && !vmMap.isExternalNode() && vmMap.getNodeGroupId() == null).collect(Collectors.toList()); + long nodeCount = vmMaps.isEmpty() ? kubernetesCluster.getNodeCount() : defaultNodes.size(); + KubernetesClusterNodeGroupVO nodeGroup = createDefaultNodeGroup(kubernetesCluster, null, null, nodeCount); + for (KubernetesClusterVmMapVO vmMap : defaultNodes) { + vmMap.setNodeGroupId(nodeGroup.getId()); + kubernetesClusterVmMapDao.update(vmMap.getId(), vmMap); + } + logger.info("Created the default node group {} of Kubernetes cluster {} with {} nodes", nodeGroup, kubernetesCluster, nodeCount); + } + }); + if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + labelledClusterIds.add(kubernetesCluster.getId()); + } + } catch (Exception e) { + logger.warn("Failed to create the default node group of Kubernetes cluster {}", kubernetesCluster, e); + } + } + if (labelledClusterIds.isEmpty()) { + return; + } + // Label the existing nodes in the background, the clusters may not be reachable yet + _stateScanner.schedule(new ManagedContextRunnable() { + @Override + protected void runInContext() { + for (Long clusterId : labelledClusterIds) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); + KubernetesClusterNodeGroupVO nodeGroup = kubernetesClusterNodeGroupDao.findDefaultByClusterId(clusterId); + if (kubernetesCluster == null || nodeGroup == null || !KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + continue; + } + try { + createNodeGroupWorker(kubernetesCluster, nodeGroup).updateNodeGroupLabelsAndTaints(null, null); + } catch (Exception e) { + logger.warn("Failed to label the nodes of the default node group of Kubernetes cluster {}, " + + "they will be labelled on the next update of the node group", kubernetesCluster, e); + } + } + } + }, 120, TimeUnit.SECONDS); + } + + /** + * Updates the default node group of the cluster after an operation on the worker nodes of the cluster size, + * such as scaling the cluster or adding and removing external nodes + */ + protected void syncDefaultNodeGroupWithCluster(final long clusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); + KubernetesClusterNodeGroupVO nodeGroup = kubernetesClusterNodeGroupDao.findDefaultByClusterId(clusterId); + if (kubernetesCluster == null || nodeGroup == null) { + return; + } + nodeGroup.setNodeCount(Math.max(0, kubernetesCluster.getNodeCount() - getExternalNodeCount(clusterId))); + Long serviceOfferingId = kubernetesCluster.getWorkerNodeServiceOfferingId() != null ? + kubernetesCluster.getWorkerNodeServiceOfferingId() : kubernetesCluster.getServiceOfferingId(); + if (serviceOfferingId != null) { + nodeGroup.setServiceOfferingId(serviceOfferingId); + } + nodeGroup.setAutoscalingEnabled(kubernetesCluster.getAutoscalingEnabled()); + nodeGroup.setMinSize(kubernetesCluster.getMinSize()); + nodeGroup.setMaxSize(kubernetesCluster.getMaxSize()); + kubernetesClusterNodeGroupDao.update(nodeGroup.getId(), nodeGroup); + } + + /** + * Updates the node count and service offering of the node group from its nodes, and the node count, cores and + * memory of the cluster accordingly. The cluster row is locked as several node groups can be scaled at the same time + * @param newServiceOfferingId the new service offering of the node group, null to keep the existing one + */ + public KubernetesClusterNodeGroupVO syncNodeGroupEntries(final long clusterId, final long nodeGroupId, final Long newServiceOfferingId) { + return Transaction.execute((TransactionCallback) status -> { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.lockRow(clusterId, true); + KubernetesClusterNodeGroupVO nodeGroup = kubernetesClusterNodeGroupDao.findByIdIncludingRemoved(nodeGroupId); + long oldCount = nodeGroup.getNodeCount(); + long newCount = kubernetesClusterVmMapDao.listByNodeGroupId(nodeGroupId).size(); + ServiceOfferingVO oldOffering = serviceOfferingDao.findByIdIncludingRemoved(nodeGroup.getServiceOfferingId()); + ServiceOfferingVO offering = newServiceOfferingId != null ? serviceOfferingDao.findByIdIncludingRemoved(newServiceOfferingId) : oldOffering; + + nodeGroup.setNodeCount(newCount); + nodeGroup.setServiceOfferingId(offering.getId()); + kubernetesClusterNodeGroupDao.update(nodeGroupId, nodeGroup); + + kubernetesCluster.setCores(Math.max(0, kubernetesCluster.getCores() - (long) oldOffering.getCpu() * oldCount + (long) offering.getCpu() * newCount)); + kubernetesCluster.setMemory(Math.max(0, kubernetesCluster.getMemory() - (long) oldOffering.getRamSize() * oldCount + (long) offering.getRamSize() * newCount)); + if (nodeGroup.isDefault()) { + kubernetesCluster.setNodeCount(Math.max(0, kubernetesCluster.getNodeCount() - oldCount + newCount)); + if (newServiceOfferingId != null) { + kubernetesCluster.setWorkerNodeServiceOfferingId(offering.getId()); + } + } else { + kubernetesCluster.setNodeGroupNodeCount(Math.max(0, kubernetesCluster.getNodeGroupNodeCount() - oldCount + newCount)); + } + kubernetesClusterDao.update(clusterId, kubernetesCluster); + return nodeGroup; + }); + } + + /** + * Grants the owner of the cluster the permission to deploy nodes from the default CKS template, which is owned + * by the system. The permission is revoked by the last node group operation of the cluster using it + */ + public void grantNodeGroupTemplateLaunchPermission(final long clusterId, final long templateId, final long accountId) { + executeWithKubernetesClusterLock(clusterId, NODE_GROUPS_LOCK, () -> { + if (launchPermissionDao.findByTemplateAndAccount(templateId, accountId) == null) { + launchPermissionDao.persist(new LaunchPermissionVO(templateId, accountId)); + kubernetesClusterDetailsDao.addDetail(clusterId, NODE_GROUP_TEMPLATE_LAUNCH_PERMISSION_DETAIL, String.valueOf(templateId), false); + } + return null; + }); + } + + public void revokeNodeGroupTemplateLaunchPermission(final long clusterId, final long nodeGroupId, final long templateId, final long accountId) { + executeWithKubernetesClusterLock(clusterId, NODE_GROUPS_LOCK, () -> { + boolean otherNodeGroupsScaling = kubernetesClusterNodeGroupDao.listByClusterId(clusterId).stream() + .anyMatch(nodeGroup -> nodeGroup.getId() != nodeGroupId && KubernetesClusterNodeGroup.State.Scaling.equals(nodeGroup.getState())); + if (!otherNodeGroupsScaling && kubernetesClusterDetailsDao.findDetail(clusterId, NODE_GROUP_TEMPLATE_LAUNCH_PERMISSION_DETAIL) != null) { + launchPermissionDao.removePermissions(templateId, Collections.singletonList(accountId)); + kubernetesClusterDetailsDao.removeDetail(clusterId, NODE_GROUP_TEMPLATE_LAUNCH_PERMISSION_DETAIL); + } + return null; + }); + } + + /** + * Starts an operation on a node group: the node group must not have another operation in progress and the cluster + * must be Running, or Scaling for operations on its other node groups. The cluster stays in the Scaling state + * while operations on its node groups are in progress, which prevents the operations on the cluster itself + * @param newNodeGroup whether the node group is a new one to be persisted + */ + protected KubernetesClusterNodeGroupVO startNodeGroupOperation(final long clusterId, final KubernetesClusterNodeGroupVO nodeGroup, + final KubernetesClusterNodeGroup.State operationState, final boolean newNodeGroup) { + return executeWithKubernetesClusterLock(clusterId, NODE_GROUPS_LOCK, () -> { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); + KubernetesClusterNodeGroupVO operationNodeGroup = nodeGroup; + if (!newNodeGroup) { + operationNodeGroup = kubernetesClusterNodeGroupDao.findById(nodeGroup.getId()); + if (operationNodeGroup == null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster node group ID"); + } + if (operationNodeGroup.getState().isBusy()) { + throw new PermissionDeniedException(String.format("Node group %s is in %s state, wait for the operation in progress to complete", + operationNodeGroup.getName(), operationNodeGroup.getState())); + } + } + boolean nodeGroupOperationsInProgress = kubernetesClusterNodeGroupDao.listByClusterId(clusterId).stream() + .anyMatch(group -> group.getState().isBusy()); + if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + if (!stateTransitTo(clusterId, KubernetesCluster.Event.ScaleUpRequested)) { + throw new CloudRuntimeException(String.format("Unable to start the operation on the node groups of Kubernetes cluster %s", kubernetesCluster.getName())); + } + } else if (!KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState()) || !nodeGroupOperationsInProgress) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state, node groups can only be managed for a Running cluster", + kubernetesCluster.getName(), kubernetesCluster.getState())); + } + operationNodeGroup.setState(operationState); + operationNodeGroup.setUpdated(new Date()); + if (newNodeGroup) { + return kubernetesClusterNodeGroupDao.persist(operationNodeGroup); + } + kubernetesClusterNodeGroupDao.update(operationNodeGroup.getId(), operationNodeGroup); + return operationNodeGroup; + }); + } + + /** + * Completes an operation on a node group, and moves the cluster back to the Running state when no operation is + * in progress on its node groups anymore + */ + protected void finishNodeGroupOperation(final long clusterId, final long nodeGroupId, final boolean success) { + executeWithKubernetesClusterLock(clusterId, NODE_GROUPS_LOCK, () -> { + KubernetesClusterNodeGroupVO nodeGroup = kubernetesClusterNodeGroupDao.findById(nodeGroupId); + if (nodeGroup != null) { + nodeGroup.setState(success ? KubernetesClusterNodeGroup.State.Ready : KubernetesClusterNodeGroup.State.Error); + nodeGroup.setUpdated(new Date()); + kubernetesClusterNodeGroupDao.update(nodeGroupId, nodeGroup); + } + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); + if (kubernetesCluster != null && KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState()) && + kubernetesClusterNodeGroupDao.listByClusterId(clusterId).stream().noneMatch(group -> group.getState().isBusy())) { + stateTransitTo(clusterId, KubernetesCluster.Event.OperationSucceeded); + } + return null; + }); + } + + /** + * Marks the operations on node groups which did not complete in time, e.g. due to a management server restart, + * as failed so that the node groups and their cluster can be managed again + */ + protected void failStaleNodeGroupOperations() { + long staleTime = System.currentTimeMillis() - (KubernetesClusterScaleTimeout.value() + 600) * 1000L; + for (KubernetesClusterNodeGroupVO nodeGroup : kubernetesClusterNodeGroupDao.listByStates(KubernetesClusterNodeGroup.State.Scaling, + KubernetesClusterNodeGroup.State.Updating, KubernetesClusterNodeGroup.State.Deleting)) { + if (nodeGroup.getUpdated() != null && nodeGroup.getUpdated().getTime() > staleTime) { + continue; + } + logger.warn("Operation on node group {} in state {} did not complete in time, marking it as failed", nodeGroup, nodeGroup.getState()); + try { + finishNodeGroupOperation(nodeGroup.getClusterId(), nodeGroup.getId(), false); + } catch (CloudRuntimeException e) { + logger.warn("Failed to mark the operation on node group {} as failed", nodeGroup, e); + } + } + } + + protected KubernetesClusterVO validateClusterForNodeGroupOperation(final Long clusterId, final String cmdName) { + KubernetesClusterVO kubernetesCluster = clusterId == null ? null : kubernetesClusterDao.findById(clusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + if (!isCommandSupported(kubernetesCluster, cmdName)) { + throw new InvalidParameterValueException(String.format("Node groups are not supported for an externally managed Kubernetes cluster (%s)", kubernetesCluster.getName())); + } + if (!Arrays.asList(KubernetesCluster.State.Running, KubernetesCluster.State.Scaling).contains(kubernetesCluster.getState())) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state, node groups can only be managed for a Running cluster", + kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + return kubernetesCluster; + } + + protected KubernetesClusterNodeGroupVO getNodeGroupOrThrow(final Long nodeGroupId) { + KubernetesClusterNodeGroupVO nodeGroup = nodeGroupId == null ? null : kubernetesClusterNodeGroupDao.findById(nodeGroupId); + if (nodeGroup == null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster node group ID"); + } + return nodeGroup; + } + + protected ServiceOfferingVO validateNodeGroupServiceOffering(final Long serviceOfferingId, final KubernetesClusterVO kubernetesCluster) { + ServiceOfferingVO serviceOffering = serviceOfferingId == null ? null : serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Invalid service offering ID"); + } + KubernetesSupportedVersion clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (clusterVersion == null) { + throw new CloudRuntimeException(String.format("Invalid Kubernetes version associated with Kubernetes cluster : %s", kubernetesCluster.getName())); + } + checkServiceOfferingForNodesScale(serviceOffering, kubernetesCluster, clusterVersion); + return serviceOffering; + } + + /** + * Validates the template of a node group, which defaults to the template of the worker nodes of the cluster. + * Node group nodes must run on the same hypervisor and architecture as the rest of the cluster + */ + protected VMTemplateVO validateNodeGroupTemplate(final Long templateId, final KubernetesClusterVO kubernetesCluster) { + Long nodeGroupTemplateId = templateId; + if (nodeGroupTemplateId == null) { + nodeGroupTemplateId = kubernetesCluster.getWorkerNodeTemplateId() != null ? kubernetesCluster.getWorkerNodeTemplateId() : kubernetesCluster.getTemplateId(); + } + VMTemplateVO template = templateDao.findById(nodeGroupTemplateId); + if (template == null) { + throw new InvalidParameterValueException("Invalid template ID"); + } + if (template.getId() != kubernetesCluster.getTemplateId() && !template.isForCks()) { + throw new InvalidParameterValueException(String.format("Template %s is not a CKS template", template.getName())); + } + VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId()); + if (clusterTemplate != null && (!clusterTemplate.getHypervisorType().equals(template.getHypervisorType()) || + !clusterTemplate.getArch().equals(template.getArch()))) { + throw new InvalidParameterValueException(String.format("Template %s must have the same hypervisor (%s) and architecture (%s) as the Kubernetes cluster %s", + template.getName(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch(), kubernetesCluster.getName())); + } + if (CollectionUtils.isEmpty(templateJoinDao.newTemplateView(template, kubernetesCluster.getZoneId(), true))) { + throw new InvalidParameterValueException(String.format("Template %s is not in Ready state in the zone of the Kubernetes cluster %s", + template.getName(), kubernetesCluster.getName())); + } + return template; + } + + /** + * The default node group has to accept the system pods, which have no tolerations for the user defined taints + */ + protected void validateNodeGroupTaints(final boolean isDefault, final List taints) { + if (isDefault && KubernetesClusterNodeGroupUtil.hasSchedulingTaints(taints)) { + throw new InvalidParameterValueException("The default node group has to accept the system pods, only taints with the PreferNoSchedule effect are allowed for it"); + } + } + + protected void validateNodeGroupClusterSize(final KubernetesClusterVO kubernetesCluster, final long additionalNodes) { + if (additionalNodes <= 0) { + return; + } + int maxClusterSize = KubernetesMaxClusterSize.valueIn(kubernetesCluster.getAccountId()); + if (kubernetesCluster.getControlNodeCount() + kubernetesCluster.getNodeCount() + kubernetesCluster.getNodeGroupNodeCount() + additionalNodes > maxClusterSize) { + throw new InvalidParameterValueException( + String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize)); + } + } + + /** + * Validates the size of a node group, which must contain at least one node for the default node group and stay + * within the minimum and maximum size of the node group when its autoscaling is enabled + */ + protected void validateNodeGroupSize(final boolean isDefault, final boolean autoscalingEnabled, final Long minSize, + final Long maxSize, final long size) { + if (size < (isDefault ? 1 : 0)) { + throw new InvalidParameterValueException(isDefault ? "The default node group must have at least one node" : "Node group size can not be negative"); + } + if (autoscalingEnabled && (size < minSize || size > maxSize)) { + throw new InvalidParameterValueException(String.format("Node group size %d must be within its autoscaling minimum size %d and maximum size %d", + size, minSize, maxSize)); + } + } + + /** + * Validates the autoscaling settings of a node group: the maximum size of the node group, with the size of the + * control nodes, external nodes and other node groups (their maximum size when autoscaled), can not exceed the + * maximum cluster size. The autoscaling of node groups other than the default one requires an autoscaler image + * supporting node groups + * @param nodeGroup the node group, null for a new node group + */ + protected void validateNodeGroupAutoscaling(final KubernetesClusterVO kubernetesCluster, final KubernetesClusterNodeGroupVO nodeGroup, + final boolean isDefault, final Long minSize, final Long maxSize) { + KubernetesSupportedVersion clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (clusterVersion == null || !KubernetesVersionManagerImpl.versionSupportsAutoscaling(clusterVersion)) { + throw new InvalidParameterValueException(String.format("Autoscaling requires Kubernetes Version %s or above", + KubernetesVersionManagerImpl.MINIMUN_AUTOSCALER_SUPPORTED_VERSION)); + } + ApiServiceConfiguration.validateEndpointUrl(); + if (minSize == null || maxSize == null) { + throw new InvalidParameterValueException("Autoscaling requires minsize and maxsize to be passed"); + } + long lowestMinSize = isDefault ? 1 : 0; + if (minSize < lowestMinSize) { + throw new InvalidParameterValueException(String.format("Minsize must be at least %d", lowestMinSize)); + } + if (maxSize < 1 || maxSize < minSize) { + throw new InvalidParameterValueException("Maxsize must be at least 1 and not lower than minsize"); + } + if (!isDefault && StringUtils.isBlank(KubernetesClusterAutoscalerImage.value())) { + throw new InvalidParameterValueException(String.format("Autoscaling of node groups other than the default one requires a cluster autoscaler image " + + "supporting node groups, set by the global setting %s", KubernetesClusterAutoscalerImage.key())); + } + long otherNodes = kubernetesCluster.getControlNodeCount() + getExternalNodeCount(kubernetesCluster.getId()); + for (KubernetesClusterNodeGroupVO group : kubernetesClusterNodeGroupDao.listByClusterId(kubernetesCluster.getId())) { + if (nodeGroup != null && group.getId() == nodeGroup.getId()) { + continue; + } + otherNodes += group.isAutoscalingEnabled() && group.getMaxSize() != null ? Math.max(group.getMaxSize(), group.getNodeCount()) : group.getNodeCount(); + } + int maxClusterSize = KubernetesMaxClusterSize.valueIn(kubernetesCluster.getAccountId()); + if (otherNodes + maxSize > maxClusterSize) { + throw new InvalidParameterValueException(String.format("Maximum cluster size can not exceed %d, the maximum size of the node group can be at most %d. " + + "Please contact your administrator", maxClusterSize, Math.max(0, maxClusterSize - otherNodes))); + } + } + + protected void ensureResourceLimitsForNodeGroup(final KubernetesClusterVO kubernetesCluster, final long additionalVms, + final long additionalCpu, final long additionalRamMb) { + Account owner = accountDao.findById(kubernetesCluster.getAccountId()); + try { + if (additionalVms > 0) { + resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.user_vm, additionalVms); + } + if (additionalCpu > 0) { + resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.cpu, additionalCpu); + } + if (additionalRamMb > 0) { + resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.memory, additionalRamMb); + } + } catch (ResourceAllocationException e) { + throw new CloudRuntimeException("Resource limits prevent scaling the node group: " + e.getMessage(), e); + } + } + + protected KubernetesClusterNodeGroupWorker createNodeGroupWorker(final KubernetesClusterVO kubernetesCluster, + final KubernetesClusterNodeGroupVO nodeGroup) { + KubernetesClusterNodeGroupWorker worker = new KubernetesClusterNodeGroupWorker(kubernetesCluster, nodeGroup, this); + return ComponentContext.inject(worker); + } + + /** + * @return a worker able to deploy the cluster autoscaler, which needs the API keys of the cluster owner + */ + protected KubernetesClusterNodeGroupWorker createNodeGroupAutoscalerWorker(final KubernetesClusterVO kubernetesCluster, + final KubernetesClusterNodeGroupVO nodeGroup) { + KubernetesClusterNodeGroupWorker worker = createNodeGroupWorker(kubernetesCluster, nodeGroup); + worker.setKeys(getServiceUserKeys(kubernetesCluster)); + return worker; + } + + /** + * Updates the autoscaling settings of the node group, and of the cluster for its default node group + */ + protected void setNodeGroupAutoscaling(final KubernetesClusterNodeGroupVO nodeGroup, final boolean enabled, final Long minSize, final Long maxSize) { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + nodeGroup.setAutoscalingEnabled(enabled); + nodeGroup.setMinSize(enabled ? minSize : null); + nodeGroup.setMaxSize(enabled ? maxSize : null); + kubernetesClusterNodeGroupDao.update(nodeGroup.getId(), nodeGroup); + if (nodeGroup.isDefault()) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(nodeGroup.getClusterId()); + kubernetesCluster.setAutoscalingEnabled(enabled); + kubernetesCluster.setMinSize(enabled ? minSize : null); + kubernetesCluster.setMaxSize(enabled ? maxSize : null); + kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); + } + } + }); + } + + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_GROUP_CREATE, + eventDescription = "creating Kubernetes cluster node group", async = true) + public KubernetesClusterNodeGroup createKubernetesClusterNodeGroup(CreateKubernetesClusterNodeGroupCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterVO kubernetesCluster = validateClusterForNodeGroupOperation(cmd.getClusterId(), cmd.getActualCommandName()); + final String name = StringUtils.trim(cmd.getName()); + KubernetesClusterNodeGroupUtil.validateNodeGroupName(name); + if (kubernetesClusterNodeGroupDao.findByClusterIdAndName(kubernetesCluster.getId(), name) != null) { + throw new InvalidParameterValueException(String.format("A node group with name %s already exists in the Kubernetes cluster %s", name, kubernetesCluster.getName())); + } + final Long size = cmd.getSize(); + if (size == null) { + throw new InvalidParameterValueException("Node group size must be passed"); + } + final boolean autoscalingEnabled = Boolean.TRUE.equals(cmd.isAutoscalingEnabled()); + if (!autoscalingEnabled && (cmd.getMinSize() != null || cmd.getMaxSize() != null)) { + throw new InvalidParameterValueException(String.format("%s and %s can only be passed along with %s=true", + ApiConstants.MIN_SIZE, ApiConstants.MAX_SIZE, ApiConstants.AUTOSCALING_ENABLED)); + } + if (autoscalingEnabled) { + validateNodeGroupAutoscaling(kubernetesCluster, null, false, cmd.getMinSize(), cmd.getMaxSize()); + } + validateNodeGroupSize(false, autoscalingEnabled, cmd.getMinSize(), cmd.getMaxSize(), size); + ServiceOfferingVO serviceOffering = validateNodeGroupServiceOffering(cmd.getServiceOfferingId(), kubernetesCluster); + VMTemplateVO template = validateNodeGroupTemplate(cmd.getTemplateId(), kubernetesCluster); + String labels = KubernetesClusterNodeGroupUtil.labelsToString(KubernetesClusterNodeGroupUtil.getLabelsFromApiMap(cmd.getLabels())); + String taints = KubernetesClusterNodeGroupUtil.taintsToString(KubernetesClusterNodeGroupUtil.getTaintsFromApiMap(cmd.getTaints())); + validateNodeGroupClusterSize(kubernetesCluster, size); + ensureResourceLimitsForNodeGroup(kubernetesCluster, size, serviceOffering.getCpu() * size, serviceOffering.getRamSize() * size); + + KubernetesClusterNodeGroupVO newNodeGroup = new KubernetesClusterNodeGroupVO(kubernetesCluster.getId(), + name, serviceOffering.getId(), template.getId(), labels, taints); + newNodeGroup.setAutoscalingEnabled(autoscalingEnabled); + newNodeGroup.setMinSize(autoscalingEnabled ? cmd.getMinSize() : null); + newNodeGroup.setMaxSize(autoscalingEnabled ? cmd.getMaxSize() : null); + final KubernetesClusterNodeGroupVO nodeGroup = startNodeGroupOperation(kubernetesCluster.getId(), newNodeGroup, + KubernetesClusterNodeGroup.State.Scaling, true); + boolean success = false; + try { + KubernetesClusterNodeGroupWorker worker = autoscalingEnabled ? createNodeGroupAutoscalerWorker(kubernetesCluster, nodeGroup) : + createNodeGroupWorker(kubernetesCluster, nodeGroup); + worker.scaleNodeGroup(size, null, null); + if (autoscalingEnabled) { + worker.updateClusterAutoscaler(); + } + success = true; + } catch (CloudRuntimeException e) { + // Do not keep an empty node group when none of its nodes could be provisioned + if (CollectionUtils.isEmpty(kubernetesClusterVmMapDao.listByNodeGroupId(nodeGroup.getId()))) { + kubernetesClusterNodeGroupDao.remove(nodeGroup.getId()); + } + throw e; + } finally { + finishNodeGroupOperation(kubernetesCluster.getId(), nodeGroup.getId(), success); + } + return kubernetesClusterNodeGroupDao.findById(nodeGroup.getId()); + } + + @Override + public ListResponse listKubernetesClusterNodeGroups(ListKubernetesClusterNodeGroupsCmd cmd) { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + Long clusterId = cmd.getClusterId(); + KubernetesClusterNodeGroupVO nodeGroup = null; + if (cmd.getId() != null) { + nodeGroup = getNodeGroupOrThrow(cmd.getId()); + if (clusterId != null && clusterId != nodeGroup.getClusterId()) { + throw new InvalidParameterValueException("The node group does not belong to the specified Kubernetes cluster"); + } + clusterId = nodeGroup.getClusterId(); + } + if (clusterId == null) { + throw new InvalidParameterValueException(String.format("Either %s or %s must be specified", ApiConstants.ID, ApiConstants.CLUSTER_ID)); + } + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), null, false, kubernetesCluster); + + List nodeGroups = nodeGroup != null ? List.of(nodeGroup) : + kubernetesClusterNodeGroupDao.listByClusterId(clusterId); + if (StringUtils.isNotBlank(cmd.getName())) { + nodeGroups = nodeGroups.stream().filter(group -> group.getName().equals(cmd.getName())).collect(Collectors.toList()); + } + int startIndex = cmd.getStartIndex() == null ? 0 : (int) Math.min(cmd.getStartIndex(), nodeGroups.size()); + int endIndex = cmd.getPageSizeVal() == null ? nodeGroups.size() : (int) Math.min(startIndex + cmd.getPageSizeVal(), nodeGroups.size()); + List responses = nodeGroups.subList(startIndex, endIndex).stream() + .map(group -> createKubernetesClusterNodeGroupResponse(group.getId())).collect(Collectors.toList()); + ListResponse response = new ListResponse<>(); + response.setResponses(responses, nodeGroups.size()); + return response; + } + + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_GROUP_UPDATE, + eventDescription = "updating Kubernetes cluster node group", async = true) + public KubernetesClusterNodeGroup updateKubernetesClusterNodeGroup(UpdateKubernetesClusterNodeGroupCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterNodeGroupVO nodeGroup = getNodeGroupOrThrow(cmd.getId()); + KubernetesClusterVO kubernetesCluster = validateClusterForNodeGroupOperation(nodeGroup.getClusterId(), cmd.getActualCommandName()); + boolean updateLabels = MapUtils.isNotEmpty(cmd.getLabels()) || cmd.isCleanupLabels(); + boolean updateTaints = MapUtils.isNotEmpty(cmd.getTaints()) || cmd.isCleanupTaints(); + boolean updateAutoscaling = cmd.isAutoscalingEnabled() != null || cmd.getMinSize() != null || cmd.getMaxSize() != null; + if (MapUtils.isNotEmpty(cmd.getLabels()) && cmd.isCleanupLabels()) { + throw new InvalidParameterValueException(String.format("%s can not be passed along with %s", ApiConstants.CLEANUP_NODE_LABELS, ApiConstants.NODE_LABELS)); + } + if (MapUtils.isNotEmpty(cmd.getTaints()) && cmd.isCleanupTaints()) { + throw new InvalidParameterValueException(String.format("%s can not be passed along with %s", ApiConstants.CLEANUP_NODE_TAINTS, ApiConstants.NODE_TAINTS)); + } + if (!updateLabels && !updateTaints && !updateAutoscaling) { + throw new InvalidParameterValueException("Nothing to update, labels, taints or autoscaling settings must be passed"); + } + String labels = nodeGroup.getLabels(); + if (updateLabels) { + labels = cmd.isCleanupLabels() ? null : + KubernetesClusterNodeGroupUtil.labelsToString(KubernetesClusterNodeGroupUtil.getLabelsFromApiMap(cmd.getLabels())); + } + String taints = nodeGroup.getTaints(); + if (updateTaints) { + List taintList = cmd.isCleanupTaints() ? Collections.emptyList() : + KubernetesClusterNodeGroupUtil.getTaintsFromApiMap(cmd.getTaints()); + validateNodeGroupTaints(nodeGroup.isDefault(), taintList); + taints = KubernetesClusterNodeGroupUtil.taintsToString(taintList); + } + final boolean autoscalingEnabled = cmd.isAutoscalingEnabled() != null ? cmd.isAutoscalingEnabled() : nodeGroup.isAutoscalingEnabled(); + final Long minSize = autoscalingEnabled ? (cmd.getMinSize() != null ? cmd.getMinSize() : nodeGroup.getMinSize()) : null; + final Long maxSize = autoscalingEnabled ? (cmd.getMaxSize() != null ? cmd.getMaxSize() : nodeGroup.getMaxSize()) : null; + if (updateAutoscaling) { + if (!autoscalingEnabled && (cmd.getMinSize() != null || cmd.getMaxSize() != null)) { + throw new InvalidParameterValueException(String.format("%s and %s can not be passed when autoscaling is disabled", + ApiConstants.MIN_SIZE, ApiConstants.MAX_SIZE)); + } + if (autoscalingEnabled) { + validateNodeGroupAutoscaling(kubernetesCluster, nodeGroup, nodeGroup.isDefault(), minSize, maxSize); + if (nodeGroup.getNodeCount() < minSize || nodeGroup.getNodeCount() > maxSize) { + throw new InvalidParameterValueException(String.format("The size %d of node group %s must be within the minimum size %d and maximum size %d, " + + "scale the node group first", nodeGroup.getNodeCount(), nodeGroup.getName(), minSize, maxSize)); + } + } + } + + final String oldLabels = nodeGroup.getLabels(); + final String oldTaints = nodeGroup.getTaints(); + final boolean oldAutoscalingEnabled = nodeGroup.isAutoscalingEnabled(); + final Long oldMinSize = nodeGroup.getMinSize(); + final Long oldMaxSize = nodeGroup.getMaxSize(); + nodeGroup = startNodeGroupOperation(kubernetesCluster.getId(), nodeGroup, KubernetesClusterNodeGroup.State.Updating, false); + boolean success = false; + try { + KubernetesClusterNodeGroupWorker worker = updateAutoscaling ? createNodeGroupAutoscalerWorker(kubernetesCluster, nodeGroup) : + createNodeGroupWorker(kubernetesCluster, nodeGroup); + if (updateLabels || updateTaints) { + nodeGroup.setLabels(labels); + nodeGroup.setTaints(taints); + kubernetesClusterNodeGroupDao.update(nodeGroup.getId(), nodeGroup); + // Nodes added later are registered with the new labels and taints, apply them to the existing nodes too + worker.updateNodeGroupLabelsAndTaints(oldLabels, oldTaints); + } + if (updateAutoscaling) { + setNodeGroupAutoscaling(nodeGroup, autoscalingEnabled, minSize, maxSize); + try { + worker.updateClusterAutoscaler(); + } catch (CloudRuntimeException e) { + setNodeGroupAutoscaling(nodeGroup, oldAutoscalingEnabled, oldMinSize, oldMaxSize); + throw e; + } + } + success = true; + } finally { + finishNodeGroupOperation(kubernetesCluster.getId(), nodeGroup.getId(), success); + } + return kubernetesClusterNodeGroupDao.findById(nodeGroup.getId()); + } + + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_GROUP_SCALE, + eventDescription = "scaling Kubernetes cluster node group", async = true) + public KubernetesClusterNodeGroup scaleKubernetesClusterNodeGroup(ScaleKubernetesClusterNodeGroupCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterNodeGroupVO nodeGroup = getNodeGroupOrThrow(cmd.getId()); + KubernetesClusterVO kubernetesCluster = validateClusterForNodeGroupOperation(nodeGroup.getClusterId(), cmd.getActualCommandName()); + final Long size = cmd.getSize(); + final List nodeIds = cmd.getNodeIds(); + if (size == null && CollectionUtils.isEmpty(nodeIds) && cmd.getServiceOfferingId() == null) { + throw new InvalidParameterValueException("Either size, nodeids or serviceofferingid must be passed"); + } + if (CollectionUtils.isNotEmpty(nodeIds) && (size != null || cmd.getServiceOfferingId() != null)) { + throw new InvalidParameterValueException("nodeids can not be passed along with size or serviceofferingid"); + } + List nodeGroupVms = kubernetesClusterVmMapDao.listByNodeGroupId(nodeGroup.getId()); + if (CollectionUtils.isNotEmpty(nodeIds)) { + Set nodeGroupVmIds = nodeGroupVms.stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toSet()); + if (!nodeGroupVmIds.containsAll(nodeIds)) { + throw new InvalidParameterValueException(String.format("Some of the nodes do not belong to the node group %s", nodeGroup.getName())); + } + } + + ServiceOfferingVO existingOffering = serviceOfferingDao.findByIdIncludingRemoved(nodeGroup.getServiceOfferingId()); + ServiceOfferingVO serviceOffering = null; + if (cmd.getServiceOfferingId() != null && cmd.getServiceOfferingId() != nodeGroup.getServiceOfferingId()) { + serviceOffering = validateNodeGroupServiceOffering(cmd.getServiceOfferingId(), kubernetesCluster); + if (serviceOffering.getRamSize() < existingOffering.getRamSize() || + serviceOffering.getCpu() * serviceOffering.getSpeed() < existingOffering.getCpu() * existingOffering.getSpeed()) { + throw new InvalidParameterValueException(String.format("Node group %s cannot be scaled down for service offering. Service offering %s offers lesser resources as compared to service offering %s", + nodeGroup.getName(), serviceOffering.getName(), existingOffering.getName())); + } + } + ServiceOfferingVO newOffering = serviceOffering != null ? serviceOffering : existingOffering; + long currentSize = nodeGroupVms.size(); + long newSize = CollectionUtils.isNotEmpty(nodeIds) ? currentSize - nodeIds.stream().distinct().count() : (size != null ? size : currentSize); + if (size != null || CollectionUtils.isNotEmpty(nodeIds)) { + validateNodeGroupSize(nodeGroup.isDefault(), nodeGroup.isAutoscalingEnabled(), nodeGroup.getMinSize(), nodeGroup.getMaxSize(), newSize); + } + if (newSize > currentSize) { + validateNodeGroupClusterSize(kubernetesCluster, newSize - currentSize); + validateNodeGroupTemplate(nodeGroup.getTemplateId(), kubernetesCluster); + } + long additionalCpu = (long) newOffering.getCpu() * newSize - (long) existingOffering.getCpu() * currentSize; + long additionalRam = (long) newOffering.getRamSize() * newSize - (long) existingOffering.getRamSize() * currentSize; + ensureResourceLimitsForNodeGroup(kubernetesCluster, newSize - currentSize, additionalCpu, additionalRam); + + nodeGroup = startNodeGroupOperation(kubernetesCluster.getId(), nodeGroup, KubernetesClusterNodeGroup.State.Scaling, false); + boolean success = false; + try { + createNodeGroupWorker(kubernetesCluster, nodeGroup).scaleNodeGroup(size, nodeIds, serviceOffering); + success = true; + } finally { + finishNodeGroupOperation(kubernetesCluster.getId(), nodeGroup.getId(), success); + } + return kubernetesClusterNodeGroupDao.findById(nodeGroup.getId()); + } + + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_GROUP_DELETE, + eventDescription = "deleting Kubernetes cluster node group", async = true) + public boolean deleteKubernetesClusterNodeGroup(DeleteKubernetesClusterNodeGroupCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterNodeGroupVO nodeGroup = getNodeGroupOrThrow(cmd.getId()); + KubernetesClusterVO kubernetesCluster = validateClusterForNodeGroupOperation(nodeGroup.getClusterId(), cmd.getActualCommandName()); + if (nodeGroup.isDefault()) { + throw new InvalidParameterValueException(String.format("Node group %s is the default node group of Kubernetes cluster %s and can not be deleted", + nodeGroup.getName(), kubernetesCluster.getName())); + } + final boolean autoscalingEnabled = nodeGroup.isAutoscalingEnabled(); + nodeGroup = startNodeGroupOperation(kubernetesCluster.getId(), nodeGroup, KubernetesClusterNodeGroup.State.Deleting, false); + boolean success = false; + try { + KubernetesClusterNodeGroupWorker worker = autoscalingEnabled ? createNodeGroupAutoscalerWorker(kubernetesCluster, nodeGroup) : + createNodeGroupWorker(kubernetesCluster, nodeGroup); + if (autoscalingEnabled) { + // Stop autoscaling the node group before removing its nodes + setNodeGroupAutoscaling(nodeGroup, false, null, null); + worker.updateClusterAutoscaler(); + } + success = worker.deleteNodeGroup(); + } finally { + finishNodeGroupOperation(kubernetesCluster.getId(), nodeGroup.getId(), success); + } + return success; + } + + protected KubernetesClusterNodeResponse createKubernetesClusterNodeResponse(final KubernetesClusterVmMapVO vmMap, + final KubernetesClusterNodeGroupVO nodeGroup) { + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmMap.getVmId()); + if (vm == null) { + return null; + } + KubernetesClusterNodeResponse response = new KubernetesClusterNodeResponse(); + response.setId(vm.getUuid()); + response.setName(vm.getHostName()); + response.setHostName(vm.getHostName() != null ? vm.getHostName().toLowerCase() : null); + response.setState(vm.getState() != null ? vm.getState().toString() : null); + NicVO nic = nicDao.findDefaultNicForVM(vm.getId()); + if (nic != null) { + response.setIpAddress(nic.getIPv4Address()); + } + if (nodeGroup != null) { + response.setNodeGroupId(nodeGroup.getUuid()); + response.setNodeGroupName(nodeGroup.getName()); + } + response.setLabels(KubernetesClusterNodeGroupUtil.labelsFromString(vmMap.getLabels())); + response.setTaints(KubernetesClusterNodeGroupUtil.taintsFromString(vmMap.getTaints()).stream() + .map(KubernetesClusterNodeGroupUtil.Taint::toString).collect(Collectors.toList())); + return response; + } + + @Override + @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_UPDATE, + eventDescription = "updating Kubernetes cluster node", async = true) + public KubernetesClusterNodeResponse updateKubernetesClusterNode(UpdateKubernetesClusterNodeCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + final KubernetesClusterVmMapVO vmMap = cmd.getId() == null ? null : kubernetesClusterVmMapDao.findByVmId(cmd.getId()); + if (vmMap == null || vmMap.getNodeGroupId() == null) { + throw new InvalidParameterValueException("The instance is not a node of a node group of a Kubernetes cluster"); + } + KubernetesClusterNodeGroupVO nodeGroup = getNodeGroupOrThrow(vmMap.getNodeGroupId()); + KubernetesClusterVO kubernetesCluster = validateClusterForNodeGroupOperation(vmMap.getClusterId(), cmd.getActualCommandName()); + boolean updateLabels = MapUtils.isNotEmpty(cmd.getLabels()) || cmd.isCleanupLabels(); + boolean updateTaints = MapUtils.isNotEmpty(cmd.getTaints()) || cmd.isCleanupTaints(); + if (MapUtils.isNotEmpty(cmd.getLabels()) && cmd.isCleanupLabels()) { + throw new InvalidParameterValueException(String.format("%s can not be passed along with %s", ApiConstants.CLEANUP_NODE_LABELS, ApiConstants.NODE_LABELS)); + } + if (MapUtils.isNotEmpty(cmd.getTaints()) && cmd.isCleanupTaints()) { + throw new InvalidParameterValueException(String.format("%s can not be passed along with %s", ApiConstants.CLEANUP_NODE_TAINTS, ApiConstants.NODE_TAINTS)); + } + if (!updateLabels && !updateTaints) { + throw new InvalidParameterValueException("Nothing to update, labels or taints must be passed"); + } + String labels = vmMap.getLabels(); + if (updateLabels) { + labels = cmd.isCleanupLabels() ? null : + KubernetesClusterNodeGroupUtil.labelsToString(KubernetesClusterNodeGroupUtil.getLabelsFromApiMap(cmd.getLabels())); + } + String taints = vmMap.getTaints(); + if (updateTaints) { + List taintList = cmd.isCleanupTaints() ? Collections.emptyList() : + KubernetesClusterNodeGroupUtil.getTaintsFromApiMap(cmd.getTaints()); + validateNodeGroupTaints(nodeGroup.isDefault(), taintList); + taints = KubernetesClusterNodeGroupUtil.taintsToString(taintList); + } + + final String oldLabels = vmMap.getLabels(); + final String oldTaints = vmMap.getTaints(); + nodeGroup = startNodeGroupOperation(kubernetesCluster.getId(), nodeGroup, KubernetesClusterNodeGroup.State.Updating, false); + boolean success = false; + try { + vmMap.setLabels(labels); + vmMap.setTaints(taints); + kubernetesClusterVmMapDao.update(vmMap.getId(), vmMap); + createNodeGroupWorker(kubernetesCluster, nodeGroup).updateNodeLabelsAndTaints(vmMap, oldLabels, oldTaints); + success = true; + } finally { + finishNodeGroupOperation(kubernetesCluster.getId(), nodeGroup.getId(), success); + } + return createKubernetesClusterNodeResponse(kubernetesClusterVmMapDao.findById(vmMap.getId()), nodeGroup); + } + + @Override + public KubernetesClusterNodeGroupResponse createKubernetesClusterNodeGroupResponse(long nodeGroupId) { + KubernetesClusterNodeGroupVO nodeGroup = kubernetesClusterNodeGroupDao.findByIdIncludingRemoved(nodeGroupId); + KubernetesClusterNodeGroupResponse response = new KubernetesClusterNodeGroupResponse(); + response.setId(nodeGroup.getUuid()); + response.setName(nodeGroup.getName()); + response.setDefault(nodeGroup.isDefault()); + response.setState(nodeGroup.getState() != null ? nodeGroup.getState().toString() : null); + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findByIdIncludingRemoved(nodeGroup.getClusterId()); + if (kubernetesCluster != null) { + response.setClusterId(kubernetesCluster.getUuid()); + response.setClusterName(kubernetesCluster.getName()); + } + ServiceOfferingVO offering = serviceOfferingDao.findByIdIncludingRemoved(nodeGroup.getServiceOfferingId()); + if (offering != null) { + response.setServiceOfferingId(offering.getUuid()); + response.setServiceOfferingName(offering.getName()); + response.setCpuNumber(offering.getCpu()); + response.setMemory(offering.getRamSize()); + } + VMTemplateVO template = ApiDBUtils.findTemplateById(nodeGroup.getTemplateId()); + if (template != null) { + response.setTemplateId(template.getUuid()); + response.setTemplateName(template.getName()); + } + response.setSize(nodeGroup.getNodeCount()); + response.setLabels(KubernetesClusterNodeGroupUtil.labelsFromString(nodeGroup.getLabels())); + response.setTaints(KubernetesClusterNodeGroupUtil.taintsFromString(nodeGroup.getTaints()).stream() + .map(KubernetesClusterNodeGroupUtil.Taint::toString).collect(Collectors.toList())); + response.setAutoscalingEnabled(nodeGroup.isAutoscalingEnabled()); + response.setMinSize(nodeGroup.getMinSize()); + response.setMaxSize(nodeGroup.getMaxSize()); + List vmIds = new ArrayList<>(); + List nodes = new ArrayList<>(); + for (KubernetesClusterVmMapVO vmMap : kubernetesClusterVmMapDao.listByNodeGroupId(nodeGroupId)) { + KubernetesClusterNodeResponse node = createKubernetesClusterNodeResponse(vmMap, nodeGroup); + if (node != null) { + vmIds.add(node.getId()); + nodes.add(node); + } + } + response.setVirtualMachineIds(vmIds); + response.setNodes(nodes); + response.setCreated(nodeGroup.getCreated()); + return response; + } + @Override public List> getCommands() { List> cmdList = new ArrayList>(); @@ -2785,6 +3733,12 @@ public List> getCommands() { cmdList.add(AddNodesToKubernetesClusterCmd.class); cmdList.add(RemoveNodesFromKubernetesClusterCmd.class); cmdList.add(UpdateKubernetesClusterAffinityGroupCmd.class); + cmdList.add(CreateKubernetesClusterNodeGroupCmd.class); + cmdList.add(ListKubernetesClusterNodeGroupsCmd.class); + cmdList.add(UpdateKubernetesClusterNodeGroupCmd.class); + cmdList.add(ScaleKubernetesClusterNodeGroupCmd.class); + cmdList.add(DeleteKubernetesClusterNodeGroupCmd.class); + cmdList.add(UpdateKubernetesClusterNodeCmd.class); return cmdList; } @@ -2865,6 +3819,11 @@ protected void runInContext() { } public void reallyRun() { + try { + failStaleNodeGroupOperations(); + } catch (Exception e) { + logger.warn("Failed to check the operations in progress on Kubernetes cluster node groups", e); + } try { // run through Kubernetes clusters in 'Running' state and ensure all the VM's are Running in the cluster List runningKubernetesClusters = kubernetesClusterDao.findManagedKubernetesClustersInState(KubernetesCluster.State.Running); @@ -3001,6 +3960,8 @@ public boolean start() { getProjectKubernetesAccountRole(); + createMissingDefaultNodeGroups(); + _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS); _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS); @@ -3089,7 +4050,8 @@ public ConfigKey[] getConfigKeys() { KubernetesControlNodeInstallReattempts, KubernetesWorkerNodeInstallAttemptWait, KubernetesWorkerNodeInstallReattempts, - KubernetesEtcdNodeStartPort + KubernetesEtcdNodeStartPort, + KubernetesClusterAutoscalerImage }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNodeGroup.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNodeGroup.java new file mode 100644 index 000000000000..cb1644318fc3 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNodeGroup.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import java.util.Date; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +/** + * KubernetesClusterNodeGroup describes a pool of worker nodes of a CloudManaged Kubernetes cluster. + * All the nodes of a node group share the same service offering, template, Kubernetes node labels and taints, + * and the size of a node group can be managed by the cluster autoscaler within its minimum and maximum size. + * Every CloudManaged cluster has a default node group holding the worker nodes of the cluster size, which can not be + * deleted and keeps at least one schedulable node for the system pods. External nodes do not belong to any node group. + */ +public interface KubernetesClusterNodeGroup extends InternalIdentity, Identity { + + /** + * Label set on every node of a node group, its value is the name of the node group + */ + String NODE_GROUP_LABEL = "cloudstack.apache.org/nodegroup"; + + /** + * Name of the default node group of a cluster + */ + String DEFAULT_NODE_GROUP_NAME = "default"; + + enum State { + Ready("The node group is ready, its nodes can be used"), + Scaling("Nodes are being added to or removed from the node group, or its service offering is being changed"), + Updating("The labels, taints or autoscaling settings of the node group are being updated"), + Deleting("The node group is being deleted"), + Error("The last operation on the node group failed, it can be retried"); + + private final String description; + + State(String description) { + this.description = description; + } + + public String getDescription() { + return description; + } + + /** + * @return whether an operation is in progress on a node group in this state + */ + public boolean isBusy() { + return this == Scaling || this == Updating || this == Deleting; + } + } + + long getClusterId(); + + String getName(); + + boolean isDefault(); + + State getState(); + + long getServiceOfferingId(); + + long getTemplateId(); + + long getNodeCount(); + + String getLabels(); + + String getTaints(); + + boolean isAutoscalingEnabled(); + + Long getMinSize(); + + Long getMaxSize(); + + Date getCreated(); + + Date getUpdated(); + + Date getRemoved(); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNodeGroupVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNodeGroupVO.java new file mode 100644 index 000000000000..216de3ef569b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterNodeGroupVO.java @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "kubernetes_cluster_node_group") +public class KubernetesClusterNodeGroupVO implements KubernetesClusterNodeGroup { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "cluster_id") + private long clusterId; + + @Column(name = "name") + private String name; + + @Column(name = "is_default") + private boolean defaultGroup; + + @Column(name = "state") + @Enumerated(value = EnumType.STRING) + private State state; + + @Column(name = "service_offering_id") + private long serviceOfferingId; + + @Column(name = "template_id") + private long templateId; + + @Column(name = "node_count") + private long nodeCount; + + @Column(name = "labels", length = 4096) + private String labels; + + @Column(name = "taints", length = 4096) + private String taints; + + @Column(name = "autoscaling_enabled") + private boolean autoscalingEnabled; + + @Column(name = "min_size") + private Long minSize; + + @Column(name = "max_size") + private Long maxSize; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = "updated") + @Temporal(value = TemporalType.TIMESTAMP) + private Date updated; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + public KubernetesClusterNodeGroupVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public KubernetesClusterNodeGroupVO(long clusterId, String name, long serviceOfferingId, long templateId, + String labels, String taints) { + this(); + this.clusterId = clusterId; + this.name = name; + this.serviceOfferingId = serviceOfferingId; + this.templateId = templateId; + this.labels = labels; + this.taints = taints; + this.nodeCount = 0; + this.state = State.Ready; + } + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public long getClusterId() { + return clusterId; + } + + @Override + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public boolean isDefault() { + return defaultGroup; + } + + /** + * Same as {@link #isDefault()}, named after the field for the search builders + */ + public boolean isDefaultGroup() { + return defaultGroup; + } + + public void setDefault(boolean defaultGroup) { + this.defaultGroup = defaultGroup; + } + + @Override + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } + + @Override + public long getServiceOfferingId() { + return serviceOfferingId; + } + + public void setServiceOfferingId(long serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + @Override + public long getTemplateId() { + return templateId; + } + + public void setTemplateId(long templateId) { + this.templateId = templateId; + } + + @Override + public long getNodeCount() { + return nodeCount; + } + + public void setNodeCount(long nodeCount) { + this.nodeCount = nodeCount; + } + + @Override + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + @Override + public String getTaints() { + return taints; + } + + public void setTaints(String taints) { + this.taints = taints; + } + + @Override + public boolean isAutoscalingEnabled() { + return autoscalingEnabled; + } + + public void setAutoscalingEnabled(boolean autoscalingEnabled) { + this.autoscalingEnabled = autoscalingEnabled; + } + + @Override + public Long getMinSize() { + return minSize; + } + + public void setMinSize(Long minSize) { + this.minSize = minSize; + } + + @Override + public Long getMaxSize() { + return maxSize; + } + + public void setMaxSize(Long maxSize) { + this.maxSize = maxSize; + } + + @Override + public Date getCreated() { + return created; + } + + @Override + public Date getUpdated() { + return updated; + } + + public void setUpdated(Date updated) { + this.updated = updated; + } + + @Override + public Date getRemoved() { + return removed; + } + + @Override + public String toString() { + return String.format("KubernetesClusterNodeGroup %s", + ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "name", "clusterId", "defaultGroup", "state")); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 1d19127dc54a..f11107d648b8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -34,7 +34,15 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpdateKubernetesClusterAffinityGroupCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.CreateKubernetesClusterNodeGroupCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.DeleteKubernetesClusterNodeGroupCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.ListKubernetesClusterNodeGroupsCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.ScaleKubernetesClusterNodeGroupCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.UpdateKubernetesClusterNodeCmd; +import org.apache.cloudstack.api.command.user.kubernetes.nodegroup.UpdateKubernetesClusterNodeGroupCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterNodeGroupResponse; +import org.apache.cloudstack.api.response.KubernetesClusterNodeResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.RemoveVirtualMachinesFromKubernetesClusterResponse; @@ -144,6 +152,17 @@ public interface KubernetesClusterService extends PluggableService, Configurable true, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesClusterAutoscalerImage = new ConfigKey("Advanced", String.class, + "cloud.kubernetes.cluster.autoscaler.image", + "", + "Container image of the cluster autoscaler deployed in the Kubernetes clusters with autoscaling enabled, instead of the image " + + "of the autoscaler manifest of the Kubernetes version ISO. The cluster autoscaler gets one --nodes=:: argument " + + "per node group with autoscaling enabled, where is the ID of the cluster for its default node group and the ID of the " + + "node group for the other node groups. The image must support node groups to enable autoscaling for node groups other " + + "than the default one, the upstream image only supports the default node group", + true, + KubernetesServiceEnabled.key()); + KubernetesCluster findById(final Long id); KubernetesCluster createUnmanagedKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException; @@ -185,4 +204,18 @@ public interface KubernetesClusterService extends PluggableService, Configurable boolean isDirectAccess(Network network); void cleanupForAccount(Account account); + + KubernetesClusterNodeGroup createKubernetesClusterNodeGroup(CreateKubernetesClusterNodeGroupCmd cmd) throws CloudRuntimeException; + + ListResponse listKubernetesClusterNodeGroups(ListKubernetesClusterNodeGroupsCmd cmd); + + KubernetesClusterNodeGroup updateKubernetesClusterNodeGroup(UpdateKubernetesClusterNodeGroupCmd cmd) throws CloudRuntimeException; + + KubernetesClusterNodeGroup scaleKubernetesClusterNodeGroup(ScaleKubernetesClusterNodeGroupCmd cmd) throws CloudRuntimeException; + + boolean deleteKubernetesClusterNodeGroup(DeleteKubernetesClusterNodeGroupCmd cmd) throws CloudRuntimeException; + + KubernetesClusterNodeGroupResponse createKubernetesClusterNodeGroupResponse(long nodeGroupId); + + KubernetesClusterNodeResponse updateKubernetesClusterNode(UpdateKubernetesClusterNodeCmd cmd) throws CloudRuntimeException; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 7dfd0043e320..b4c9027e77ab 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -148,6 +148,9 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "csi_enabled") private boolean csiEnabled; + @Column(name = "node_group_node_count") + private long nodeGroupNodeCount; + @Override public long getId() { return id; @@ -265,9 +268,21 @@ public void setNodeCount(long nodeCount) { this.nodeCount = nodeCount; } + /** + * @return total number of worker nodes in all the node groups of the cluster. + * These nodes are not included in {@link #getNodeCount()}, which only counts the default worker pool + */ + public long getNodeGroupNodeCount() { + return nodeGroupNodeCount; + } + + public void setNodeGroupNodeCount(long nodeGroupNodeCount) { + this.nodeGroupNodeCount = nodeGroupNodeCount; + } + @Override public long getTotalNodeCount() { - return this.controlNodeCount + this.nodeCount + this.getEtcdNodeCount(); + return this.controlNodeCount + this.nodeCount + this.nodeGroupNodeCount + this.getEtcdNodeCount(); } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java index 6c45c63e16d8..669a911dd879 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java @@ -54,6 +54,15 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap { @Column(name = "kubernetes_node_version") String nodeVersion; + @Column(name = "node_group_id") + Long nodeGroupId; + + @Column(name = "labels", length = 4096) + String labels; + + @Column(name = "taints", length = 4096) + String taints; + public KubernetesClusterVmMapVO() { } @@ -127,4 +136,41 @@ public String getNodeVersion() { public void setNodeVersion(String nodeVersion) { this.nodeVersion = nodeVersion; } + + public Long getNodeGroupId() { + return nodeGroupId; + } + + public void setNodeGroupId(Long nodeGroupId) { + this.nodeGroupId = nodeGroupId; + } + + /** + * @return the Kubernetes labels of the node, in addition to the labels of its node group + */ + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + /** + * @return the Kubernetes taints of the node, in addition to the taints of its node group + */ + public String getTaints() { + return taints; + } + + public void setTaints(String taints) { + this.taints = taints; + } + + /** + * @return whether the node is a worker node of a node group other than the given (default) node group + */ + public boolean isInNodeGroupOtherThan(Long nodeGroupId) { + return this.nodeGroupId != null && !this.nodeGroupId.equals(nodeGroupId); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 07c9f12a50e2..d85ab6c281eb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -94,11 +94,14 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroupVO; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterNodeGroupDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeGroupUtil; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; @@ -232,6 +235,7 @@ public class KubernetesClusterActionWorker { protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; protected KubernetesClusterDetailsDao kubernetesClusterDetailsDao; protected KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; + protected KubernetesClusterNodeGroupDao kubernetesClusterNodeGroupDao; protected KubernetesSupportedVersionDao kubernetesSupportedVersionDao; protected KubernetesCluster kubernetesCluster; @@ -286,6 +290,7 @@ protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluste this.kubernetesClusterDetailsDao = clusterManager.kubernetesClusterDetailsDao; this.kubernetesClusterVmMapDao = clusterManager.kubernetesClusterVmMapDao; this.kubernetesClusterAffinityGroupMapDao = clusterManager.kubernetesClusterAffinityGroupMapDao; + this.kubernetesClusterNodeGroupDao = clusterManager.kubernetesClusterNodeGroupDao; this.kubernetesSupportedVersionDao = clusterManager.kubernetesSupportedVersionDao; this.manager = clusterManager; } @@ -416,6 +421,30 @@ protected File getManagementServerSshPublicKeyFile() { return new File(keyFile); } + /** + * @return the node group of the worker nodes provisioned by this worker, the default node group of the cluster + * unless overridden. Null for clusters without node groups + */ + protected KubernetesClusterNodeGroupVO getWorkerNodeGroup() { + return kubernetesClusterNodeGroupDao.findDefaultByClusterId(kubernetesCluster.getId()); + } + + /** + * @return the ID of the node group to which the worker nodes provisioned by this worker belong + */ + protected Long getWorkerNodeGroupId() { + KubernetesClusterNodeGroupVO nodeGroup = getWorkerNodeGroup(); + return nodeGroup != null ? nodeGroup.getId() : null; + } + + /** + * @return additional kubelet arguments (node labels, taints) for the worker nodes provisioned by this worker + */ + protected String getWorkerNodeKubeletExtraArgs() { + KubernetesClusterNodeGroupVO nodeGroup = getWorkerNodeGroup(); + return nodeGroup != null ? KubernetesClusterNodeGroupUtil.getKubeletExtraArgs(nodeGroup) : null; + } + protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode, boolean isExternalNode, boolean isEtcdNode, boolean markForManualUpgrade) { @@ -427,6 +456,9 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { newClusterVmMap.setExternalNode(isExternalNode); newClusterVmMap.setManualUpgrade(markForManualUpgrade); newClusterVmMap.setEtcdNode(isEtcdNode); + if (!isControlNode && !isEtcdNode && !isExternalNode) { + newClusterVmMap.setNodeGroupId(getWorkerNodeGroupId()); + } if (!isEtcdNode) { newClusterVmMap.setNodeVersion(kubernetesVersion.getSemanticVersion()); } @@ -1039,6 +1071,7 @@ public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIs final String routerIpKey = "{{ k8s.vr.iso.mounted.ip }}"; final String installWaitTime = "{{ k8s.install.wait.time }}"; final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + final String kubeletExtraArgsKey = "{{ k8s.kubelet.extra.args }}"; final Long waitTime = KubernetesClusterService.KubernetesWorkerNodeInstallAttemptWait.value(); final Long reattempts = KubernetesClusterService.KubernetesWorkerNodeInstallReattempts.value(); @@ -1064,6 +1097,7 @@ public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIs k8sNodeConfig = k8sNodeConfig.replace(routerIpKey, routerIp); k8sNodeConfig = k8sNodeConfig.replace(installWaitTime, String.valueOf(waitTime)); k8sNodeConfig = k8sNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts)); + k8sNodeConfig = k8sNodeConfig.replace(kubeletExtraArgsKey, StringUtils.defaultString(getWorkerNodeKubeletExtraArgs())); k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index ac6a64e2fb0c..c0ca608d6dfb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -370,6 +370,7 @@ public boolean destroy() throws CloudRuntimeException { annotationDao.removeByEntityType(AnnotationService.EntityType.KUBERNETES_CLUSTER.name(), kubernetesCluster.getUuid()); kubernetesClusterDetailsDao.removeDetails(kubernetesCluster.getId()); kubernetesClusterAffinityGroupMapDao.removeByClusterId(kubernetesCluster.getId()); + kubernetesClusterNodeGroupDao.removeByClusterId(kubernetesCluster.getId()); boolean deleted = kubernetesClusterDao.remove(kubernetesCluster.getId()); if (!deleted) { logMessage(Level.WARN, String.format("Failed to delete Kubernetes cluster: %s", kubernetesCluster), null); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeGroupWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeGroupWorker.java new file mode 100644 index 000000000000..4edb4fc8cbb2 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeGroupWorker.java @@ -0,0 +1,435 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; + +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.VirtualMachineMigrationException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroupVO; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeGroupUtil; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.offering.ServiceOffering; +import com.cloud.storage.VMTemplateVO; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.UserVmVO; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + +/** + * Provisions, scales and removes the worker nodes of a node group of a CloudManaged Kubernetes cluster. + * The nodes of a node group are deployed like the other worker nodes, but with the service offering, template, + * Kubernetes node labels and taints of the node group. + * Operations on different node groups of a cluster can run at the same time: the state of the cluster is managed by + * {@link KubernetesClusterManagerImpl} for all of them, the counters of the cluster are updated under a row lock and + * the network rules of the cluster under a cluster lock, and only the nodes of the node group are cleaned up on failure. + */ +public class KubernetesClusterNodeGroupWorker extends KubernetesClusterScaleWorker { + + private static final int MAX_HOST_NAME_LENGTH = 63; + private static final int NODE_READY_WAIT_DURATION = 15000; + + private KubernetesClusterNodeGroupVO nodeGroup; + + public KubernetesClusterNodeGroupWorker(final KubernetesCluster kubernetesCluster, + final KubernetesClusterNodeGroupVO nodeGroup, + final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + this.nodeGroup = nodeGroup; + this.originalState = kubernetesCluster.getState(); + } + + @Override + protected void init() { + super.init(); + VMTemplateVO nodeGroupTemplate = templateDao.findById(nodeGroup.getTemplateId()); + if (nodeGroupTemplate != null) { + this.workerNodeTemplate = nodeGroupTemplate; + } + this.scaleTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterScaleTimeout.value() * 1000; + } + + @Override + protected KubernetesClusterNodeGroupVO getWorkerNodeGroup() { + return nodeGroup; + } + + @Override + protected String getWorkerNodeHostName(String suffix) { + if (nodeGroup.isDefault()) { + return super.getWorkerNodeHostName(suffix); + } + String nodeGroupName = nodeGroup.getName(); + int maxPrefixLength = MAX_HOST_NAME_LENGTH - nodeGroupName.length() - suffix.length() - 2; + String prefix = StringUtils.stripEnd(StringUtils.truncate(kubernetesClusterNodeNamePrefix, maxPrefixLength), "-"); + return String.format("%s-%s-%s", prefix, nodeGroupName, suffix); + } + + @Override + protected ServiceOffering getServiceOfferingForNodeTypeOnCluster(KubernetesClusterNodeType nodeType, KubernetesCluster cluster) { + if (WORKER == nodeType) { + return serviceOfferingDao.findById(nodeGroup.getServiceOfferingId()); + } + return super.getServiceOfferingForNodeTypeOnCluster(nodeType, cluster); + } + + /** + * Node group operations never change the state of the cluster, which is managed by the manager for all the node + * groups of the cluster, so failures only log and throw + */ + @Override + protected void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, + final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + super.logTransitStateAndThrow(logLevel, message, null, null, e); + } + + @Override + protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { + logger.debug("Not changing the state of the Kubernetes cluster {} on event {} during an operation on its node group {}", + kubernetesCluster, e, nodeGroup); + return true; + } + + /** + * The launch permission of the default template is granted and revoked by + * {@link #grantTemplateLaunchPermissionIfNeeded()} as other node groups may be using it at the same time + */ + @Override + protected void deleteTemplateLaunchPermission() { + } + + protected List getNodeGroupVmMaps() { + return kubernetesClusterVmMapDao.listByNodeGroupId(nodeGroup.getId()); + } + + /** + * Removes the VMs of the node group which were not part of it before a failed scale up, the VMs of the other + * node groups which may be scaled up at the same time are kept + */ + @Override + protected void cleanupNewlyCreatedVms(Set originalVmIds) { + Set otherVmIds = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()).stream() + .filter(vmMap -> !Long.valueOf(nodeGroup.getId()).equals(vmMap.getNodeGroupId())) + .map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toSet()); + otherVmIds.addAll(originalVmIds); + super.cleanupNewlyCreatedVms(otherVmIds); + } + + /** + * Recreates the SSH firewall and port forwarding rules of all the nodes of the cluster. The node list is taken + * under the cluster network rules lock, as other node groups may add or remove nodes at the same time + */ + @Override + protected void scaleKubernetesClusterNetworkRules(final List clusterVMIds) throws ManagementServerException { + try { + manager.executeWithKubernetesClusterLock(kubernetesCluster.getId(), KubernetesClusterManagerImpl.NETWORK_RULES_LOCK, () -> { + List vmMaps = getKubernetesClusterVMMaps(); + List vmIds = vmMaps.stream().filter(vm -> !vm.isExternalNode() && !vm.isEtcdNode()) + .map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + vmIds.addAll(vmMaps.stream().filter(KubernetesClusterVmMapVO::isExternalNode) + .map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList())); + try { + super.scaleKubernetesClusterNetworkRules(vmIds); + } catch (ManagementServerException e) { + throw new CloudRuntimeException(e.getMessage(), e); + } + return null; + }); + } catch (CloudRuntimeException e) { + throw new ManagementServerException(e.getMessage(), e); + } + } + + private void initClusterServerIpSshPort() { + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + if (StringUtils.isEmpty(publicIpAddress)) { + logAndThrow(Level.ERROR, String.format("Unable to retrieve the public IP address of the Kubernetes cluster : %s", kubernetesCluster.getName())); + } + } + + /** + * Updates the node count and service offering of the node group and the counters of the cluster from the nodes + * of the node group, which keeps them consistent even when an operation fails half-way + * @param newServiceOffering the new service offering of the node group, null to keep the existing one + */ + protected void syncNodeGroupEntry(final ServiceOffering newServiceOffering) { + nodeGroup = manager.syncNodeGroupEntries(kubernetesCluster.getId(), nodeGroup.getId(), + newServiceOffering != null ? newServiceOffering.getId() : null); + kubernetesCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + } + + /** + * Grants the owner of the cluster the permission to deploy the node group nodes from the default CKS template, + * which is owned by the system. The permission is revoked by the last node group operation of the cluster using it + * @return whether the template of the node group requires the permission, to be released after the scale up + */ + private boolean grantTemplateLaunchPermissionIfNeeded() { + if (owner == null || nodeGroup.getTemplateId() != kubernetesCluster.getTemplateId()) { + return false; + } + manager.grantNodeGroupTemplateLaunchPermission(kubernetesCluster.getId(), nodeGroup.getTemplateId(), owner.getId()); + return true; + } + + private void revokeTemplateLaunchPermission() { + manager.revokeNodeGroupTemplateLaunchPermission(kubernetesCluster.getId(), nodeGroup.getId(), nodeGroup.getTemplateId(), owner.getId()); + } + + /** + * Waits for the given new nodes to join the cluster and become ready, the nodes added by other node groups + * at the same time are not waited for + */ + protected boolean waitForNodesReady(final List nodeVMs) { + for (UserVm vm : nodeVMs) { + String nodeName = vm.getHostName().toLowerCase(); + if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, + getControlNodeLoginUser(), sshKeyFile, nodeName, scaleTimeoutTime, NODE_READY_WAIT_DURATION)) { + logger.warn("Node {} of node group {} of Kubernetes cluster {} is not ready", nodeName, nodeGroup, kubernetesCluster); + return false; + } + } + return true; + } + + private void rollbackScaleUp(final Set originalVmIds, final String message, final Exception e) throws CloudRuntimeException { + cleanupNewlyCreatedVms(originalVmIds); + try { + scaleKubernetesClusterNetworkRules(null); + } catch (ManagementServerException ex) { + logger.warn("Failed to update the network rules of Kubernetes cluster {} after removing the new nodes of node group {}", + kubernetesCluster, nodeGroup, ex); + } + syncNodeGroupEntry(null); + logAndThrow(Level.ERROR, message, e); + } + + private void scaleUpNodeGroup(final long newVmCount) throws CloudRuntimeException { + Set originalVmIds = getNodeGroupVmMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toSet()); + boolean releaseTemplateLaunchPermission = grantTemplateLaunchPermissionIfNeeded(); + try { + List nodeVMs = new ArrayList<>(); + try { + nodeVMs = provisionKubernetesClusterNodeVms(newVmCount, 0, publicIpAddress, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId()); + updateLoginUserDetails(nodeVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList())); + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + rollbackScaleUp(originalVmIds, String.format("Scaling failed for node group %s of Kubernetes cluster : %s, unable to provision node VM", + nodeGroup.getName(), kubernetesCluster.getName()), e); + } + try { + scaleKubernetesClusterNetworkRules(null); + } catch (ManagementServerException e) { + rollbackScaleUp(originalVmIds, String.format("Scaling failed for node group %s of Kubernetes cluster : %s, unable to update network rules", + nodeGroup.getName(), kubernetesCluster.getName()), e); + } + attachIsoKubernetesVMs(nodeVMs); + syncNodeGroupEntry(null); + boolean nodesReady = waitForNodesReady(nodeVMs); + detachIsoKubernetesVMs(nodeVMs); + if (!nodesReady) { + rollbackScaleUp(originalVmIds, String.format("Scaling unsuccessful for node group %s of Kubernetes cluster : %s as its new nodes are not ready", + nodeGroup.getName(), kubernetesCluster.getName()), null); + } + } finally { + if (releaseTemplateLaunchPermission) { + revokeTemplateLaunchPermission(); + } + } + } + + private void scaleDownNodeGroup(final List vmMaps) throws CloudRuntimeException { + if (CollectionUtils.isEmpty(vmMaps)) { + return; + } + try { + removeNodesFromCluster(vmMaps); + } finally { + syncNodeGroupEntry(null); + } + } + + private void scaleNodeGroupOffering(final ServiceOffering serviceOffering) throws CloudRuntimeException { + for (KubernetesClusterVmMapVO vmMap : getNodeGroupVmMaps()) { + UserVmVO userVM = userVmDao.findById(vmMap.getVmId()); + if (userVM.getServiceOfferingId() == serviceOffering.getId()) { + continue; + } + boolean result = false; + try { + result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap<>()); + } catch (RuntimeException | ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException e) { + logAndThrow(Level.ERROR, String.format("Scaling node group %s of Kubernetes cluster : %s failed, unable to scale VM : %s due to %s", + nodeGroup.getName(), kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()), e); + } + if (!result) { + logAndThrow(Level.WARN, String.format("Scaling node group %s of Kubernetes cluster : %s failed, unable to scale VM : %s", + nodeGroup.getName(), kubernetesCluster.getName(), userVM.getDisplayName())); + } + if (System.currentTimeMillis() > scaleTimeoutTime) { + logAndThrow(Level.WARN, String.format("Scaling node group %s of Kubernetes cluster : %s failed, scaling action timed out", + nodeGroup.getName(), kubernetesCluster.getName())); + } + } + syncNodeGroupEntry(serviceOffering); + } + + /** + * Scales the node group for its size and/or service offering + * @param size new number of nodes of the node group, null to keep the current size + * @param nodeIds IDs of the node group VMs to be removed, can not be passed along with size + * @param serviceOffering new service offering of the node group, null to keep the current one + */ + public boolean scaleNodeGroup(final Long size, final List nodeIds, final ServiceOffering serviceOffering) throws CloudRuntimeException { + init(); + logger.info("Scaling node group {} of Kubernetes cluster {}", nodeGroup, kubernetesCluster); + initClusterServerIpSshPort(); + if (serviceOffering != null && serviceOffering.getId() != nodeGroup.getServiceOfferingId()) { + scaleNodeGroupOffering(serviceOffering); + } + List vmMaps = getNodeGroupVmMaps(); + if (CollectionUtils.isNotEmpty(nodeIds)) { + List vmMapsToRemove = vmMaps.stream() + .filter(vmMap -> nodeIds.contains(vmMap.getVmId())).collect(Collectors.toList()); + scaleDownNodeGroup(vmMapsToRemove); + } else if (size != null && size < vmMaps.size()) { + // Remove the most recently added nodes first + List vmMapsToRemove = new ArrayList<>(vmMaps.subList(size.intValue(), vmMaps.size())); + Collections.reverse(vmMapsToRemove); + scaleDownNodeGroup(vmMapsToRemove); + } else if (size != null && size > vmMaps.size()) { + scaleUpNodeGroup(size - vmMaps.size()); + } + return true; + } + + /** + * Removes all the nodes of the node group from the cluster and removes the node group + */ + public boolean deleteNodeGroup() throws CloudRuntimeException { + init(); + logger.info("Deleting node group {} of Kubernetes cluster {}", nodeGroup, kubernetesCluster); + List vmMaps = getNodeGroupVmMaps(); + if (CollectionUtils.isNotEmpty(vmMaps)) { + initClusterServerIpSshPort(); + List vmMapsToRemove = new ArrayList<>(vmMaps); + Collections.reverse(vmMapsToRemove); + scaleDownNodeGroup(vmMapsToRemove); + } + kubernetesClusterNodeGroupDao.remove(nodeGroup.getId()); + return true; + } + + /** + * Brings the labels and taints of a node from the old ones to the new ones, both being the merge of the labels and + * taints of the node group with the labels and taints of the node. The node group label is (re)applied too + */ + private void applyNodeLabelsAndTaints(final KubernetesClusterVmMapVO vmMap, final String oldLabels, final String newLabels, + final String oldTaints, final String newTaints) throws CloudRuntimeException { + UserVmVO userVM = userVmDao.findById(vmMap.getVmId()); + if (userVM == null) { + return; + } + String nodeName = userVM.getHostName().toLowerCase(); + for (String command : KubernetesClusterNodeGroupUtil.getLabelAndTaintUpdateCommands(nodeName, nodeGroup.getName(), + oldLabels, newLabels, oldTaints, newTaints)) { + Pair result = null; + try { + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, + command, 10000, 10000, 60000); + } catch (Exception e) { + logAndThrow(Level.ERROR, String.format("Failed to update labels and taints of node %s in node group %s of Kubernetes cluster : %s", + nodeName, nodeGroup.getName(), kubernetesCluster.getName()), e); + } + if (!result.first()) { + logAndThrow(Level.ERROR, String.format("Failed to update labels and taints of node %s in node group %s of Kubernetes cluster : %s, %s", + nodeName, nodeGroup.getName(), kubernetesCluster.getName(), result.second())); + } + } + } + + /** + * Applies the labels and taints of the node group, merged with the labels and taints of each node, as well as the + * node group label to the existing nodes of the node group + * @param oldLabels labels of the node group before the update, removed from the nodes when not present anymore + * @param oldTaints taints of the node group before the update, removed from the nodes when not present anymore + */ + public boolean updateNodeGroupLabelsAndTaints(final String oldLabels, final String oldTaints) throws CloudRuntimeException { + init(); + List vmMaps = getNodeGroupVmMaps(); + if (CollectionUtils.isEmpty(vmMaps)) { + return true; + } + initClusterServerIpSshPort(); + for (KubernetesClusterVmMapVO vmMap : vmMaps) { + applyNodeLabelsAndTaints(vmMap, + KubernetesClusterNodeGroupUtil.mergeLabels(oldLabels, vmMap.getLabels()), + KubernetesClusterNodeGroupUtil.mergeLabels(nodeGroup.getLabels(), vmMap.getLabels()), + KubernetesClusterNodeGroupUtil.mergeTaints(oldTaints, vmMap.getTaints()), + KubernetesClusterNodeGroupUtil.mergeTaints(nodeGroup.getTaints(), vmMap.getTaints())); + } + return true; + } + + /** + * Applies the labels and taints of a node of the node group, merged with the labels and taints of the node group + * @param vmMap the node, with its new labels and taints + * @param oldNodeLabels labels of the node before the update, removed from the node when not present anymore + * @param oldNodeTaints taints of the node before the update, removed from the node when not present anymore + */ + public boolean updateNodeLabelsAndTaints(final KubernetesClusterVmMapVO vmMap, final String oldNodeLabels, + final String oldNodeTaints) throws CloudRuntimeException { + init(); + initClusterServerIpSshPort(); + applyNodeLabelsAndTaints(vmMap, + KubernetesClusterNodeGroupUtil.mergeLabels(nodeGroup.getLabels(), oldNodeLabels), + KubernetesClusterNodeGroupUtil.mergeLabels(nodeGroup.getLabels(), vmMap.getLabels()), + KubernetesClusterNodeGroupUtil.mergeTaints(nodeGroup.getTaints(), oldNodeTaints), + KubernetesClusterNodeGroupUtil.mergeTaints(nodeGroup.getTaints(), vmMap.getTaints())); + return true; + } + + /** + * Deploys the cluster autoscaler for the node groups of the cluster with autoscaling enabled, or removes it when + * autoscaling is not enabled for any node group anymore + */ + public boolean updateClusterAutoscaler() throws CloudRuntimeException { + init(); + applyClusterAutoscaler(); + return true; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 72dd9816858d..a259c699506d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -79,6 +79,8 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroupVO; +import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.network.IpAddress; import com.cloud.network.Network; @@ -430,6 +432,34 @@ protected List provisionKubernetesClusterNodeVms(final long nodeCount, f return provisionKubernetesClusterNodeVms(nodeCount, 0, controlIpAddress, domainId, accountId); } + /** + * @return the CPU cores and memory (MB) used by the nodes of the node groups of the cluster other than the + * default one, whose nodes are accounted with the cluster size + */ + protected Pair calculateNodeGroupsCapacity() { + long cores = 0L; + long memory = 0L; + if (!(kubernetesCluster instanceof KubernetesClusterVO) || ((KubernetesClusterVO) kubernetesCluster).getNodeGroupNodeCount() == 0) { + return new Pair<>(cores, memory); + } + for (KubernetesClusterNodeGroupVO nodeGroup : kubernetesClusterNodeGroupDao.listByClusterId(kubernetesCluster.getId())) { + if (nodeGroup.isDefault()) { + continue; + } + ServiceOffering offering = serviceOfferingDao.findById(nodeGroup.getServiceOfferingId()); + if (offering == null) { + continue; + } + cores += offering.getCpu() * nodeGroup.getNodeCount(); + memory += offering.getRamSize() * nodeGroup.getNodeCount(); + } + return new Pair<>(cores, memory); + } + + protected String getWorkerNodeHostName(String suffix) { + return String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, suffix); + } + protected UserVm createKubernetesNode(String joinIp, Long domainId, Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { UserVm nodeVm = null; @@ -448,7 +478,7 @@ protected UserVm createKubernetesNode(String joinIp, Long domainId, Long account customParameterMap.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); } String suffix = Long.toHexString(System.currentTimeMillis()); - String hostName = String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, suffix); + String hostName = getWorkerNodeHostName(suffix); String k8sNodeConfig = null; try { k8sNodeConfig = getKubernetesNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), false); @@ -953,50 +983,111 @@ private KubernetesClusterVO updateKubernetesClusterEntry(final Boolean autoscale return kubernetesClusterVO; } - protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AutoscaleRequested); + /** + * Updates the autoscaling settings of the default node group, which are the autoscaling settings of the cluster + */ + private void updateDefaultNodeGroupAutoscaling(final boolean enable, final Long minSize, final Long maxSize) { + KubernetesClusterNodeGroupVO defaultNodeGroup = kubernetesClusterNodeGroupDao.findDefaultByClusterId(kubernetesCluster.getId()); + if (defaultNodeGroup == null) { + return; } + defaultNodeGroup.setAutoscalingEnabled(enable); + defaultNodeGroup.setMinSize(minSize); + defaultNodeGroup.setMaxSize(maxSize); + kubernetesClusterNodeGroupDao.update(defaultNodeGroup.getId(), defaultNodeGroup); + } - File pkFile = getManagementServerSshPublicKeyFile(); - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); - publicIpAddress = publicIpSshPort.first(); - sshPort = publicIpSshPort.second(); + /** + * @return the min:max:id specs of the node groups with autoscaling enabled, passed to the cluster autoscaler as + * --nodes arguments. The ID of the cluster identifies its default node group, as expected by the upstream cluster + * autoscaler, and the ID of the node group identifies the other node groups + */ + protected List getClusterAutoscalerNodeGroupSpecs() { + List specs = new ArrayList<>(); + List nodeGroups = kubernetesClusterNodeGroupDao.listByClusterId(kubernetesCluster.getId()); + if (nodeGroups.stream().noneMatch(KubernetesClusterNodeGroupVO::isDefault)) { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster.getAutoscalingEnabled()) { + specs.add(String.format("%d:%d:%s", cluster.getMinSize(), cluster.getMaxSize(), cluster.getUuid())); + } + } + for (KubernetesClusterNodeGroupVO nodeGroup : nodeGroups) { + if (!nodeGroup.isAutoscalingEnabled()) { + continue; + } + String id = nodeGroup.isDefault() ? kubernetesCluster.getUuid() : nodeGroup.getUuid(); + specs.add(String.format("%d:%d:%s", nodeGroup.getMinSize(), nodeGroup.getMaxSize(), id)); + } + return specs; + } - try { - if (enable) { - String command = String.format("sudo /opt/bin/autoscale-kube-cluster -i %s -e -M %d -m %d", - kubernetesCluster.getUuid(), maxSize, minSize); + protected String getClusterAutoscalerCommand(List nodeGroupSpecs) { + if (CollectionUtils.isEmpty(nodeGroupSpecs)) { + return String.format("sudo %s%s -d", scriptPath, autoscaleScriptFilename); + } + StringBuilder command = new StringBuilder(String.format("sudo %s%s -e", scriptPath, autoscaleScriptFilename)); + for (String spec : nodeGroupSpecs) { + command.append(" -n ").append(spec); + } + String image = KubernetesClusterService.KubernetesClusterAutoscalerImage.value(); + if (StringUtils.isNotBlank(image)) { + command.append(String.format(" -I '%s'", image.trim())); + } + return command.toString(); + } + + /** + * Deploys the cluster autoscaler for the node groups of the cluster with autoscaling enabled, or removes it when + * autoscaling is not enabled for any node group. Deployments are serialized per cluster so that the autoscaler is + * always deployed with the latest settings of all the node groups + */ + protected void applyClusterAutoscaler() throws CloudRuntimeException { + manager.executeWithKubernetesClusterLock(kubernetesCluster.getId(), KubernetesClusterManagerImpl.AUTOSCALER_LOCK, () -> { + File pkFile = getManagementServerSshPublicKeyFile(); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + String command = getClusterAutoscalerCommand(getClusterAutoscalerNodeGroupSpecs()); + try { Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); - - // Maybe the file isn't present. Try and copy it + // The script may be missing or outdated on the control node. Copy the current one if (!result.first()) { - logMessage(Level.INFO, "Autoscaling files missing. Adding them now", null); + logMessage(Level.INFO, "Autoscaling files missing or outdated. Adding them now", null); retrieveScriptFiles(); copyScripts(publicIpAddress, sshPort); - if (!createCloudStackSecret(keys)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", - kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new CloudRuntimeException(String.format("Failed to setup keys for Kubernetes cluster %s", kubernetesCluster.getName())); } - - // If at first you don't succeed ... result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); if (!result.first()) { throw new CloudRuntimeException(result.second()); } } - updateKubernetesClusterEntry(true, minSize, maxSize); - } else { - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, String.format("sudo /opt/bin/autoscale-kube-cluster -d"), - 10000, 10000, 60000); - if (!result.first()) { - throw new CloudRuntimeException(result.second()); - } - updateKubernetesClusterEntry(false, null, null); + } catch (CloudRuntimeException e) { + throw e; + } catch (Exception e) { + throw new CloudRuntimeException(e.getMessage(), e); + } + return null; + }); + } + + protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AutoscaleRequested); + } + KubernetesClusterVO previousCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + try { + updateKubernetesClusterEntry(enable, enable ? minSize : null, enable ? maxSize : null); + updateDefaultNodeGroupAutoscaling(enable, enable ? minSize : null, enable ? maxSize : null); + try { + applyClusterAutoscaler(); + } catch (CloudRuntimeException e) { + updateKubernetesClusterEntry(previousCluster.getAutoscalingEnabled(), previousCluster.getMinSize(), previousCluster.getMaxSize()); + updateDefaultNodeGroupAutoscaling(previousCluster.getAutoscalingEnabled(), previousCluster.getMinSize(), previousCluster.getMaxSize()); + throw e; } return true; } catch (Exception e) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 003a852f3dbe..a010b4415bbe 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -47,6 +47,7 @@ import com.cloud.exception.VirtualMachineMigrationException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroupVO; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; @@ -73,12 +74,12 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private Map serviceOfferingNodeTypeMap; private Long clusterSize; private List nodeIds; - private KubernetesCluster.State originalState; - private Network network; + protected KubernetesCluster.State originalState; + protected Network network; private Long minSize; private Long maxSize; private Boolean isAutoscalingEnabled; - private long scaleTimeoutTime; + protected long scaleTimeoutTime; protected KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); @@ -112,7 +113,7 @@ protected void init() { this.network = networkDao.findById(kubernetesCluster.getNetworkId()); } - private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message, final Exception e) throws CloudRuntimeException { + protected void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message, final Exception e) throws CloudRuntimeException { KubernetesCluster cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); if (cluster != null && KubernetesCluster.State.Scaling.equals(cluster.getState())) { logTransitStateAndThrow(logLevel, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); @@ -121,7 +122,7 @@ private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final } } - private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message) throws CloudRuntimeException { + protected void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message) throws CloudRuntimeException { logTransitStateToFailedIfNeededAndThrow(logLevel, message, null); } @@ -194,7 +195,7 @@ private void scaleKubernetesClusterVpcTierRules(final List clusterVMIds) t * @param clusterVMIds * @throws ManagementServerException */ - private void scaleKubernetesClusterNetworkRules(final List clusterVMIds) throws ManagementServerException { + protected void scaleKubernetesClusterNetworkRules(final List clusterVMIds) throws ManagementServerException { if (manager.isDirectAccess(network)) { if (logger.isDebugEnabled()) logger.debug("Network: {} for Kubernetes cluster: {} is not an isolated network " + @@ -231,11 +232,14 @@ private KubernetesClusterVO updateKubernetesClusterEntryForNodeType(final Long n protected Pair calculateNewClusterCountAndCapacity(Long newWorkerSize, KubernetesClusterNodeType nodeType, ServiceOffering serviceOffering) { long cores; long memory; - long totalClusterSize = newWorkerSize == null ? kubernetesCluster.getTotalNodeCount() : (newWorkerSize + kubernetesCluster.getControlNodeCount() + kubernetesCluster.getEtcdNodeCount()); + // Node groups have their own service offering, the default offering only applies to the remaining nodes + long totalClusterSize = (newWorkerSize == null ? kubernetesCluster.getNodeCount() : newWorkerSize) + + kubernetesCluster.getControlNodeCount() + kubernetesCluster.getEtcdNodeCount(); if (nodeType == DEFAULT) { - cores = serviceOffering.getCpu() * totalClusterSize; - memory = serviceOffering.getRamSize() * totalClusterSize; + Pair nodeGroupsCapacity = calculateNodeGroupsCapacity(); + cores = serviceOffering.getCpu() * totalClusterSize + nodeGroupsCapacity.first(); + memory = serviceOffering.getRamSize() * totalClusterSize + nodeGroupsCapacity.second(); } else { long nodeCount = getNodeCountForType(nodeType, kubernetesCluster); Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster); @@ -394,7 +398,7 @@ private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); } - private void removeNodesFromCluster(List vmMaps) throws CloudRuntimeException { + protected void removeNodesFromCluster(List vmMaps) throws CloudRuntimeException { for (KubernetesClusterVmMapVO vmMapVO : vmMaps) { UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); logger.info("Removing vm {} from cluster {}", userVM, kubernetesCluster); @@ -445,7 +449,9 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { } List vmList; if (this.nodeIds != null) { - vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds).stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList()); + Long defaultNodeGroupId = getDefaultNodeGroupId(); + vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds).stream() + .filter(vm -> !vm.isExternalNode() && !vm.isInNodeGroupOtherThan(defaultNodeGroupId)).collect(Collectors.toList()); } else { vmList = getWorkerNodesToRemove(); if (vmList.isEmpty()) { @@ -457,9 +463,18 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { removeNodesFromCluster(vmList); } + /** + * @return the ID of the default node group of the cluster, whose nodes are the worker nodes of the cluster size + */ + protected Long getDefaultNodeGroupId() { + KubernetesClusterNodeGroupVO defaultNodeGroup = kubernetesClusterNodeGroupDao.findDefaultByClusterId(kubernetesCluster.getId()); + return defaultNodeGroup != null ? defaultNodeGroup.getId() : null; + } + public List getWorkerNodesToRemove() { + Long defaultNodeGroupId = getDefaultNodeGroupId(); List workerVMsMap = getKubernetesClusterVMMaps().stream() - .filter(vm -> !vm.isExternalNode() && !vm.isControlNode() && !vm.isEtcdNode()) + .filter(vm -> !vm.isExternalNode() && !vm.isControlNode() && !vm.isEtcdNode() && !vm.isInNodeGroupOtherThan(defaultNodeGroupId)) .collect(Collectors.toList()); int totalWorkerNodes = workerVMsMap.size(); int desiredWorkerNodes = clusterSize == null ? (int) kubernetesCluster.getNodeCount() : clusterSize.intValue(); @@ -471,7 +486,7 @@ public List getWorkerNodesToRemove() { return new ArrayList<>(workerVMsMap.subList(startIndex, totalWorkerNodes)); } - private void cleanupNewlyCreatedVms(Set originalVmIds) { + protected void cleanupNewlyCreatedVms(Set originalVmIds) { List currentVmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); for (KubernetesClusterVmMapVO clusterVmMap : currentVmMaps) { if (originalVmIds.contains(clusterVmMap.getVmId())) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNodeGroupDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNodeGroupDao.java new file mode 100644 index 000000000000..8b2121874aef --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNodeGroupDao.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.dao; + +import java.util.List; + +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroupVO; +import com.cloud.utils.db.GenericDao; + +public interface KubernetesClusterNodeGroupDao extends GenericDao { + + List listByClusterId(long clusterId); + + KubernetesClusterNodeGroupVO findByClusterIdAndName(long clusterId, String name); + + /** + * @return the default node group of the cluster, null for clusters without node groups such as ExternalManaged clusters + */ + KubernetesClusterNodeGroupVO findDefaultByClusterId(long clusterId); + + List listByStates(KubernetesClusterNodeGroup.State... states); + + int removeByClusterId(long clusterId); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNodeGroupDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNodeGroupDaoImpl.java new file mode 100644 index 000000000000..b0fa68d06a86 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterNodeGroupDaoImpl.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.dao; + +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroupVO; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class KubernetesClusterNodeGroupDaoImpl extends GenericDaoBase + implements KubernetesClusterNodeGroupDao { + + private final SearchBuilder clusterIdSearch; + private final SearchBuilder stateSearch; + + public KubernetesClusterNodeGroupDaoImpl() { + clusterIdSearch = createSearchBuilder(); + clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + clusterIdSearch.and("name", clusterIdSearch.entity().getName(), SearchCriteria.Op.EQ); + clusterIdSearch.and("defaultGroup", clusterIdSearch.entity().isDefaultGroup(), SearchCriteria.Op.EQ); + clusterIdSearch.done(); + + stateSearch = createSearchBuilder(); + stateSearch.and("state", stateSearch.entity().getState(), SearchCriteria.Op.IN); + stateSearch.done(); + } + + /** + * @return the node groups of the cluster, the default node group first + */ + @Override + public List listByClusterId(long clusterId) { + SearchCriteria sc = clusterIdSearch.create(); + sc.setParameters("clusterId", clusterId); + Filter filter = new Filter(KubernetesClusterNodeGroupVO.class, "defaultGroup", Boolean.FALSE, null, null); + filter.addOrderBy(KubernetesClusterNodeGroupVO.class, "id", Boolean.TRUE); + return listBy(sc, filter); + } + + @Override + public KubernetesClusterNodeGroupVO findDefaultByClusterId(long clusterId) { + SearchCriteria sc = clusterIdSearch.create(); + sc.setParameters("clusterId", clusterId); + sc.setParameters("defaultGroup", true); + return findOneBy(sc); + } + + @Override + public List listByStates(KubernetesClusterNodeGroup.State... states) { + SearchCriteria sc = stateSearch.create(); + sc.setParameters("state", (Object[]) states); + return listBy(sc); + } + + @Override + public KubernetesClusterNodeGroupVO findByClusterIdAndName(long clusterId, String name) { + SearchCriteria sc = clusterIdSearch.create(); + sc.setParameters("clusterId", clusterId); + sc.setParameters("name", name); + return findOneBy(sc); + } + + @Override + public int removeByClusterId(long clusterId) { + SearchCriteria sc = clusterIdSearch.create(); + sc.setParameters("clusterId", clusterId); + return remove(sc); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java index 71b90ef2e6a0..48634aa4a5ff 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java @@ -32,7 +32,13 @@ public interface KubernetesClusterVmMapDao extends GenericDao listByClusterIdAndVmType(long clusterId, KubernetesClusterNodeType nodeType); + List listByNodeGroupId(long nodeGroupId); + KubernetesClusterVmMapVO findByVmId(long vmId); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java index 9607783b50e4..02cda97f0ef0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java @@ -18,6 +18,9 @@ import java.util.List; +import javax.inject.Inject; + +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroupVO; import com.cloud.kubernetes.cluster.KubernetesServiceHelper; import org.springframework.stereotype.Component; @@ -34,8 +37,13 @@ @Component public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase implements KubernetesClusterVmMapDao { + @Inject + private KubernetesClusterNodeGroupDao kubernetesClusterNodeGroupDao; + private final SearchBuilder clusterIdSearch; private final SearchBuilder vmIdSearch; + private final SearchBuilder defaultPoolWorkerSearch; + private final SearchBuilder nodeGroupSearch; public KubernetesClusterVmMapDaoImpl() { clusterIdSearch = createSearchBuilder(); @@ -48,6 +56,19 @@ public KubernetesClusterVmMapDaoImpl() { vmIdSearch = createSearchBuilder(); vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ); vmIdSearch.done(); + + defaultPoolWorkerSearch = createSearchBuilder(); + defaultPoolWorkerSearch.and("clusterId", defaultPoolWorkerSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + defaultPoolWorkerSearch.and("controlNode", defaultPoolWorkerSearch.entity().isControlNode(), SearchCriteria.Op.EQ); + defaultPoolWorkerSearch.and("etcdNode", defaultPoolWorkerSearch.entity().isEtcdNode(), SearchCriteria.Op.EQ); + defaultPoolWorkerSearch.and().op("noNodeGroup", defaultPoolWorkerSearch.entity().getNodeGroupId(), SearchCriteria.Op.NULL); + defaultPoolWorkerSearch.or("defaultNodeGroupId", defaultPoolWorkerSearch.entity().getNodeGroupId(), SearchCriteria.Op.EQ); + defaultPoolWorkerSearch.cp(); + defaultPoolWorkerSearch.done(); + + nodeGroupSearch = createSearchBuilder(); + nodeGroupSearch.and("nodeGroupId", nodeGroupSearch.entity().getNodeGroupId(), SearchCriteria.Op.EQ); + nodeGroupSearch.done(); } @Override @@ -99,12 +120,26 @@ public List listByClusterIdAndVmType(long clusterId, K sc.setParameters("controlNode", false); sc.setParameters("etcdNode", true); } else { + // The worker nodes of the cluster size are the nodes of the default node group and the external nodes, + // the nodes of the other node groups are managed through their node group + sc = defaultPoolWorkerSearch.create(); + sc.setParameters("clusterId", clusterId); sc.setParameters("controlNode", false); sc.setParameters("etcdNode", false); + KubernetesClusterNodeGroupVO defaultNodeGroup = kubernetesClusterNodeGroupDao.findDefaultByClusterId(clusterId); + sc.setParameters("defaultNodeGroupId", defaultNodeGroup != null ? defaultNodeGroup.getId() : -1L); } return listBy(sc); } + @Override + public List listByNodeGroupId(long nodeGroupId) { + SearchCriteria sc = nodeGroupSearch.create(); + sc.setParameters("nodeGroupId", nodeGroupId); + Filter filter = new Filter(KubernetesClusterVmMapVO.class, "id", Boolean.TRUE, null, null); + return listBy(sc, filter); + } + @Override public KubernetesClusterVmMapVO findByVmId(long vmId) { SearchBuilder sb = createSearchBuilder(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeGroupUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeGroupUtil.java new file mode 100644 index 000000000000..0a7a4adc5d3e --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeGroupUtil.java @@ -0,0 +1,335 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; + +/** + * Helpers to validate and (de)serialize the name, Kubernetes node labels and taints of a node group. + * Labels are stored as comma separated key=value pairs and taints as comma separated key[=value]:effect + * entries, which is the format expected by the kubelet --node-labels and --register-with-taints arguments. + */ +public class KubernetesClusterNodeGroupUtil { + + public static final int MAX_NODE_GROUP_NAME_LENGTH = 20; + public static final Set TAINT_EFFECTS = Set.of("NoSchedule", "PreferNoSchedule", "NoExecute"); + + public static final String KEY = "key"; + public static final String VALUE = "value"; + public static final String EFFECT = "effect"; + + private static final Pattern NODE_GROUP_NAME_PATTERN = Pattern.compile("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"); + private static final Pattern LABEL_NAME_PATTERN = Pattern.compile("^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$"); + private static final Pattern DNS_SUBDOMAIN_PATTERN = Pattern.compile("^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"); + private static final int MAX_LABEL_NAME_LENGTH = 63; + private static final int MAX_LABEL_PREFIX_LENGTH = 253; + + public static class Taint { + private final String key; + private final String value; + private final String effect; + + public Taint(String key, String value, String effect) { + this.key = key; + this.value = StringUtils.defaultString(value); + this.effect = effect; + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + + public String getEffect() { + return effect; + } + + @Override + public String toString() { + return StringUtils.isEmpty(value) ? String.format("%s:%s", key, effect) : String.format("%s=%s:%s", key, value, effect); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Taint)) { + return false; + } + Taint taint = (Taint) o; + return key.equals(taint.key) && value.equals(taint.value) && effect.equals(taint.effect); + } + + @Override + public int hashCode() { + return Objects.hash(key, value, effect); + } + } + + public static void validateNodeGroupName(String name) { + if (StringUtils.isBlank(name) || name.length() > MAX_NODE_GROUP_NAME_LENGTH || !NODE_GROUP_NAME_PATTERN.matcher(name).matches()) { + throw new InvalidParameterValueException(String.format("Invalid node group name: %s. It must consist of at most %d lower case " + + "alphanumeric characters or '-', and must start and end with an alphanumeric character", name, MAX_NODE_GROUP_NAME_LENGTH)); + } + } + + /** + * Validates a Kubernetes label or taint key: an optional DNS subdomain prefix followed by '/' and a name. + * Keys in the kubernetes.io and k8s.io namespaces cannot be set by the kubelet and the node group label is reserved + */ + protected static void validateKey(String key, String type) { + if (StringUtils.isBlank(key)) { + throw new InvalidParameterValueException(String.format("Empty %s key", type)); + } + String prefix = null; + String name = key; + int separatorIndex = key.indexOf('/'); + if (separatorIndex >= 0) { + prefix = key.substring(0, separatorIndex); + name = key.substring(separatorIndex + 1); + if (prefix.isEmpty() || prefix.length() > MAX_LABEL_PREFIX_LENGTH || !DNS_SUBDOMAIN_PATTERN.matcher(prefix).matches()) { + throw new InvalidParameterValueException(String.format("Invalid prefix of the %s key: %s", type, key)); + } + } + if (name.length() > MAX_LABEL_NAME_LENGTH || !LABEL_NAME_PATTERN.matcher(name).matches()) { + throw new InvalidParameterValueException(String.format("Invalid %s key: %s", type, key)); + } + if (prefix != null && (isRestrictedNamespace(prefix) || KubernetesClusterNodeGroup.NODE_GROUP_LABEL.equals(key))) { + throw new InvalidParameterValueException(String.format("The %s key %s uses a reserved namespace", type, key)); + } + } + + private static boolean isRestrictedNamespace(String prefix) { + for (String namespace : List.of("kubernetes.io", "k8s.io")) { + if (prefix.equals(namespace) || prefix.endsWith("." + namespace)) { + return true; + } + } + return false; + } + + protected static void validateValue(String value, String key, String type) { + if (StringUtils.isEmpty(value)) { + return; + } + if (value.length() > MAX_LABEL_NAME_LENGTH || !LABEL_NAME_PATTERN.matcher(value).matches()) { + throw new InvalidParameterValueException(String.format("Invalid value of the %s %s: %s", type, key, value)); + } + } + + /** + * Converts and validates the labels passed to an API as a map parameter + * (labels[0].key=k&labels[0].value=v) into an ordered key-value map + */ + public static Map getLabelsFromApiMap(Map> labelsMap) { + Map labels = new LinkedHashMap<>(); + if (MapUtils.isEmpty(labelsMap)) { + return labels; + } + for (Map entry : labelsMap.values()) { + String key = StringUtils.trim(entry.get(KEY)); + String value = StringUtils.trimToEmpty(entry.get(VALUE)); + validateKey(key, "label"); + validateValue(value, key, "label"); + if (labels.containsKey(key)) { + throw new InvalidParameterValueException(String.format("Duplicate label key: %s", key)); + } + labels.put(key, value); + } + return labels; + } + + /** + * Converts and validates the taints passed to an API as a map parameter + * (taints[0].key=k&taints[0].value=v&taints[0].effect=NoSchedule) + */ + public static List getTaintsFromApiMap(Map> taintsMap) { + List taints = new ArrayList<>(); + if (MapUtils.isEmpty(taintsMap)) { + return taints; + } + for (Map entry : taintsMap.values()) { + String key = StringUtils.trim(entry.get(KEY)); + String value = StringUtils.trimToEmpty(entry.get(VALUE)); + String effect = StringUtils.trim(entry.get(EFFECT)); + validateKey(key, "taint"); + validateValue(value, key, "taint"); + if (!TAINT_EFFECTS.contains(effect)) { + throw new InvalidParameterValueException(String.format("Invalid effect of the taint %s: %s. Valid values are: %s", + key, effect, String.join(", ", TAINT_EFFECTS))); + } + if (taints.stream().anyMatch(t -> t.getKey().equals(key) && t.getEffect().equals(effect))) { + throw new InvalidParameterValueException(String.format("Duplicate taint with key %s and effect %s", key, effect)); + } + taints.add(new Taint(key, value, effect)); + } + return taints; + } + + /** + * @return the labels of a node: the labels of its node group and its own labels, which override the labels of the + * node group with the same key + */ + public static String mergeLabels(String nodeGroupLabels, String nodeLabels) { + Map labels = labelsFromString(nodeGroupLabels); + labels.putAll(labelsFromString(nodeLabels)); + return labelsToString(labels); + } + + /** + * @return the taints of a node: the taints of its node group and its own taints, which override the taints of the + * node group with the same key and effect + */ + public static String mergeTaints(String nodeGroupTaints, String nodeTaints) { + List taints = taintsFromString(nodeGroupTaints); + for (Taint taint : taintsFromString(nodeTaints)) { + taints.removeIf(t -> t.getKey().equals(taint.getKey()) && t.getEffect().equals(taint.getEffect())); + taints.add(taint); + } + return taintsToString(taints); + } + + /** + * @return whether some of the taints prevent the pods without matching tolerations from running on the nodes, + * which is not allowed for the default node group as it has to accept the system pods + */ + public static boolean hasSchedulingTaints(Collection taints) { + return taints != null && taints.stream().anyMatch(taint -> !"PreferNoSchedule".equals(taint.getEffect())); + } + + public static String labelsToString(Map labels) { + if (MapUtils.isEmpty(labels)) { + return null; + } + return labels.entrySet().stream().map(e -> String.format("%s=%s", e.getKey(), e.getValue())).collect(Collectors.joining(",")); + } + + public static Map labelsFromString(String labels) { + Map map = new LinkedHashMap<>(); + if (StringUtils.isBlank(labels)) { + return map; + } + for (String label : labels.split(",")) { + String[] keyValue = label.split("=", 2); + map.put(keyValue[0], keyValue.length > 1 ? keyValue[1] : ""); + } + return map; + } + + public static String taintsToString(Collection taints) { + if (taints == null || taints.isEmpty()) { + return null; + } + return taints.stream().map(Taint::toString).collect(Collectors.joining(",")); + } + + public static List taintsFromString(String taints) { + List list = new ArrayList<>(); + if (StringUtils.isBlank(taints)) { + return list; + } + for (String taint : taints.split(",")) { + int effectIndex = taint.lastIndexOf(':'); + String keyValue = taint.substring(0, effectIndex); + String effect = taint.substring(effectIndex + 1); + String[] parts = keyValue.split("=", 2); + list.add(new Taint(parts[0], parts.length > 1 ? parts[1] : "", effect)); + } + return list; + } + + /** + * @return the kubelet arguments registering the node group nodes with the node group label, + * the user defined labels and the taints of the node group + */ + public static String getKubeletExtraArgs(KubernetesClusterNodeGroup nodeGroup) { + Map labels = new LinkedHashMap<>(); + labels.put(KubernetesClusterNodeGroup.NODE_GROUP_LABEL, nodeGroup.getName()); + labels.putAll(labelsFromString(nodeGroup.getLabels())); + String args = String.format("--node-labels=%s", labelsToString(labels)); + if (StringUtils.isNotBlank(nodeGroup.getTaints())) { + args += String.format(" --register-with-taints=%s", nodeGroup.getTaints()); + } + return args; + } + + /** + * @return the kubectl commands to run against a node of the node group to bring its labels and taints + * to the new ones: labels and taints which are not present anymore are removed and all the new ones + * as well as the node group label are (re)applied, so the commands are idempotent and can be retried + */ + public static List getLabelAndTaintUpdateCommands(String nodeName, String nodeGroupName, String oldLabels, String newLabels, + String oldTaints, String newTaints) { + List commands = new ArrayList<>(); + Map oldLabelsMap = labelsFromString(oldLabels); + // The node group label is (re)applied too, nodes added before the node group existed do not have it + Map newLabelsMap = new LinkedHashMap<>(); + newLabelsMap.put(KubernetesClusterNodeGroup.NODE_GROUP_LABEL, nodeGroupName); + newLabelsMap.putAll(labelsFromString(newLabels)); + List labelArgs = new ArrayList<>(); + for (String key : oldLabelsMap.keySet()) { + if (!newLabelsMap.containsKey(key)) { + labelArgs.add(key + "-"); + } + } + for (Map.Entry label : newLabelsMap.entrySet()) { + labelArgs.add(String.format("%s=%s", label.getKey(), label.getValue())); + } + if (!labelArgs.isEmpty()) { + commands.add(String.format("sudo /opt/bin/kubectl label node %s %s --overwrite", nodeName, String.join(" ", labelArgs))); + } + + List oldTaintsList = taintsFromString(oldTaints); + List newTaintsList = taintsFromString(newTaints); + List removedTaintArgs = new ArrayList<>(); + for (Taint taint : oldTaintsList) { + boolean keptWithSameEffect = newTaintsList.stream().anyMatch(t -> t.getKey().equals(taint.getKey()) && t.getEffect().equals(taint.getEffect())); + if (!keptWithSameEffect) { + removedTaintArgs.add(String.format("%s:%s-", taint.getKey(), taint.getEffect())); + } + } + if (!removedTaintArgs.isEmpty()) { + // kubectl fails to remove a taint which is not present, e.g. already removed by a previous attempt + for (String removedTaintArg : removedTaintArgs) { + commands.add(String.format("sudo /opt/bin/kubectl taint node %s %s || true", nodeName, removedTaintArg)); + } + } + if (!newTaintsList.isEmpty()) { + commands.add(String.format("sudo /opt/bin/kubectl taint node %s %s --overwrite", nodeName, + newTaintsList.stream().map(Taint::toString).collect(Collectors.joining(" ")))); + } + return commands; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 8bf0f3625508..e10e611cbb63 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -202,6 +202,17 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "Root disk size in GB for each node") private Long nodeRootDiskSize; + @Parameter(name = ApiConstants.NODE_LABELS, type = CommandType.MAP, + description = "Kubernetes labels for the nodes of the default node group of a CloudManaged cluster. " + + "Example: nodelabels[0].key=disktype&nodelabels[0].value=ssd", since = "24.0") + private Map> nodeLabels; + + @Parameter(name = ApiConstants.NODE_TAINTS, type = CommandType.MAP, + description = "Kubernetes taints for the nodes of the default node group of a CloudManaged cluster. The default node group " + + "has to accept the system pods, so only the PreferNoSchedule effect is allowed. " + + "Example: nodetaints[0].key=dedicated&nodetaints[0].value=system&nodetaints[0].effect=PreferNoSchedule", since = "24.0") + private Map> nodeTaints; + @Parameter(name = ApiConstants.CLUSTER_TYPE, type = CommandType.STRING, description = "Type of the cluster: CloudManaged, ExternalManaged. The default value is CloudManaged.", since="4.19.0") private String clusterType; @@ -317,6 +328,14 @@ public Long getNodeRootDiskSize() { } } + public Map> getNodeLabels() { + return nodeLabels; + } + + public Map> getNodeTaints() { + return nodeTaints; + } + public String getClusterType() { if (clusterType == null) { return KubernetesCluster.ClusterType.CloudManaged.toString(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/CreateKubernetesClusterNodeGroupCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/CreateKubernetesClusterNodeGroupCmd.java new file mode 100644 index 000000000000..2c9b97b4617b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/CreateKubernetesClusterNodeGroupCmd.java @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.nodegroup; + +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterNodeGroupResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "createKubernetesClusterNodeGroup", + description = "Creates a node group of worker nodes with their own service offering, template, labels and taints " + + "in a running CloudManaged Kubernetes cluster", + responseObject = KubernetesClusterNodeGroupResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "24.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class CreateKubernetesClusterNodeGroupCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL(accessType = SecurityChecker.AccessType.OperateEntry) + @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = KubernetesClusterResponse.class, required = true, + description = "The ID of the Kubernetes cluster") + private Long clusterId; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, + description = "Name of the node group, unique in the Kubernetes cluster. At most 20 lower case alphanumeric characters or '-'") + private String name; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, required = true, + description = "The ID of the service offering for the nodes of the node group") + private Long serviceOfferingId; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, + description = "The ID of the CKS template for the nodes of the node group. Defaults to the template of the worker nodes of the cluster") + private Long templateId; + + @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, required = true, + description = "Number of nodes in the node group, can be 0") + private Long size; + + @Parameter(name = ApiConstants.NODE_LABELS, type = CommandType.MAP, + description = "Kubernetes labels for the nodes of the node group. " + + "Example: nodelabels[0].key=disktype&nodelabels[0].value=ssd") + private Map> labels; + + @Parameter(name = ApiConstants.NODE_TAINTS, type = CommandType.MAP, + description = "Kubernetes taints for the nodes of the node group, effect is one of NoSchedule, PreferNoSchedule, NoExecute. " + + "Example: nodetaints[0].key=gpu&nodetaints[0].value=true&nodetaints[0].effect=NoSchedule") + private Map> taints; + + @Parameter(name = ApiConstants.AUTOSCALING_ENABLED, type = CommandType.BOOLEAN, + description = "Whether the size of the node group is managed by the cluster autoscaler, within its minsize and maxsize") + private Boolean autoscalingEnabled; + + @Parameter(name = ApiConstants.MIN_SIZE, type = CommandType.LONG, + description = "Minimum size of the node group when autoscaling is enabled") + private Long minSize; + + @Parameter(name = ApiConstants.MAX_SIZE, type = CommandType.LONG, + description = "Maximum size of the node group when autoscaling is enabled") + private Long maxSize; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getClusterId() { + return clusterId; + } + + public String getName() { + return name; + } + + public Long getServiceOfferingId() { + return serviceOfferingId; + } + + public Long getTemplateId() { + return templateId; + } + + public Long getSize() { + return size; + } + + public Map> getLabels() { + return labels; + } + + public Map> getTaints() { + return taints; + } + + public Boolean isAutoscalingEnabled() { + return autoscalingEnabled; + } + + public Long getMinSize() { + return minSize; + } + + public Long getMaxSize() { + return maxSize; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_GROUP_CREATE; + } + + @Override + public String getEventDescription() { + return String.format("Creating node group %s for Kubernetes cluster ID: %s", getName(), getResourceUuid(ApiConstants.CLUSTER_ID)); + } + + @Override + public long getEntityOwnerId() { + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getClusterId()); + return cluster != null ? cluster.getAccountId() : CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public Long getApiResourceId() { + return getClusterId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + KubernetesClusterNodeGroup nodeGroup = kubernetesClusterService.createKubernetesClusterNodeGroup(this); + KubernetesClusterNodeGroupResponse response = kubernetesClusterService.createKubernetesClusterNodeGroupResponse(nodeGroup.getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/DeleteKubernetesClusterNodeGroupCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/DeleteKubernetesClusterNodeGroupCmd.java new file mode 100644 index 000000000000..3dae843163fc --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/DeleteKubernetesClusterNodeGroupCmd.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.nodegroup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterNodeGroupResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "deleteKubernetesClusterNodeGroup", + description = "Deletes a node group of a running CloudManaged Kubernetes cluster, draining and destroying all its nodes", + responseObject = SuccessResponse.class, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "24.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class DeleteKubernetesClusterNodeGroupCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = KubernetesClusterNodeGroupResponse.class, required = true, + description = "The ID of the node group") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_GROUP_DELETE; + } + + @Override + public String getEventDescription() { + return String.format("Deleting Kubernetes cluster node group ID: %s", getResourceUuid(ApiConstants.ID)); + } + + @Override + public long getEntityOwnerId() { + KubernetesClusterNodeGroup nodeGroup = _entityMgr.findById(KubernetesClusterNodeGroup.class, getId()); + if (nodeGroup != null) { + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, nodeGroup.getClusterId()); + if (cluster != null) { + return cluster.getAccountId(); + } + } + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + if (!kubernetesClusterService.deleteKubernetesClusterNodeGroup(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to delete Kubernetes cluster node group ID: %d", getId())); + } + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/ListKubernetesClusterNodeGroupsCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/ListKubernetesClusterNodeGroupsCmd.java new file mode 100644 index 000000000000..fa0a49cbad81 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/ListKubernetesClusterNodeGroupsCmd.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.nodegroup; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterNodeGroupResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ListResponse; + +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "listKubernetesClusterNodeGroups", + description = "Lists the node groups of a Kubernetes cluster", + responseObject = KubernetesClusterNodeGroupResponse.class, + responseView = ResponseView.Restricted, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "24.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListKubernetesClusterNodeGroupsCmd extends BaseListCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = KubernetesClusterNodeGroupResponse.class, + description = "The ID of the node group") + private Long id; + + @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = KubernetesClusterResponse.class, + description = "The ID of the Kubernetes cluster. Required when id is not passed") + private Long clusterId; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the node group") + private String name; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Long getClusterId() { + return clusterId; + } + + public String getName() { + return name; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + ListResponse response = kubernetesClusterService.listKubernetesClusterNodeGroups(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/ScaleKubernetesClusterNodeGroupCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/ScaleKubernetesClusterNodeGroupCmd.java new file mode 100644 index 000000000000..14955f635c79 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/ScaleKubernetesClusterNodeGroupCmd.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.nodegroup; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterNodeGroupResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "scaleKubernetesClusterNodeGroup", + description = "Scales a node group of a running CloudManaged Kubernetes cluster for its size or service offering", + responseObject = KubernetesClusterNodeGroupResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "24.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ScaleKubernetesClusterNodeGroupCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = KubernetesClusterNodeGroupResponse.class, required = true, + description = "The ID of the node group") + private Long id; + + @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, + description = "New number of nodes in the node group") + private Long size; + + @Parameter(name = ApiConstants.NODE_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = UserVmResponse.class, + description = "The IDs of the node group nodes to be removed. Can not be passed along with size or serviceofferingid") + private List nodeIds; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, + description = "The ID of the new service offering for the nodes of the node group") + private Long serviceOfferingId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Long getSize() { + return size; + } + + public List getNodeIds() { + return nodeIds; + } + + public Long getServiceOfferingId() { + return serviceOfferingId; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_GROUP_SCALE; + } + + @Override + public String getEventDescription() { + return String.format("Scaling Kubernetes cluster node group ID: %s", getResourceUuid(ApiConstants.ID)); + } + + @Override + public long getEntityOwnerId() { + KubernetesClusterNodeGroup nodeGroup = _entityMgr.findById(KubernetesClusterNodeGroup.class, getId()); + if (nodeGroup != null) { + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, nodeGroup.getClusterId()); + if (cluster != null) { + return cluster.getAccountId(); + } + } + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + KubernetesClusterNodeGroup nodeGroup = kubernetesClusterService.scaleKubernetesClusterNodeGroup(this); + KubernetesClusterNodeGroupResponse response = kubernetesClusterService.createKubernetesClusterNodeGroupResponse(nodeGroup.getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/UpdateKubernetesClusterNodeCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/UpdateKubernetesClusterNodeCmd.java new file mode 100644 index 000000000000..ab7e736a2b49 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/UpdateKubernetesClusterNodeCmd.java @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.nodegroup; + +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterNodeResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; + +@APICommand(name = "updateKubernetesClusterNode", + description = "Updates the Kubernetes labels and taints of a node of a node group, in addition to the labels and taints of " + + "its node group. A label or taint of the node overrides the label of the node group with the same key, or the " + + "taint of the node group with the same key and effect", + responseObject = KubernetesClusterNodeResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "24.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class UpdateKubernetesClusterNodeCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = UserVmResponse.class, required = true, + description = "The ID of the instance of the node") + private Long id; + + @Parameter(name = ApiConstants.NODE_LABELS, type = CommandType.MAP, + description = "New Kubernetes labels of the node, replacing its existing ones. " + + "Example: nodelabels[0].key=disktype&nodelabels[0].value=ssd") + private Map> labels; + + @Parameter(name = ApiConstants.NODE_TAINTS, type = CommandType.MAP, + description = "New Kubernetes taints of the node, replacing its existing ones. " + + "Example: nodetaints[0].key=gpu&nodetaints[0].value=true&nodetaints[0].effect=NoSchedule") + private Map> taints; + + @Parameter(name = ApiConstants.CLEANUP_NODE_LABELS, type = CommandType.BOOLEAN, + description = "Removes all the Kubernetes labels of the node, keeping the labels of its node group. Can not be passed along with nodelabels") + private Boolean cleanupLabels; + + @Parameter(name = ApiConstants.CLEANUP_NODE_TAINTS, type = CommandType.BOOLEAN, + description = "Removes all the Kubernetes taints of the node, keeping the taints of its node group. Can not be passed along with nodetaints") + private Boolean cleanupTaints; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Map> getLabels() { + return labels; + } + + public Map> getTaints() { + return taints; + } + + public boolean isCleanupLabels() { + return Boolean.TRUE.equals(cleanupLabels); + } + + public boolean isCleanupTaints() { + return Boolean.TRUE.equals(cleanupTaints); + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_UPDATE; + } + + @Override + public String getEventDescription() { + return String.format("Updating Kubernetes cluster node, instance ID: %s", getResourceUuid(ApiConstants.ID)); + } + + @Override + public long getEntityOwnerId() { + VirtualMachine vm = _entityMgr.findById(VirtualMachine.class, getId()); + return vm != null ? vm.getAccountId() : CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + KubernetesClusterNodeResponse response = kubernetesClusterService.updateKubernetesClusterNode(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/UpdateKubernetesClusterNodeGroupCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/UpdateKubernetesClusterNodeGroupCmd.java new file mode 100644 index 000000000000..47ccaaada256 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/nodegroup/UpdateKubernetesClusterNodeGroupCmd.java @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.nodegroup; + +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterNodeGroupResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "updateKubernetesClusterNodeGroup", + description = "Updates the Kubernetes labels and taints or the autoscaling settings of a node group. The new labels and taints " + + "replace the existing ones and are applied to the existing nodes of the node group as well as to the nodes added later", + responseObject = KubernetesClusterNodeGroupResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "24.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class UpdateKubernetesClusterNodeGroupCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = KubernetesClusterNodeGroupResponse.class, required = true, + description = "The ID of the node group") + private Long id; + + @Parameter(name = ApiConstants.NODE_LABELS, type = CommandType.MAP, + description = "New Kubernetes labels for the nodes of the node group, replacing the existing ones. " + + "Example: nodelabels[0].key=disktype&nodelabels[0].value=ssd") + private Map> labels; + + @Parameter(name = ApiConstants.NODE_TAINTS, type = CommandType.MAP, + description = "New Kubernetes taints for the nodes of the node group, replacing the existing ones. " + + "Example: nodetaints[0].key=gpu&nodetaints[0].value=true&nodetaints[0].effect=NoSchedule") + private Map> taints; + + @Parameter(name = ApiConstants.CLEANUP_NODE_LABELS, type = CommandType.BOOLEAN, + description = "Removes all the Kubernetes labels of the node group, can not be passed along with nodelabels") + private Boolean cleanupLabels; + + @Parameter(name = ApiConstants.CLEANUP_NODE_TAINTS, type = CommandType.BOOLEAN, + description = "Removes all the Kubernetes taints of the node group, can not be passed along with nodetaints") + private Boolean cleanupTaints; + + @Parameter(name = ApiConstants.AUTOSCALING_ENABLED, type = CommandType.BOOLEAN, + description = "Whether the size of the node group is managed by the cluster autoscaler, within its minsize and maxsize") + private Boolean autoscalingEnabled; + + @Parameter(name = ApiConstants.MIN_SIZE, type = CommandType.LONG, + description = "Minimum size of the node group when autoscaling is enabled") + private Long minSize; + + @Parameter(name = ApiConstants.MAX_SIZE, type = CommandType.LONG, + description = "Maximum size of the node group when autoscaling is enabled") + private Long maxSize; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Map> getLabels() { + return labels; + } + + public Map> getTaints() { + return taints; + } + + public boolean isCleanupLabels() { + return Boolean.TRUE.equals(cleanupLabels); + } + + public boolean isCleanupTaints() { + return Boolean.TRUE.equals(cleanupTaints); + } + + public Boolean isAutoscalingEnabled() { + return autoscalingEnabled; + } + + public Long getMinSize() { + return minSize; + } + + public Long getMaxSize() { + return maxSize; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODE_GROUP_UPDATE; + } + + @Override + public String getEventDescription() { + return String.format("Updating Kubernetes cluster node group ID: %s", getResourceUuid(ApiConstants.ID)); + } + + @Override + public long getEntityOwnerId() { + KubernetesClusterNodeGroup nodeGroup = _entityMgr.findById(KubernetesClusterNodeGroup.class, getId()); + if (nodeGroup != null) { + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, nodeGroup.getClusterId()); + if (cluster != null) { + return cluster.getAccountId(); + } + } + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + KubernetesClusterNodeGroup nodeGroup = kubernetesClusterService.updateKubernetesClusterNodeGroup(this); + KubernetesClusterNodeGroupResponse response = kubernetesClusterService.createKubernetesClusterNodeGroupResponse(nodeGroup.getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterNodeGroupResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterNodeGroupResponse.java new file mode 100644 index 000000000000..07dad99244d5 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterNodeGroupResponse.java @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@SuppressWarnings("unused") +@EntityReference(value = {KubernetesClusterNodeGroup.class}) +public class KubernetesClusterNodeGroupResponse extends BaseResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "The ID of the node group") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "The name of the node group") + private String name; + + @SerializedName(ApiConstants.IS_DEFAULT) + @Param(description = "Whether the node group is the default node group of the Kubernetes cluster, which holds the worker nodes " + + "of the cluster size and can not be deleted") + private Boolean isDefault; + + @SerializedName(ApiConstants.STATE) + @Param(description = "The state of the node group") + private String state; + + @SerializedName(ApiConstants.CLUSTER_ID) + @Param(description = "The ID of the Kubernetes cluster of the node group") + private String clusterId; + + @SerializedName(ApiConstants.CLUSTER_NAME) + @Param(description = "The name of the Kubernetes cluster of the node group") + private String clusterName; + + @SerializedName(ApiConstants.SERVICE_OFFERING_ID) + @Param(description = "The ID of the service offering of the node group nodes") + private String serviceOfferingId; + + @SerializedName(ApiConstants.SERVICE_OFFERING_NAME) + @Param(description = "The name of the service offering of the node group nodes") + private String serviceOfferingName; + + @SerializedName(ApiConstants.CPU_NUMBER) + @Param(description = "The number of CPU cores of each node of the node group") + private Integer cpuNumber; + + @SerializedName(ApiConstants.MEMORY) + @Param(description = "The memory (in MiB) of each node of the node group") + private Integer memory; + + @SerializedName(ApiConstants.TEMPLATE_ID) + @Param(description = "The ID of the template of the node group nodes") + private String templateId; + + @SerializedName(ApiConstants.TEMPLATE_NAME) + @Param(description = "The name of the template of the node group nodes") + private String templateName; + + @SerializedName(ApiConstants.SIZE) + @Param(description = "The number of nodes in the node group") + private Long size; + + @SerializedName(ApiConstants.NODE_LABELS) + @Param(description = "The Kubernetes labels of the node group nodes") + private Map labels; + + @SerializedName(ApiConstants.NODE_TAINTS) + @Param(description = "The Kubernetes taints of the node group nodes, in the key[=value]:effect format") + private List taints; + + @SerializedName(ApiConstants.AUTOSCALING_ENABLED) + @Param(description = "Whether the size of the node group is managed by the cluster autoscaler") + private Boolean autoscalingEnabled; + + @SerializedName(ApiConstants.MIN_SIZE) + @Param(description = "The minimum size of the node group when autoscaling is enabled") + private Long minSize; + + @SerializedName(ApiConstants.MAX_SIZE) + @Param(description = "The maximum size of the node group when autoscaling is enabled") + private Long maxSize; + + @SerializedName(ApiConstants.VIRTUAL_MACHINE_IDS) + @Param(description = "The IDs of the instances of the node group nodes") + private List virtualMachineIds; + + @SerializedName("nodes") + @Param(description = "The nodes of the node group, with their own Kubernetes labels and taints", + responseObject = KubernetesClusterNodeResponse.class) + private List nodes; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "The date when the node group was created") + private Date created; + + public KubernetesClusterNodeGroupResponse() { + setObjectName("kubernetesclusternodegroup"); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Boolean getDefault() { + return isDefault; + } + + public void setDefault(Boolean isDefault) { + this.isDefault = isDefault; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public void setClusterId(String clusterId) { + this.clusterId = clusterId; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public void setServiceOfferingId(String serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + public void setServiceOfferingName(String serviceOfferingName) { + this.serviceOfferingName = serviceOfferingName; + } + + public void setCpuNumber(Integer cpuNumber) { + this.cpuNumber = cpuNumber; + } + + public void setMemory(Integer memory) { + this.memory = memory; + } + + public void setTemplateId(String templateId) { + this.templateId = templateId; + } + + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + + public List getTaints() { + return taints; + } + + public void setTaints(List taints) { + this.taints = taints; + } + + public Boolean getAutoscalingEnabled() { + return autoscalingEnabled; + } + + public void setAutoscalingEnabled(Boolean autoscalingEnabled) { + this.autoscalingEnabled = autoscalingEnabled; + } + + public Long getMinSize() { + return minSize; + } + + public void setMinSize(Long minSize) { + this.minSize = minSize; + } + + public Long getMaxSize() { + return maxSize; + } + + public void setMaxSize(Long maxSize) { + this.maxSize = maxSize; + } + + public List getVirtualMachineIds() { + return virtualMachineIds; + } + + public void setVirtualMachineIds(List virtualMachineIds) { + this.virtualMachineIds = virtualMachineIds; + } + + public List getNodes() { + return nodes; + } + + public void setNodes(List nodes) { + this.nodes = nodes; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterNodeResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterNodeResponse.java new file mode 100644 index 000000000000..569308563a2e --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterNodeResponse.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@SuppressWarnings("unused") +public class KubernetesClusterNodeResponse extends BaseResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "The ID of the instance of the node") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "The name of the instance of the node") + private String name; + + @SerializedName(ApiConstants.HOST_NAME) + @Param(description = "The host name of the instance, which is the name of the node in the Kubernetes cluster") + private String hostName; + + @SerializedName(ApiConstants.STATE) + @Param(description = "The state of the instance of the node") + private String state; + + @SerializedName(ApiConstants.IP_ADDRESS) + @Param(description = "The IP address of the instance of the node in the network of the Kubernetes cluster") + private String ipAddress; + + @SerializedName(ApiConstants.NODE_GROUP_ID) + @Param(description = "The ID of the node group of the node") + private String nodeGroupId; + + @SerializedName(ApiConstants.NODE_GROUP_NAME) + @Param(description = "The name of the node group of the node") + private String nodeGroupName; + + @SerializedName(ApiConstants.NODE_LABELS) + @Param(description = "The Kubernetes labels of the node, in addition to the labels of its node group") + private Map labels; + + @SerializedName(ApiConstants.NODE_TAINTS) + @Param(description = "The Kubernetes taints of the node in the key[=value]:effect format, in addition to the taints of its node group") + private List taints; + + public KubernetesClusterNodeResponse() { + setObjectName("kubernetesclusternode"); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public void setState(String state) { + this.state = state; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public void setNodeGroupId(String nodeGroupId) { + this.nodeGroupId = nodeGroupId; + } + + public void setNodeGroupName(String nodeGroupName) { + this.nodeGroupName = nodeGroupName; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + + public List getTaints() { + return taints; + } + + public void setTaints(List taints) { + this.taints = taints; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 932d722de354..af4da628c7e5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -157,9 +157,13 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple private Long controlNodes; @SerializedName(ApiConstants.SIZE) - @Param(description = "The size (worker nodes count) of the Kubernetes cluster") + @Param(description = "The size of the Kubernetes cluster: the number of worker nodes of its default node group and of its external nodes") private Long clusterSize; + @SerializedName(ApiConstants.WORKER_NODES) + @Param(description = "The total number of worker nodes of the Kubernetes cluster, in all its node groups and external nodes", since = "24.0") + private Long workerNodes; + @SerializedName(ApiConstants.STATE) @Param(description = "The state of the Kubernetes cluster") private String state; @@ -184,6 +188,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "The list of virtualmachines associated with this Kubernetes cluster") private List virtualMachines; + @SerializedName(ApiConstants.NODE_GROUPS) + @Param(description = "The node groups of the Kubernetes cluster", responseObject = KubernetesClusterNodeGroupResponse.class, since = "24.0") + private List nodeGroups; + @SerializedName(ApiConstants.IP_ADDRESS) @Param(description = "Public IP Address of the cluster") private String ipAddress; @@ -400,6 +408,14 @@ public void setClusterSize(Long clusterSize) { this.clusterSize = clusterSize; } + public Long getWorkerNodes() { + return workerNodes; + } + + public void setWorkerNodes(Long workerNodes) { + this.workerNodes = workerNodes; + } + public String getCores() { return cores; } @@ -508,6 +524,14 @@ public void setVirtualMachines(List virtualMachines) { this.virtualMachines = virtualMachines; } + public void setNodeGroups(List nodeGroups) { + this.nodeGroups = nodeGroups; + } + + public List getNodeGroups() { + return nodeGroups; + } + public List getVirtualMachines() { return virtualMachines; } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml index 053366786292..3e3b1d4d9b1c 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -33,6 +33,7 @@ + diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index fd9617e69dec..58d8a3b497c2 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -251,6 +251,15 @@ write_files: if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then export PATH=$PATH:/opt/bin fi + KUBELET_NODE_ARGS="{{ k8s.kubelet.extra.args }}" + if [ -n "${KUBELET_NODE_ARGS}" ] && ! grep -qF -- "${KUBELET_NODE_ARGS}" /etc/default/kubelet 2>/dev/null; then + if grep -q "^KUBELET_EXTRA_ARGS=" /etc/default/kubelet 2>/dev/null; then + sed -i "s|^KUBELET_EXTRA_ARGS=\(.*\)$|KUBELET_EXTRA_ARGS=\1 ${KUBELET_NODE_ARGS}|" /etc/default/kubelet + else + echo "KUBELET_EXTRA_ARGS=${KUBELET_NODE_ARGS}" >> /etc/default/kubelet + fi + systemctl daemon-reload + fi kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification sudo touch /home/cloud/success diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/autoscale-kube-cluster b/plugins/integrations/kubernetes-service/src/main/resources/script/autoscale-kube-cluster index ac3fd8eab4ce..332d8cceee18 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/autoscale-kube-cluster +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/autoscale-kube-cluster @@ -26,10 +26,13 @@ Arguments: -d, --disable Disables autoscaling -M, --maxsize number Maximum size of the cluster -m, --minsize number Minimum size of the cluster + -n, --nodes string Node group to autoscale, as ::. Can be repeated, replaces -i, -M and -m + -I, --image string Container image of the cluster autoscaler, instead of the image of the manifest Other arguments: -h, --help Display this help message and exit Examples: - ./autoscale-kube-cluster -e -M 3 -m 1 + ./autoscale-kube-cluster -i -e -M 3 -m 1 + ./autoscale-kube-cluster -e -n 1:3: -n 0:5: ./autoscale-kube-cluster -d USAGE exit 0 @@ -38,6 +41,8 @@ ID="" ENABLE="" MINSIZE="" MAXSIZE="" +NODES=() +IMAGE="" while [ -n "$1" ]; do case "$1" in -h | --help) @@ -51,7 +56,7 @@ while [ -n "$1" ]; do ENABLE="true" shift 1 ;; - -d | --enable) + -d | --disable) ENABLE="false" shift 1 ;; @@ -63,31 +68,67 @@ while [ -n "$1" ]; do MINSIZE=$2 shift 2 ;; + -n | --nodes) + NODES+=("$2") + shift 2 + ;; + -I | --image) + IMAGE=$2 + shift 2 + ;; -*|*) echo "ERROR: no such option $1. -h or --help for help" exit 1 ;; esac done -if [ $ENABLE == "true" ] ; then - if [ -e /opt/autoscaler/autoscaler_tmpl.yaml ]; then - sed -e "s//$ID/g" -e "s//$MINSIZE/g" -e "s//$MAXSIZE/g" /opt/autoscaler/autoscaler_tmpl.yaml > /opt/autoscaler/autoscaler_now.yaml - /opt/bin/kubectl apply -f /opt/autoscaler/autoscaler_now.yaml - exit 0 - else + +AUTOSCALER_TEMPLATE="/opt/autoscaler/autoscaler_tmpl.yaml" +AUTOSCALER_MANIFEST="/opt/autoscaler/autoscaler_now.yaml" + +# Writes the autoscaler manifest from the template, with one --nodes argument per node group in place of the +# --nodes=:: argument of the template, and the given image if any +function render_manifest() { + awk -v nodes="${NODES[*]}" ' + /--nodes=::/ { + match($0, /^[ \t]*-[ \t]*/) + indent = substr($0, 1, RLENGTH) + count = split(nodes, specs, " ") + for (i = 1; i <= count; i++) { + print indent "--nodes=" specs[i] + } + next + } + { print } + ' "${AUTOSCALER_TEMPLATE}" > "${AUTOSCALER_MANIFEST}" + if [ -n "${IMAGE}" ]; then + sed -i -E "s#^([[:space:]]*-?[[:space:]]*image:[[:space:]]*).*#\1${IMAGE}#" "${AUTOSCALER_MANIFEST}" + fi +} + +if [ "$ENABLE" == "true" ] ; then + if [ ${#NODES[@]} -eq 0 ]; then + if [ -z "$ID" ] || [ -z "$MINSIZE" ] || [ -z "$MAXSIZE" ]; then + echo "ERROR: either -n or -i, -m and -M must be passed to enable autoscaling" + exit 1 + fi + NODES=("${MINSIZE}:${MAXSIZE}:${ID}") + fi + if [ ! -e "${AUTOSCALER_TEMPLATE}" ]; then mkdir -p /opt/autoscaler AUTOSCALER_URL="https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" - autoscaler_conf_file="/opt/autoscaler/autoscaler_tmpl.yaml" - curl -sSL ${AUTOSCALER_URL} -o ${autoscaler_conf_file} - if [ $? -ne 0 ]; then + if ! curl -sSL ${AUTOSCALER_URL} -o "${AUTOSCALER_TEMPLATE}"; then + rm -f "${AUTOSCALER_TEMPLATE}" echo "Unable to connect to the internet to download the autoscaler deployment and image" exit 1 - else - sed -e "s//$ID/g" -e "s//$MINSIZE/g" -e "s//$MAXSIZE/g" /opt/autoscaler/autoscaler_tmpl.yaml > /opt/autoscaler/autoscaler_now.yaml - /opt/bin/kubectl apply -f /opt/autoscaler/autoscaler_now.yaml - exit 0 fi fi + render_manifest + /opt/bin/kubectl apply -f "${AUTOSCALER_MANIFEST}" + exit 0 +elif [ "$ENABLE" == "false" ] ; then + /opt/bin/kubectl delete deployment -n kube-system cluster-autoscaler --ignore-not-found else - /opt/bin/kubectl delete deployment -n kube-system cluster-autoscaler + echo "ERROR: either -e or -d must be passed. -h or --help for help" + exit 1 fi diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 1fab5420c3c3..6df2ba034c5e 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -28,6 +28,8 @@ import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterNodeGroupDao; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeGroupUtil; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.network.Network; @@ -38,6 +40,7 @@ import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; @@ -76,6 +79,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; @@ -118,6 +122,9 @@ public class KubernetesClusterManagerImplTest { @Mock private HostDao hostDao; + @Mock + private KubernetesClusterNodeGroupDao kubernetesClusterNodeGroupDao; + @Spy @InjectMocks KubernetesClusterManagerImpl kubernetesClusterManager; @@ -914,4 +921,181 @@ public void testValidateNodeAffinityGroupsAffinityNoExistingWorkers() { Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()); } + private VMTemplateVO mockNodeGroupTemplate(long id, boolean forCks, Hypervisor.HypervisorType hypervisorType, CPU.CPUArch arch) { + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.lenient().when(template.getId()).thenReturn(id); + Mockito.lenient().when(template.getName()).thenReturn("template-" + id); + Mockito.lenient().when(template.isForCks()).thenReturn(forCks); + Mockito.lenient().when(template.getHypervisorType()).thenReturn(hypervisorType); + Mockito.lenient().when(template.getArch()).thenReturn(arch); + Mockito.lenient().when(templateDao.findById(id)).thenReturn(template); + return template; + } + + private KubernetesClusterVO mockNodeGroupCluster() { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.lenient().when(cluster.getName()).thenReturn("cluster"); + Mockito.lenient().when(cluster.getTemplateId()).thenReturn(1L); + Mockito.lenient().when(cluster.getWorkerNodeTemplateId()).thenReturn(2L); + Mockito.lenient().when(cluster.getZoneId()).thenReturn(5L); + mockNodeGroupTemplate(1L, false, Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64); + return cluster; + } + + @Test + public void testValidateNodeGroupTemplateDefaultsToWorkerTemplate() { + KubernetesClusterVO cluster = mockNodeGroupCluster(); + VMTemplateVO workerTemplate = mockNodeGroupTemplate(2L, true, Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64); + Mockito.when(templateJoinDao.newTemplateView(workerTemplate, 5L, true)).thenReturn(List.of(Mockito.mock(TemplateJoinVO.class))); + Assert.assertSame(workerTemplate, kubernetesClusterManager.validateNodeGroupTemplate(null, cluster)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupTemplateNotForCks() { + KubernetesClusterVO cluster = mockNodeGroupCluster(); + mockNodeGroupTemplate(3L, false, Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64); + kubernetesClusterManager.validateNodeGroupTemplate(3L, cluster); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupTemplateDifferentArch() { + KubernetesClusterVO cluster = mockNodeGroupCluster(); + mockNodeGroupTemplate(3L, true, Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64); + kubernetesClusterManager.validateNodeGroupTemplate(3L, cluster); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupTemplateNotReady() { + KubernetesClusterVO cluster = mockNodeGroupCluster(); + VMTemplateVO template = mockNodeGroupTemplate(3L, true, Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64); + Mockito.when(templateJoinDao.newTemplateView(template, 5L, true)).thenReturn(Collections.emptyList()); + kubernetesClusterManager.validateNodeGroupTemplate(3L, cluster); + } + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupSizeDefaultNodeGroupEmpty() { + kubernetesClusterManager.validateNodeGroupSize(true, false, null, null, 0); + } + + @Test + public void testValidateNodeGroupSizeEmptyNodeGroup() { + kubernetesClusterManager.validateNodeGroupSize(false, false, null, null, 0); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupSizeAboveAutoscalingMaxSize() { + kubernetesClusterManager.validateNodeGroupSize(false, true, 1L, 3L, 4); + } + + @Test + public void testValidateNodeGroupSizeWithinAutoscalingRange() { + kubernetesClusterManager.validateNodeGroupSize(false, true, 0L, 3L, 3); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupTaintsDefaultNodeGroupNoSchedule() { + kubernetesClusterManager.validateNodeGroupTaints(true, List.of(new KubernetesClusterNodeGroupUtil.Taint("gpu", "true", "NoSchedule"))); + } + + @Test + public void testValidateNodeGroupTaintsDefaultNodeGroupPreferNoSchedule() { + kubernetesClusterManager.validateNodeGroupTaints(true, List.of(new KubernetesClusterNodeGroupUtil.Taint("gpu", "true", "PreferNoSchedule"))); + kubernetesClusterManager.validateNodeGroupTaints(false, List.of(new KubernetesClusterNodeGroupUtil.Taint("gpu", "true", "NoExecute"))); + } + + private void mockKubernetesClusterLock() { + Mockito.doAnswer(invocation -> ((Supplier) invocation.getArgument(2)).get()) + .when(kubernetesClusterManager).executeWithKubernetesClusterLock(Mockito.anyLong(), Mockito.anyString(), Mockito.any()); + } + + private KubernetesClusterNodeGroupVO mockNodeGroup(long id, KubernetesClusterNodeGroup.State state) { + KubernetesClusterNodeGroupVO nodeGroup = Mockito.mock(KubernetesClusterNodeGroupVO.class); + Mockito.lenient().when(nodeGroup.getId()).thenReturn(id); + Mockito.lenient().when(nodeGroup.getName()).thenReturn("group-" + id); + Mockito.lenient().when(nodeGroup.getState()).thenReturn(state); + Mockito.lenient().when(kubernetesClusterNodeGroupDao.findById(id)).thenReturn(nodeGroup); + return nodeGroup; + } + + private KubernetesClusterVO mockClusterInState(KubernetesCluster.State state) { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.lenient().when(cluster.getName()).thenReturn("cluster"); + Mockito.lenient().when(cluster.getState()).thenReturn(state); + Mockito.when(kubernetesClusterDao.findById(1L)).thenReturn(cluster); + return cluster; + } + + @Test + public void testStartNodeGroupOperationOnRunningCluster() { + mockKubernetesClusterLock(); + mockClusterInState(KubernetesCluster.State.Running); + KubernetesClusterNodeGroupVO nodeGroup = mockNodeGroup(2L, KubernetesClusterNodeGroup.State.Ready); + Mockito.when(kubernetesClusterNodeGroupDao.listByClusterId(1L)).thenReturn(List.of(nodeGroup)); + Mockito.doReturn(true).when(kubernetesClusterManager).stateTransitTo(1L, KubernetesCluster.Event.ScaleUpRequested); + + kubernetesClusterManager.startNodeGroupOperation(1L, nodeGroup, KubernetesClusterNodeGroup.State.Scaling, false); + + Mockito.verify(nodeGroup).setState(KubernetesClusterNodeGroup.State.Scaling); + Mockito.verify(kubernetesClusterNodeGroupDao).update(2L, nodeGroup); + } + + @Test + public void testStartNodeGroupOperationWhileOtherNodeGroupScaling() { + mockKubernetesClusterLock(); + mockClusterInState(KubernetesCluster.State.Scaling); + KubernetesClusterNodeGroupVO nodeGroup = mockNodeGroup(2L, KubernetesClusterNodeGroup.State.Ready); + KubernetesClusterNodeGroupVO otherNodeGroup = mockNodeGroup(3L, KubernetesClusterNodeGroup.State.Scaling); + Mockito.when(kubernetesClusterNodeGroupDao.listByClusterId(1L)).thenReturn(List.of(nodeGroup, otherNodeGroup)); + + kubernetesClusterManager.startNodeGroupOperation(1L, nodeGroup, KubernetesClusterNodeGroup.State.Updating, false); + + Mockito.verify(kubernetesClusterManager, Mockito.never()).stateTransitTo(Mockito.anyLong(), Mockito.any()); + Mockito.verify(nodeGroup).setState(KubernetesClusterNodeGroup.State.Updating); + } + + @Test(expected = PermissionDeniedException.class) + public void testStartNodeGroupOperationWhileClusterScaling() { + mockKubernetesClusterLock(); + mockClusterInState(KubernetesCluster.State.Scaling); + KubernetesClusterNodeGroupVO nodeGroup = mockNodeGroup(2L, KubernetesClusterNodeGroup.State.Ready); + Mockito.when(kubernetesClusterNodeGroupDao.listByClusterId(1L)).thenReturn(List.of(nodeGroup)); + + kubernetesClusterManager.startNodeGroupOperation(1L, nodeGroup, KubernetesClusterNodeGroup.State.Scaling, false); + } + + @Test(expected = PermissionDeniedException.class) + public void testStartNodeGroupOperationOnBusyNodeGroup() { + mockKubernetesClusterLock(); + mockClusterInState(KubernetesCluster.State.Scaling); + KubernetesClusterNodeGroupVO nodeGroup = mockNodeGroup(2L, KubernetesClusterNodeGroup.State.Deleting); + + kubernetesClusterManager.startNodeGroupOperation(1L, nodeGroup, KubernetesClusterNodeGroup.State.Scaling, false); + } + + @Test + public void testFinishLastNodeGroupOperation() { + mockKubernetesClusterLock(); + mockClusterInState(KubernetesCluster.State.Scaling); + KubernetesClusterNodeGroupVO nodeGroup = mockNodeGroup(2L, KubernetesClusterNodeGroup.State.Ready); + Mockito.when(kubernetesClusterNodeGroupDao.listByClusterId(1L)).thenReturn(List.of(nodeGroup)); + Mockito.doReturn(true).when(kubernetesClusterManager).stateTransitTo(1L, KubernetesCluster.Event.OperationSucceeded); + + kubernetesClusterManager.finishNodeGroupOperation(1L, 2L, false); + + Mockito.verify(nodeGroup).setState(KubernetesClusterNodeGroup.State.Error); + Mockito.verify(kubernetesClusterManager).stateTransitTo(1L, KubernetesCluster.Event.OperationSucceeded); + } + + @Test + public void testFinishNodeGroupOperationWhileOtherNodeGroupScaling() { + mockKubernetesClusterLock(); + mockClusterInState(KubernetesCluster.State.Scaling); + KubernetesClusterNodeGroupVO nodeGroup = mockNodeGroup(2L, KubernetesClusterNodeGroup.State.Ready); + KubernetesClusterNodeGroupVO otherNodeGroup = mockNodeGroup(3L, KubernetesClusterNodeGroup.State.Scaling); + Mockito.when(kubernetesClusterNodeGroupDao.listByClusterId(1L)).thenReturn(List.of(nodeGroup, otherNodeGroup)); + + kubernetesClusterManager.finishNodeGroupOperation(1L, 2L, true); + + Mockito.verify(nodeGroup).setState(KubernetesClusterNodeGroup.State.Ready); + Mockito.verify(kubernetesClusterManager, Mockito.never()).stateTransitTo(Mockito.anyLong(), Mockito.any()); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeGroupWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeGroupWorkerTest.java new file mode 100644 index 000000000000..4c515e61d36d --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterNodeGroupWorkerTest.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroupVO; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.logging.log4j.Level; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterNodeGroupWorkerTest { + + @Mock + private KubernetesCluster kubernetesCluster; + @Mock + private KubernetesClusterManagerImpl clusterManager; + @Mock + private ServiceOfferingDao serviceOfferingDao; + + private KubernetesClusterNodeGroupVO nodeGroup; + private KubernetesClusterNodeGroupWorker worker; + + @Before + public void setUp() { + nodeGroup = new KubernetesClusterNodeGroupVO(1L, "gpu", 10L, 20L, "disktype=ssd", "gpu=true:NoSchedule"); + worker = new KubernetesClusterNodeGroupWorker(kubernetesCluster, nodeGroup, clusterManager); + worker.serviceOfferingDao = serviceOfferingDao; + } + + @Test + public void testGetWorkerNodeHostName() { + worker.kubernetesClusterNodeNamePrefix = "mycluster"; + Assert.assertEquals("mycluster-gpu-18f2a3b4c5d", worker.getWorkerNodeHostName("18f2a3b4c5d")); + } + + @Test + public void testGetWorkerNodeHostNameTruncatesPrefix() { + worker.kubernetesClusterNodeNamePrefix = "a-very-long-kubernetes-cluster-name-prefix-of-43-chars"; + String hostName = worker.getWorkerNodeHostName("18f2a3b4c5d"); + Assert.assertTrue(hostName.length() <= 63); + Assert.assertTrue(hostName.endsWith("-gpu-18f2a3b4c5d")); + Assert.assertFalse(hostName.contains("--")); + } + + @Test + public void testGetWorkerNodeKubeletExtraArgs() { + Assert.assertEquals("--node-labels=cloudstack.apache.org/nodegroup=gpu,disktype=ssd --register-with-taints=gpu=true:NoSchedule", + worker.getWorkerNodeKubeletExtraArgs()); + } + + @Test + public void testGetServiceOfferingForWorkerNodeIsNodeGroupOffering() { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(10L)).thenReturn(offering); + ServiceOffering result = worker.getServiceOfferingForNodeTypeOnCluster(WORKER, kubernetesCluster); + Assert.assertSame(offering, result); + } + @Test + public void testGetWorkerNodeHostNameDefaultNodeGroup() { + nodeGroup.setDefault(true); + worker.kubernetesClusterNodeNamePrefix = "mycluster"; + Assert.assertEquals("mycluster-node-18f2a3b4c5d", worker.getWorkerNodeHostName("18f2a3b4c5d")); + } + + @Test + public void testGetWorkerNodeGroupId() { + Assert.assertSame(nodeGroup, worker.getWorkerNodeGroup()); + } + + @Test + public void testFailureDoesNotChangeClusterState() { + KubernetesClusterNodeGroupWorker spyWorker = Mockito.spy(worker); + try { + spyWorker.logTransitStateAndThrow(Level.ERROR, "failure", 1L, KubernetesCluster.Event.OperationFailed, null); + Assert.fail("Expected an exception"); + } catch (CloudRuntimeException e) { + Assert.assertEquals("failure", e.getMessage()); + } + Mockito.verify(spyWorker, Mockito.never()).stateTransitTo(Mockito.anyLong(), Mockito.any()); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index c9299bdbaa67..b379e39fe961 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -58,6 +58,7 @@ public class KubernetesClusterScaleWorkerTest { private KubernetesClusterScaleWorker worker; private static final Long defaultOfferingId = 1L; + private static final Long DEFAULT_NODE_GROUP_ID = 10L; @Before public void setUp() { @@ -135,6 +136,7 @@ public void testGetWorkerNodesToRemoveForDownsize_singleRemoval() { KubernetesClusterManagerImpl clusterManager = Mockito.mock(KubernetesClusterManagerImpl.class); KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 2L, null, false, null, null, clusterManager); KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + Mockito.doReturn(DEFAULT_NODE_GROUP_ID).when(spyWorker).getDefaultNodeGroupId(); KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); Mockito.when(vm1.isExternalNode()).thenReturn(false); @@ -165,6 +167,7 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 3L, null, false, null, null, clusterManager); KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + Mockito.doReturn(DEFAULT_NODE_GROUP_ID).when(spyWorker).getDefaultNodeGroupId(); KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); Mockito.when(vm1.isExternalNode()).thenReturn(false); @@ -187,4 +190,24 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { Assert.assertTrue(toRemove.isEmpty()); } + + @Test + public void testGetWorkerNodesToRemoveExcludesNodeGroupNodes() { + KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); + KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 1L, null, false, null, null, clusterManager); + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + Mockito.doReturn(DEFAULT_NODE_GROUP_ID).when(spyWorker).getDefaultNodeGroupId(); + + KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); + KubernetesClusterVmMapVO vm2 = Mockito.mock(KubernetesClusterVmMapVO.class); + KubernetesClusterVmMapVO nodeGroupVm = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(nodeGroupVm.isInNodeGroupOtherThan(DEFAULT_NODE_GROUP_ID)).thenReturn(true); + + Mockito.doReturn(Arrays.asList(vm1, vm2, nodeGroupVm)).when(spyWorker).getKubernetesClusterVMMaps(); + + List toRemove = spyWorker.getWorkerNodesToRemove(); + + Assert.assertEquals(1, toRemove.size()); + Assert.assertSame(vm2, toRemove.get(0)); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeGroupUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeGroupUtilTest.java new file mode 100644 index 000000000000..cc8e4bcec5ce --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeGroupUtilTest.java @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.utils; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.KubernetesClusterNodeGroup; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeGroupUtil.Taint; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterNodeGroupUtilTest { + + private Map> apiMap(String[]... entries) { + Map> map = new LinkedHashMap<>(); + int i = 0; + for (String[] entry : entries) { + Map values = new HashMap<>(); + values.put(KubernetesClusterNodeGroupUtil.KEY, entry[0]); + values.put(KubernetesClusterNodeGroupUtil.VALUE, entry[1]); + if (entry.length > 2) { + values.put(KubernetesClusterNodeGroupUtil.EFFECT, entry[2]); + } + map.put(String.valueOf(i++), values); + } + return map; + } + + @Test + public void testValidateNodeGroupNameValid() { + KubernetesClusterNodeGroupUtil.validateNodeGroupName("gpu-pool-1"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupNameUpperCase() { + KubernetesClusterNodeGroupUtil.validateNodeGroupName("GPU"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupNameTooLong() { + KubernetesClusterNodeGroupUtil.validateNodeGroupName("a-very-long-node-group-name"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateNodeGroupNameTrailingDash() { + KubernetesClusterNodeGroupUtil.validateNodeGroupName("pool-"); + } + + @Test + public void testGetLabelsFromApiMap() { + Map labels = KubernetesClusterNodeGroupUtil.getLabelsFromApiMap( + apiMap(new String[]{"disktype", "ssd"}, new String[]{"example.com/team", "data"}, new String[]{"empty", ""})); + Assert.assertEquals(3, labels.size()); + Assert.assertEquals("ssd", labels.get("disktype")); + Assert.assertEquals("data", labels.get("example.com/team")); + Assert.assertEquals("disktype=ssd,example.com/team=data,empty=", KubernetesClusterNodeGroupUtil.labelsToString(labels)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetLabelsFromApiMapRestrictedNamespace() { + KubernetesClusterNodeGroupUtil.getLabelsFromApiMap(apiMap(new String[]{"node-role.kubernetes.io/worker", ""})); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetLabelsFromApiMapReservedNodeGroupLabel() { + KubernetesClusterNodeGroupUtil.getLabelsFromApiMap(apiMap(new String[]{KubernetesClusterNodeGroup.NODE_GROUP_LABEL, "other"})); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetLabelsFromApiMapInvalidValue() { + KubernetesClusterNodeGroupUtil.getLabelsFromApiMap(apiMap(new String[]{"key", "value;rm -rf /"})); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetLabelsFromApiMapDuplicateKey() { + KubernetesClusterNodeGroupUtil.getLabelsFromApiMap(apiMap(new String[]{"key", "a"}, new String[]{"key", "b"})); + } + + @Test + public void testGetTaintsFromApiMap() { + List taints = KubernetesClusterNodeGroupUtil.getTaintsFromApiMap( + apiMap(new String[]{"gpu", "true", "NoSchedule"}, new String[]{"dedicated", "", "NoExecute"})); + Assert.assertEquals(2, taints.size()); + Assert.assertEquals("gpu=true:NoSchedule,dedicated:NoExecute", KubernetesClusterNodeGroupUtil.taintsToString(taints)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetTaintsFromApiMapInvalidEffect() { + KubernetesClusterNodeGroupUtil.getTaintsFromApiMap(apiMap(new String[]{"gpu", "true", "Never"})); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetTaintsFromApiMapDuplicateKeyAndEffect() { + KubernetesClusterNodeGroupUtil.getTaintsFromApiMap(apiMap(new String[]{"gpu", "a", "NoSchedule"}, new String[]{"gpu", "b", "NoSchedule"})); + } + + @Test + public void testTaintsFromStringRoundTrip() { + String taints = "gpu=true:NoSchedule,dedicated:NoExecute,example.com/key=v:PreferNoSchedule"; + List list = KubernetesClusterNodeGroupUtil.taintsFromString(taints); + Assert.assertEquals(3, list.size()); + Assert.assertEquals(new Taint("dedicated", "", "NoExecute"), list.get(1)); + Assert.assertEquals(taints, KubernetesClusterNodeGroupUtil.taintsToString(list)); + } + + @Test + public void testLabelsAndTaintsFromEmptyString() { + Assert.assertTrue(KubernetesClusterNodeGroupUtil.labelsFromString(null).isEmpty()); + Assert.assertTrue(KubernetesClusterNodeGroupUtil.taintsFromString("").isEmpty()); + Assert.assertNull(KubernetesClusterNodeGroupUtil.labelsToString(new HashMap<>())); + Assert.assertNull(KubernetesClusterNodeGroupUtil.taintsToString(List.of())); + } + + @Test + public void testGetKubeletExtraArgs() { + KubernetesClusterNodeGroup nodeGroup = Mockito.mock(KubernetesClusterNodeGroup.class); + Mockito.when(nodeGroup.getName()).thenReturn("gpu"); + Mockito.when(nodeGroup.getLabels()).thenReturn("disktype=ssd"); + Mockito.when(nodeGroup.getTaints()).thenReturn("gpu=true:NoSchedule"); + Assert.assertEquals("--node-labels=cloudstack.apache.org/nodegroup=gpu,disktype=ssd --register-with-taints=gpu=true:NoSchedule", + KubernetesClusterNodeGroupUtil.getKubeletExtraArgs(nodeGroup)); + } + + @Test + public void testGetKubeletExtraArgsWithoutLabelsAndTaints() { + KubernetesClusterNodeGroup nodeGroup = Mockito.mock(KubernetesClusterNodeGroup.class); + Mockito.when(nodeGroup.getName()).thenReturn("pool"); + Assert.assertEquals("--node-labels=cloudstack.apache.org/nodegroup=pool", + KubernetesClusterNodeGroupUtil.getKubeletExtraArgs(nodeGroup)); + } + + @Test + public void testGetLabelAndTaintUpdateCommands() { + List commands = KubernetesClusterNodeGroupUtil.getLabelAndTaintUpdateCommands("node-1", "pool", + "a=1,b=2", "b=3,c=4", + "gpu=true:NoSchedule,old:NoExecute", "gpu=false:NoSchedule"); + Assert.assertEquals(List.of( + "sudo /opt/bin/kubectl label node node-1 a- cloudstack.apache.org/nodegroup=pool b=3 c=4 --overwrite", + "sudo /opt/bin/kubectl taint node node-1 old:NoExecute- || true", + "sudo /opt/bin/kubectl taint node node-1 gpu=false:NoSchedule --overwrite"), commands); + } + + @Test + public void testGetLabelAndTaintUpdateCommandsCleanup() { + List commands = KubernetesClusterNodeGroupUtil.getLabelAndTaintUpdateCommands("node-1", "pool", + "a=1", null, "gpu=true:NoSchedule", null); + Assert.assertEquals(List.of( + "sudo /opt/bin/kubectl label node node-1 a- cloudstack.apache.org/nodegroup=pool --overwrite", + "sudo /opt/bin/kubectl taint node node-1 gpu:NoSchedule- || true"), commands); + } + + @Test + public void testGetLabelAndTaintUpdateCommandsOnlyNodeGroupLabel() { + Assert.assertEquals(List.of("sudo /opt/bin/kubectl label node node-1 cloudstack.apache.org/nodegroup=default --overwrite"), + KubernetesClusterNodeGroupUtil.getLabelAndTaintUpdateCommands("node-1", "default", null, null, null, null)); + } + + @Test + public void testHasSchedulingTaints() { + Assert.assertFalse(KubernetesClusterNodeGroupUtil.hasSchedulingTaints(null)); + Assert.assertFalse(KubernetesClusterNodeGroupUtil.hasSchedulingTaints(List.of( + new KubernetesClusterNodeGroupUtil.Taint("dedicated", "system", "PreferNoSchedule")))); + Assert.assertTrue(KubernetesClusterNodeGroupUtil.hasSchedulingTaints(List.of( + new KubernetesClusterNodeGroupUtil.Taint("dedicated", "system", "PreferNoSchedule"), + new KubernetesClusterNodeGroupUtil.Taint("gpu", null, "NoExecute")))); + } + @Test + public void testMergeLabelsNodeOverridesNodeGroup() { + Assert.assertEquals("pool=gpu,disktype=nvme,zone=a", + KubernetesClusterNodeGroupUtil.mergeLabels("pool=gpu,disktype=ssd", "disktype=nvme,zone=a")); + Assert.assertEquals("pool=gpu", KubernetesClusterNodeGroupUtil.mergeLabels("pool=gpu", null)); + Assert.assertNull(KubernetesClusterNodeGroupUtil.mergeLabels(null, null)); + } + + @Test + public void testMergeTaintsNodeOverridesSameKeyAndEffect() { + Assert.assertEquals("dedicated=gpu:NoExecute,dedicated=team-a:NoSchedule,maintenance:NoSchedule", + KubernetesClusterNodeGroupUtil.mergeTaints("dedicated=gpu:NoSchedule,dedicated=gpu:NoExecute", + "dedicated=team-a:NoSchedule,maintenance:NoSchedule")); + Assert.assertEquals("gpu=true:NoSchedule", KubernetesClusterNodeGroupUtil.mergeTaints(null, "gpu=true:NoSchedule")); + } +} diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 0acafc3b8015..1597749fe84e 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1,4 +1,22 @@ { +"label.kubernetes.node.group": "Node group", +"label.kubernetes.node.group.create": "Create node group", +"label.kubernetes.node.group.delete": "Delete node group", +"label.kubernetes.node.group.scale": "Scale node group", +"label.kubernetes.node.group.update": "Edit node group", +"label.kubernetes.node.groups": "Node groups", +"label.kubernetes.node.update": "Edit node labels and taints", +"label.kubernetes.node.inherited": "From the node group", +"label.cks.cluster.default.node.group.autoscalingenabled": "Autoscaling of the {group} node group", +"label.cks.cluster.default.node.group.maxsize": "Maximum size of the {group} node group", +"label.cks.cluster.default.node.group.minsize": "Minimum size of the {group} node group", +"label.cks.cluster.default.node.group.size": "Size of the {group} node group", +"label.kubernetes.node.group.autoscaling": "Autoscaling", +"label.workernodes": "Worker nodes", +"label.kubernetes.node.label.add": "Add label", +"label.kubernetes.node.labels": "Labels", +"label.kubernetes.node.taint.add": "Add taint", +"label.kubernetes.node.taints": "Taints", "message.delete.account.not.disabled": "Please disable the account before attempting to delete it.", "alert.service.domainrouter": "Domain Router", "error.dedicate.bgp.peer.failed":"Failed to dedicate BGP peer", @@ -1762,6 +1780,30 @@ "label.migrate.systemvm.to": "Migrate System VM to", "label.migrate.volume": "Migrate Volume", "label.migrate.volume.to.kms": "Migrate Volume Encryption to KMS", +"message.kubernetes.node.group.delete": "Please confirm that you want to delete this node group. All its nodes will be removed from the cluster.", +"message.kubernetes.node.inherited": "Labels and taints of the node group, applied to all its nodes. The labels and taints of the node are added to them.", +"message.kubernetes.node.labels": "Kubernetes labels of this node, in addition to the labels of its node group. A label with the same key overrides the label of the node group.", +"message.kubernetes.node.taints": "Kubernetes taints of this node, in addition to the taints of its node group. A taint with the same key and effect overrides the taint of the node group.", +"message.kubernetes.node.default.taints": "Kubernetes taints of this node, in addition to the taints of its node group. The nodes of the default node group have to accept the system pods, so only the PreferNoSchedule effect is allowed.", +"message.kubernetes.cluster.default.node.group.autoscaling": "Let the cluster autoscaler manage the size of the default node group within its minimum and maximum size.", +"message.kubernetes.cluster.default.node.group.maxsize": "Maximum number of nodes of the default node group", +"message.kubernetes.cluster.default.node.group.minsize": "Minimum number of nodes of the default node group", +"message.kubernetes.cluster.default.node.group.size": "Number of nodes of the default node group, including the external nodes", +"message.kubernetes.node.group.autoscaling": "Let the cluster autoscaler manage the size of the node group within its minimum and maximum size. Node groups other than the default one require a cluster autoscaler image supporting node groups.", +"message.kubernetes.node.group.autoscaling.range": "Maximum size must be at least 1 and not lower than the minimum size.", +"message.kubernetes.node.group.autoscaling.required": "Minimum and maximum size are required to enable autoscaling.", +"message.kubernetes.node.group.autoscaling.size": "Node group size must be within its minimum and maximum size.", +"message.kubernetes.node.group.default.taints": "Kubernetes taints applied to the nodes of the default node group. The default node group has to accept the system pods, so only the PreferNoSchedule effect is allowed.", +"message.kubernetes.node.group.maxsize": "Maximum number of nodes the cluster autoscaler can scale the node group to.", +"message.kubernetes.node.group.minsize": "Minimum number of nodes kept by the cluster autoscaler.", +"message.kubernetes.node.group.key.required": "Key is required for every label and taint.", +"message.kubernetes.node.group.labels": "Kubernetes labels applied to the nodes of the node group.", +"message.kubernetes.node.group.name": "Name of the node group, unique in the cluster: up to 20 lower case alphanumeric characters or '-', starting and ending with an alphanumeric character.", +"message.kubernetes.node.group.nothing.to.change": "Nothing to change.", +"message.kubernetes.node.group.serviceoffering": "Compute offering for the nodes of the node group.", +"message.kubernetes.node.group.size": "Number of nodes in the node group.", +"message.kubernetes.node.group.taints": "Kubernetes taints applied to the nodes of the node group. Effect is one of NoSchedule, PreferNoSchedule, NoExecute.", +"message.kubernetes.node.group.template": "CKS template for the nodes of the node group. Defaults to the template of the worker nodes of the cluster.", "message.memory.usage.info.hypervisor.additionals": "The data shown may not reflect the actual memory usage if the Instance does not have the additional hypervisor tools installed", "message.memory.usage.info.negative.value": "If the Instance's memory usage cannot be obtained from the hypervisor, the lines for free memory in the raw data graph and memory usage in the percentage graph will be disabled", "message.migrate.volume.tooltip": "Volume can be migrated to any suitable storage pool. Admin has to choose the appropriate disk offering to replace, that supports the new storage pool", @@ -3170,6 +3212,11 @@ "message.create.instance.from.backup.different.zone": "Creating Instance from Backup on a different Zone. Please ensure that the backup repository is accessible in the selected Zone.", "message.csv.empty": "Empty CSV File", "message.csv.missing.headers": "Columns are missing from headers in CSV", +"message.success.create.kubernetes.node.group": "Successfully created node group", +"message.success.delete.kubernetes.node.group": "Successfully deleted node group", +"message.success.scale.kubernetes.node.group": "Successfully scaled node group", +"message.success.update.kubernetes.node.group": "Successfully updated node group", +"message.success.update.kubernetes.node": "Successfully updated node", "message.template.ostype.different.from.backup": "Selected Template has a different OS type than the Backup. Please proceed with caution.", "message.iso.ostype.different.from.backup": "Selected ISO has a different OS type than the Backup. Please proceed with caution.", "message.action.delete.asnrange": "Please confirm the AS range that you want to delete", diff --git a/ui/public/locales/ru_RU.json b/ui/public/locales/ru_RU.json index 4fb3788bc511..f8434289ab80 100644 --- a/ui/public/locales/ru_RU.json +++ b/ui/public/locales/ru_RU.json @@ -628,6 +628,24 @@ "label.keyboard": "Keyboard language", "label.keyboardtype": "\u0422\u0438\u043f \u043a\u043b\u0430\u0432\u0438\u0430\u0442\u0443\u0440\u044b", "label.keypair": "SSH Key Pair", +"label.kubernetes.node.group": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043d\u043e\u0434", +"label.kubernetes.node.group.create": "\u0421\u043e\u0437\u0434\u0430\u0442\u044c \u0433\u0440\u0443\u043f\u043f\u0443 \u043d\u043e\u0434", +"label.kubernetes.node.group.delete": "\u0423\u0434\u0430\u043b\u0438\u0442\u044c \u0433\u0440\u0443\u043f\u043f\u0443 \u043d\u043e\u0434", +"label.kubernetes.node.group.scale": "\u041c\u0430\u0441\u0448\u0442\u0430\u0431\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0433\u0440\u0443\u043f\u043f\u0443 \u043d\u043e\u0434", +"label.kubernetes.node.group.update": "\u0418\u0437\u043c\u0435\u043d\u0438\u0442\u044c \u0433\u0440\u0443\u043f\u043f\u0443 \u043d\u043e\u0434", +"label.kubernetes.node.groups": "\u0413\u0440\u0443\u043f\u043f\u044b \u043d\u043e\u0434", +"label.kubernetes.node.update": "\u0418\u0437\u043c\u0435\u043d\u0438\u0442\u044c \u043c\u0435\u0442\u043a\u0438 \u0438 taint'\u044b \u043d\u043e\u0434\u044b", +"label.kubernetes.node.inherited": "\u041e\u0442 \u0433\u0440\u0443\u043f\u043f\u044b \u043d\u043e\u0434", +"label.cks.cluster.default.node.group.autoscalingenabled": "\u0410\u0432\u0442\u043e\u0441\u043a\u0435\u0439\u043b\u0438\u043d\u0433 \u0433\u0440\u0443\u043f\u043f\u044b {group}", +"label.cks.cluster.default.node.group.maxsize": "\u041c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0439 \u0440\u0430\u0437\u043c\u0435\u0440 \u0433\u0440\u0443\u043f\u043f\u044b {group}", +"label.cks.cluster.default.node.group.minsize": "\u041c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0439 \u0440\u0430\u0437\u043c\u0435\u0440 \u0433\u0440\u0443\u043f\u043f\u044b {group}", +"label.cks.cluster.default.node.group.size": "\u0420\u0430\u0437\u043c\u0435\u0440 \u0433\u0440\u0443\u043f\u043f\u044b {group}", +"label.kubernetes.node.group.autoscaling": "\u0410\u0432\u0442\u043e\u0441\u043a\u0435\u0439\u043b\u0438\u043d\u0433", +"label.workernodes": "\u0420\u0430\u0431\u043e\u0447\u0438\u0435 \u043d\u043e\u0434\u044b", +"label.kubernetes.node.label.add": "\u0414\u043e\u0431\u0430\u0432\u0438\u0442\u044c \u043c\u0435\u0442\u043a\u0443", +"label.kubernetes.node.labels": "\u041c\u0435\u0442\u043a\u0438", +"label.kubernetes.node.taint.add": "\u0414\u043e\u0431\u0430\u0432\u0438\u0442\u044c taint", +"label.kubernetes.node.taints": "Taints", "label.kvmnetworklabel": "\u041c\u0435\u0442\u043a\u0430 \u0442\u0440\u0430\u0444\u0438\u043a\u0430 KVM", "label.l2gatewayserviceuuid": "L2 Gateway Service Uuid", "label.l3gatewayserviceuuid": "L3 Gateway Service Uuid", @@ -1412,6 +1430,30 @@ "message.installwizard.tooltip.configureguesttraffic.guestnetmask": "\u0421\u0435\u0442\u0435\u0432\u0430\u044f \u043c\u0430\u0441\u043a\u0430 \u043f\u043e\u0434\u0441\u0435\u0442\u0438 \u0434\u043b\u044f \u0433\u043e\u0441\u0442\u0435\u0439.", "message.installwizard.tooltip.configureguesttraffic.gueststartip": "\u0414\u0438\u0430\u043f\u0430\u0437\u043e\u043d IP-\u0430\u0434\u0440\u0435\u0441\u043e\u0432, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0431\u0443\u0434\u0443\u0442 \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u044b \u0434\u043b\u044f \u0433\u043e\u0441\u0442\u0435\u0439 \u044d\u0442\u043e\u0439 \u0437\u043e\u043d\u044b. \u041f\u0440\u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0438 \u043e\u0434\u043d\u043e\u0433\u043e \u0441\u0435\u0442\u0435\u0432\u043e\u0433\u043e \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u0430 (NIC) \u044d\u0442\u0438 \u0430\u0434\u0440\u0435\u0441\u0430 \u0434\u043e\u043b\u0436\u043d\u044b \u0431\u044b\u0442\u044c \u0432 \u043f\u043e\u0434\u0441\u0435\u0442\u0438 (CIDR) \u0441\u0442\u0435\u043d\u0434\u0430.", "message.iso.desc": "\u041e\u0431\u0440\u0430\u0437 \u0434\u0438\u0441\u043a\u0430, \u0441\u043e\u0434\u0435\u0440\u0436\u0430\u0449\u0438\u0439 \u0434\u0430\u043d\u043d\u044b\u0435 \u0438\u043b\u0438 \u0437\u0430\u0433\u0440\u0443\u0437\u043e\u0447\u043d\u044b\u0439 \u043d\u043e\u0441\u0438\u0442\u0435\u043b\u044c \u0434\u043b\u044f \u041e\u0421", +"message.kubernetes.node.group.delete": "\u041f\u043e\u0434\u0442\u0432\u0435\u0440\u0434\u0438\u0442\u0435 \u0443\u0434\u0430\u043b\u0435\u043d\u0438\u0435 \u0433\u0440\u0443\u043f\u043f\u044b \u043d\u043e\u0434. \u0412\u0441\u0435 \u0435\u0451 \u043d\u043e\u0434\u044b \u0431\u0443\u0434\u0443\u0442 \u0443\u0434\u0430\u043b\u0435\u043d\u044b \u0438\u0437 \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430.", +"message.kubernetes.node.inherited": "\u041c\u0435\u0442\u043a\u0438 \u0438 taint'\u044b \u0433\u0440\u0443\u043f\u043f\u044b \u043d\u043e\u0434, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u043f\u0440\u0438\u043c\u0435\u043d\u044f\u044e\u0442\u0441\u044f \u043a\u043e \u0432\u0441\u0435\u043c \u0435\u0451 \u043d\u043e\u0434\u0430\u043c. \u041c\u0435\u0442\u043a\u0438 \u0438 taint'\u044b \u043d\u043e\u0434\u044b \u0434\u043e\u0431\u0430\u0432\u043b\u044f\u044e\u0442\u0441\u044f \u043a \u043d\u0438\u043c.", +"message.kubernetes.node.labels": "\u041c\u0435\u0442\u043a\u0438 Kubernetes \u044d\u0442\u043e\u0439 \u043d\u043e\u0434\u044b \u0432 \u0434\u043e\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u0435 \u043a \u043c\u0435\u0442\u043a\u0430\u043c \u0435\u0451 \u0433\u0440\u0443\u043f\u043f\u044b. \u041c\u0435\u0442\u043a\u0430 \u0441 \u0442\u0435\u043c \u0436\u0435 \u043a\u043b\u044e\u0447\u043e\u043c \u0437\u0430\u043c\u0435\u043d\u044f\u0435\u0442 \u043c\u0435\u0442\u043a\u0443 \u0433\u0440\u0443\u043f\u043f\u044b.", +"message.kubernetes.node.taints": "Taint'\u044b Kubernetes \u044d\u0442\u043e\u0439 \u043d\u043e\u0434\u044b \u0432 \u0434\u043e\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u0435 \u043a taint'\u0430\u043c \u0435\u0451 \u0433\u0440\u0443\u043f\u043f\u044b. Taint \u0441 \u0442\u0435\u043c \u0436\u0435 \u043a\u043b\u044e\u0447\u043e\u043c \u0438 \u044d\u0444\u0444\u0435\u043a\u0442\u043e\u043c \u0437\u0430\u043c\u0435\u043d\u044f\u0435\u0442 taint \u0433\u0440\u0443\u043f\u043f\u044b.", +"message.kubernetes.node.default.taints": "Taint'\u044b Kubernetes \u044d\u0442\u043e\u0439 \u043d\u043e\u0434\u044b \u0432 \u0434\u043e\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u0435 \u043a taint'\u0430\u043c \u0435\u0451 \u0433\u0440\u0443\u043f\u043f\u044b. \u041d\u0430 \u043d\u043e\u0434\u0430\u0445 \u0433\u0440\u0443\u043f\u043f\u044b \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e \u0434\u043e\u043b\u0436\u043d\u044b \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0442\u044c\u0441\u044f \u0441\u0438\u0441\u0442\u0435\u043c\u043d\u044b\u0435 \u043f\u043e\u0434\u044b, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u0434\u043e\u043f\u0443\u0441\u0442\u0438\u043c \u0442\u043e\u043b\u044c\u043a\u043e \u044d\u0444\u0444\u0435\u043a\u0442 PreferNoSchedule.", +"message.kubernetes.cluster.default.node.group.autoscaling": "\u0420\u0430\u0437\u043c\u0435\u0440 \u0433\u0440\u0443\u043f\u043f\u044b default \u0443\u043f\u0440\u0430\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0430\u0432\u0442\u043e\u0441\u043a\u0435\u0439\u043b\u0435\u0440\u043e\u043c \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430 \u0432 \u043f\u0440\u0435\u0434\u0435\u043b\u0430\u0445 \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u0438 \u043c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u0440\u0430\u0437\u043c\u0435\u0440\u0430.", +"message.kubernetes.cluster.default.node.group.maxsize": "\u041c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0435 \u0447\u0438\u0441\u043b\u043e \u043d\u043e\u0434 \u0433\u0440\u0443\u043f\u043f\u044b default", +"message.kubernetes.cluster.default.node.group.minsize": "\u041c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0435 \u0447\u0438\u0441\u043b\u043e \u043d\u043e\u0434 \u0433\u0440\u0443\u043f\u043f\u044b default", +"message.kubernetes.cluster.default.node.group.size": "\u0427\u0438\u0441\u043b\u043e \u043d\u043e\u0434 \u0433\u0440\u0443\u043f\u043f\u044b default \u0432\u043c\u0435\u0441\u0442\u0435 \u0441 \u0432\u043d\u0435\u0448\u043d\u0438\u043c\u0438 \u043d\u043e\u0434\u0430\u043c\u0438", +"message.kubernetes.node.group.autoscaling": "\u0420\u0430\u0437\u043c\u0435\u0440 \u0433\u0440\u0443\u043f\u043f\u044b \u043d\u043e\u0434 \u0443\u043f\u0440\u0430\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0430\u0432\u0442\u043e\u0441\u043a\u0435\u0439\u043b\u0435\u0440\u043e\u043c \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430 \u0432 \u043f\u0440\u0435\u0434\u0435\u043b\u0430\u0445 \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u0438 \u043c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u0440\u0430\u0437\u043c\u0435\u0440\u0430. \u0414\u043b\u044f \u0433\u0440\u0443\u043f\u043f, \u043a\u0440\u043e\u043c\u0435 \u0433\u0440\u0443\u043f\u043f\u044b \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e, \u043d\u0443\u0436\u0435\u043d \u043e\u0431\u0440\u0430\u0437 \u0430\u0432\u0442\u043e\u0441\u043a\u0435\u0439\u043b\u0435\u0440\u0430 \u0441 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u043a\u043e\u0439 \u0433\u0440\u0443\u043f\u043f \u043d\u043e\u0434.", +"message.kubernetes.node.group.autoscaling.range": "\u041c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0439 \u0440\u0430\u0437\u043c\u0435\u0440 \u0434\u043e\u043b\u0436\u0435\u043d \u0431\u044b\u0442\u044c \u043d\u0435 \u043c\u0435\u043d\u044c\u0448\u0435 1 \u0438 \u043d\u0435 \u043c\u0435\u043d\u044c\u0448\u0435 \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0433\u043e.", +"message.kubernetes.node.group.autoscaling.required": "\u0414\u043b\u044f \u0430\u0432\u0442\u043e\u0441\u043a\u0435\u0439\u043b\u0438\u043d\u0433\u0430 \u043d\u0443\u0436\u043d\u043e \u0443\u043a\u0430\u0437\u0430\u0442\u044c \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0439 \u0438 \u043c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0439 \u0440\u0430\u0437\u043c\u0435\u0440.", +"message.kubernetes.node.group.autoscaling.size": "\u0420\u0430\u0437\u043c\u0435\u0440 \u0433\u0440\u0443\u043f\u043f\u044b \u043d\u043e\u0434 \u0434\u043e\u043b\u0436\u0435\u043d \u0431\u044b\u0442\u044c \u0432 \u043f\u0440\u0435\u0434\u0435\u043b\u0430\u0445 \u0435\u0451 \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u0438 \u043c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u0440\u0430\u0437\u043c\u0435\u0440\u0430.", +"message.kubernetes.node.group.default.taints": "Taint'\u044b Kubernetes \u0434\u043b\u044f \u043d\u043e\u0434 \u0433\u0440\u0443\u043f\u043f\u044b \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e. \u041d\u0430 \u043d\u0435\u0439 \u0434\u043e\u043b\u0436\u043d\u044b \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0442\u044c\u0441\u044f \u0441\u0438\u0441\u0442\u0435\u043c\u043d\u044b\u0435 \u043f\u043e\u0434\u044b, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u0434\u043e\u043f\u0443\u0441\u0442\u0438\u043c \u0442\u043e\u043b\u044c\u043a\u043e \u044d\u0444\u0444\u0435\u043a\u0442 PreferNoSchedule.", +"message.kubernetes.node.group.maxsize": "\u041c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0435 \u0447\u0438\u0441\u043b\u043e \u043d\u043e\u0434, \u0434\u043e \u043a\u043e\u0442\u043e\u0440\u043e\u0433\u043e \u0430\u0432\u0442\u043e\u0441\u043a\u0435\u0439\u043b\u0435\u0440 \u043c\u043e\u0436\u0435\u0442 \u0443\u0432\u0435\u043b\u0438\u0447\u0438\u0442\u044c \u0433\u0440\u0443\u043f\u043f\u0443.", +"message.kubernetes.node.group.minsize": "\u041c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e\u0435 \u0447\u0438\u0441\u043b\u043e \u043d\u043e\u0434, \u043a\u043e\u0442\u043e\u0440\u043e\u0435 \u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u0442 \u0430\u0432\u0442\u043e\u0441\u043a\u0435\u0439\u043b\u0435\u0440.", +"message.kubernetes.node.group.key.required": "\u0414\u043b\u044f \u043a\u0430\u0436\u0434\u043e\u0439 \u043c\u0435\u0442\u043a\u0438 \u0438 taint \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u043e \u0443\u043a\u0430\u0437\u0430\u0442\u044c \u043a\u043b\u044e\u0447.", +"message.kubernetes.node.group.labels": "Kubernetes-\u043c\u0435\u0442\u043a\u0438, \u043d\u0430\u0437\u043d\u0430\u0447\u0430\u0435\u043c\u044b\u0435 \u043d\u043e\u0434\u0430\u043c \u0433\u0440\u0443\u043f\u043f\u044b.", +"message.kubernetes.node.group.name": "\u0418\u043c\u044f \u0433\u0440\u0443\u043f\u043f\u044b \u043d\u043e\u0434, \u0443\u043d\u0438\u043a\u0430\u043b\u044c\u043d\u043e\u0435 \u0432 \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0435: \u0434\u043e 20 \u0441\u0442\u0440\u043e\u0447\u043d\u044b\u0445 \u043b\u0430\u0442\u0438\u043d\u0441\u043a\u0438\u0445 \u0431\u0443\u043a\u0432, \u0446\u0438\u0444\u0440 \u0438\u043b\u0438 '-', \u043d\u0430\u0447\u0438\u043d\u0430\u0435\u0442\u0441\u044f \u0438 \u0437\u0430\u043a\u0430\u043d\u0447\u0438\u0432\u0430\u0435\u0442\u0441\u044f \u0431\u0443\u043a\u0432\u043e\u0439 \u0438\u043b\u0438 \u0446\u0438\u0444\u0440\u043e\u0439.", +"message.kubernetes.node.group.nothing.to.change": "\u041d\u0435\u0442 \u0438\u0437\u043c\u0435\u043d\u0435\u043d\u0438\u0439.", +"message.kubernetes.node.group.serviceoffering": "\u0412\u044b\u0447\u0438\u0441\u043b\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0434\u043b\u044f \u043d\u043e\u0434 \u0433\u0440\u0443\u043f\u043f\u044b.", +"message.kubernetes.node.group.size": "\u041a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e \u043d\u043e\u0434 \u0432 \u0433\u0440\u0443\u043f\u043f\u0435.", +"message.kubernetes.node.group.taints": "Kubernetes taints, \u043d\u0430\u0437\u043d\u0430\u0447\u0430\u0435\u043c\u044b\u0435 \u043d\u043e\u0434\u0430\u043c \u0433\u0440\u0443\u043f\u043f\u044b. Effect: NoSchedule, PreferNoSchedule \u0438\u043b\u0438 NoExecute.", +"message.kubernetes.node.group.template": "CKS-\u0448\u0430\u0431\u043b\u043e\u043d \u0434\u043b\u044f \u043d\u043e\u0434 \u0433\u0440\u0443\u043f\u043f\u044b. \u041f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442\u0441\u044f \u0448\u0430\u0431\u043b\u043e\u043d worker-\u043d\u043e\u0434 \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430.", "message.launch.zone": "\u0417\u043e\u043d\u0430 \u0433\u043e\u0442\u043e\u0432\u0430 \u043a \u0437\u0430\u043f\u0443\u0441\u043a\u0443, \u043f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430, \u043f\u0435\u0440\u0435\u0439\u0434\u0438\u0442\u0435 \u043a \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u043c\u0443 \u0448\u0430\u0433\u0443.", "message.lock.account": "\u041f\u043e\u0434\u0442\u0432\u0435\u0440\u0434\u0438\u0442\u0435, \u0447\u0442\u043e \u0432\u044b \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u0445\u043e\u0442\u0438\u0442\u0435 \u0437\u0430\u0431\u043b\u043e\u043a\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u044d\u0442\u0443 \u0443\u0447\u0451\u0442\u043d\u0443\u044e \u0437\u0430\u043f\u0438\u0441\u044c. \u0412\u0441\u0435 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u0438 \u0432 \u0442\u0430\u043a\u0438\u0445 \u0443\u0447\u0451\u0442\u043d\u044b\u0445 \u0437\u0430\u043f\u0438\u0441\u044f\u0445 \u043f\u043e\u0442\u0435\u0440\u044f\u044e\u0442 \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u044c \u0443\u043f\u0440\u0430\u0432\u043b\u044f\u0442\u044c \u0441\u0432\u043e\u0438\u043c\u0438 \u043e\u0431\u043b\u0430\u0447\u043d\u044b\u043c\u0438 \u0440\u0435\u0441\u0443\u0440\u0441\u0430\u043c\u0438. \u042d\u0442\u0438 \u0440\u0435\u0441\u0443\u0440\u0441\u044b \u043e\u0441\u0442\u0430\u043d\u0443\u0442\u0441\u044f \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u044b \u0434\u043b\u044f \u0434\u0440\u0443\u0433\u0438\u0445 \u0443\u0447\u0451\u0442\u043d\u044b\u0445 \u0437\u0430\u043f\u0438\u0441\u0435\u0439.", "message.migrate.instance.to.host": "\u041f\u043e\u0434\u0442\u0432\u0435\u0440\u0434\u0438\u0442\u0435, \u0447\u0442\u043e \u0432\u044b \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u0445\u043e\u0442\u0438\u0442\u0435 \u043f\u0435\u0440\u0435\u043d\u0435\u0441\u0442\u0438 \u043c\u0430\u0448\u0438\u043d\u0443 \u043d\u0430 \u0434\u0440\u0443\u0433\u043e\u0439 \u0443\u0437\u0435\u043b.", @@ -1447,6 +1489,11 @@ "message.step.2.continue": "\u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430 \u0432\u044b\u0431\u0435\u0440\u0438\u0442\u0435 \u0443\u0441\u043b\u0443\u0433\u0443 \u0434\u043b\u044f \u043f\u0440\u043e\u0434\u043e\u043b\u0436\u0435\u043d\u0438\u044f", "message.step.3.continue": "\u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430 \u0432\u044b\u0431\u0435\u0440\u0438\u0442\u0435 \u0440\u0430\u0437\u043c\u0435\u0440 \u0434\u0438\u0441\u043a\u0430 \u0434\u043b\u044f \u043f\u0440\u043e\u0434\u043e\u043b\u0436\u0435\u043d\u0438\u044f", "message.step.4.continue": "\u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430 \u0432\u044b\u0431\u0435\u0440\u0438\u0442\u0435 \u043a\u0430\u043a \u043c\u0438\u043d\u0438\u043c\u0443\u043c \u043e\u0434\u043d\u0443 \u0441\u0435\u0442\u044c \u0434\u043b\u044f \u043f\u0440\u043e\u0434\u043e\u043b\u0436\u0435\u043d\u0438\u044f.", +"message.success.create.kubernetes.node.group": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043d\u043e\u0434 \u0443\u0441\u043f\u0435\u0448\u043d\u043e \u0441\u043e\u0437\u0434\u0430\u043d\u0430", +"message.success.delete.kubernetes.node.group": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043d\u043e\u0434 \u0443\u0441\u043f\u0435\u0448\u043d\u043e \u0443\u0434\u0430\u043b\u0435\u043d\u0430", +"message.success.scale.kubernetes.node.group": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043d\u043e\u0434 \u0443\u0441\u043f\u0435\u0448\u043d\u043e \u043c\u0430\u0441\u0448\u0442\u0430\u0431\u0438\u0440\u043e\u0432\u0430\u043d\u0430", +"message.success.update.kubernetes.node.group": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043d\u043e\u0434 \u0443\u0441\u043f\u0435\u0448\u043d\u043e \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0430", +"message.success.update.kubernetes.node": "\u041d\u043e\u0434\u0430 \u0443\u0441\u043f\u0435\u0448\u043d\u043e \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0430", "message.suspend.project": "\u0412\u044b \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u0445\u043e\u0442\u0438\u0442\u0435 \u043f\u0440\u0438\u043e\u0441\u0442\u0430\u043d\u043e\u0432\u0438\u0442\u044c \u043f\u0440\u043e\u0435\u043a\u0442?", "message.template.desc": "\u041e\u0431\u0440\u0430\u0437 \u041e\u0421, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u043c\u043e\u0436\u043d\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u0432 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0437\u0430\u0433\u0440\u0443\u0436\u0430\u0435\u043c\u043e\u0439 \u0432 \u0412\u041c", "message.template.iso": "\u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430 \u0432\u044b\u0431\u0435\u0440\u0438\u0442\u0435 \u0448\u0430\u0431\u043b\u043e\u043d \u0438\u043b\u0438 ISO \u0434\u043b\u044f \u043f\u0440\u043e\u0434\u043e\u043b\u0436\u0435\u043d\u0438\u044f.", diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 7c2c9f09eae4..9de15c689c27 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -579,7 +579,8 @@ export default { searchFilters: ['name', 'domainid', 'account', 'state'], permission: ['listKubernetesClusters'], columns: (store) => { - var fields = ['name', 'state', 'clustertype', 'size', 'cpunumber', 'memory', 'kubernetesversionname'] + // The number of worker nodes of all the node groups, the size of the cluster is the size of its default node group + var fields = ['name', 'state', 'clustertype', 'workernodes', 'cpunumber', 'memory', 'kubernetesversionname'] if (['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { fields.push('account') } @@ -596,7 +597,7 @@ export default { const filters = ['cloud.managed', 'external.managed'] return filters }, - details: ['name', 'description', 'zonename', 'kubernetesversionname', 'autoscalingenabled', 'csienabled', 'minsize', 'maxsize', 'size', 'controlnodes', 'controlaffinitygroupnames', 'etcdnodes', 'etcdaffinitygroupnames', 'workeraffinitygroupnames', 'cpunumber', 'memory', 'keypair', 'cniconfigname', 'associatednetworkname', 'account', 'domain', 'zonename', 'clustertype', 'created'], + details: ['name', 'description', 'zonename', 'kubernetesversionname', 'autoscalingenabled', 'csienabled', 'minsize', 'maxsize', 'size', 'workernodes', 'controlnodes', 'controlaffinitygroupnames', 'etcdnodes', 'etcdaffinitygroupnames', 'workeraffinitygroupnames', 'cpunumber', 'memory', 'keypair', 'cniconfigname', 'associatednetworkname', 'account', 'domain', 'zonename', 'clustertype', 'created'], tabs: [ { name: 'k8s', diff --git a/ui/src/views/compute/KubernetesDefaultNodeGroupLabel.vue b/ui/src/views/compute/KubernetesDefaultNodeGroupLabel.vue new file mode 100644 index 000000000000..653974831264 --- /dev/null +++ b/ui/src/views/compute/KubernetesDefaultNodeGroupLabel.vue @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/compute/KubernetesNodeGroupForm.vue b/ui/src/views/compute/KubernetesNodeGroupForm.vue new file mode 100644 index 000000000000..91d8791047a7 --- /dev/null +++ b/ui/src/views/compute/KubernetesNodeGroupForm.vue @@ -0,0 +1,546 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/compute/KubernetesNodeGroupsTab.vue b/ui/src/views/compute/KubernetesNodeGroupsTab.vue new file mode 100644 index 000000000000..7dc25ccaf20a --- /dev/null +++ b/ui/src/views/compute/KubernetesNodeGroupsTab.vue @@ -0,0 +1,350 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/compute/KubernetesNodeLabelsForm.vue b/ui/src/views/compute/KubernetesNodeLabelsForm.vue new file mode 100644 index 000000000000..85db115b966d --- /dev/null +++ b/ui/src/views/compute/KubernetesNodeLabelsForm.vue @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/compute/KubernetesServiceTab.vue b/ui/src/views/compute/KubernetesServiceTab.vue index 3c45feea063f..b8760cfb7225 100644 --- a/ui/src/views/compute/KubernetesServiceTab.vue +++ b/ui/src/views/compute/KubernetesServiceTab.vue @@ -157,6 +157,9 @@ +