cks node groups

PendingReleaseNotes

@@ -91,3 +91,16 @@ example.ver.1 > example.ver.2:
     no device-mapper multipath configuration is required. A new
     Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase /
     NVMeTCPAdapter on the KVM side back the new pool type.
+
+  * CKS: CloudManaged Kubernetes clusters now have node groups, pools of
+    worker nodes with their own service offering, template, Kubernetes
+    labels and taints and autoscaling settings, managed with the new
+    create, list, update, scale and deleteKubernetesClusterNodeGroup APIs.
+    Every cluster has a default node group holding the worker nodes of the
+    cluster size, created on upgrade for the existing clusters; the
+    scaleKubernetesCluster API applies to it. Nodes can have their own
+    labels and taints on top of the ones of their node group through the
+    new updateKubernetesClusterNode API. Autoscaling node groups other than
+    the default one requires a cluster autoscaler image supporting node
+    groups, set by the new cloud.kubernetes.cluster.autoscaler.image global
+    setting.

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -1330,6 +1330,14 @@ public class ApiConstants {
     public static final String WORKER_AFFINITY_GROUP_NAMES = "workeraffinitygroupnames";
     public static final String ETCD_AFFINITY_GROUP_IDS = "etcdaffinitygroupids";
     public static final String ETCD_AFFINITY_GROUP_NAMES = "etcdaffinitygroupnames";
+    public static final String WORKER_NODES = "workernodes";
+    public static final String NODE_GROUP_ID = "nodegroupid";
+    public static final String NODE_GROUP_NAME = "nodegroupname";
+    public static final String NODE_GROUPS = "nodegroups";
+    public static final String NODE_LABELS = "nodelabels";
+    public static final String NODE_TAINTS = "nodetaints";
+    public static final String CLEANUP_NODE_LABELS = "cleanupnodelabels";
+    public static final String CLEANUP_NODE_TAINTS = "cleanupnodetaints";
 
     public static final String BOOT_TYPE = "boottype";
     public static final String BOOT_MODE = "bootmode";

api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java

@@ -38,6 +38,13 @@ public class KubernetesUserVmResponse extends UserVmResponse {
     @Param(description = "Kubernetes version of the node")
     private String nodeVersion;
 
+    @SerializedName(ApiConstants.NODE_GROUP_ID)
+    @Param(description = "The ID of the node group of the node, if any", since = "24.0")
+    private String nodeGroupId;
+
+    @SerializedName(ApiConstants.NODE_GROUP_NAME)
+    @Param(description = "The name of the node group of the node, if any", since = "24.0")
+    private String nodeGroupName;
 
     public void setExternalNode(boolean externalNode) {
         isExternalNode = externalNode;
@@ -48,4 +55,12 @@ public void setEtcdNode(boolean etcdNode) {
     }
 
     public void setNodeVersion(String nodeVersion) { this.nodeVersion = nodeVersion;}
+
+    public void setNodeGroupId(String nodeGroupId) {
+        this.nodeGroupId = nodeGroupId;
+    }
+
+    public void setNodeGroupName(String nodeGroupName) {
+        this.nodeGroupName = nodeGroupName;
+    }
 }

engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql

@@ -239,3 +239,38 @@ CREATE TABLE IF NOT EXISTS `cloud`.`instance_boot_group_details` (
     PRIMARY KEY (`id`),
     CONSTRAINT `fk_instance_boot_group_details__group_id` FOREIGN KEY (`boot_group_id`) REFERENCES `instance_boot_group` (`id`) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- CKS node groups: pools of worker nodes with their own service offering, template, labels, taints and autoscaling settings.
+-- Every CloudManaged cluster has a default node group (is_default = 1) holding the worker nodes of the cluster size,
+-- the node groups of existing clusters are created by the management server on startup.
+CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_node_group` (
+    `id`                  bigint unsigned NOT NULL AUTO_INCREMENT,
+    `uuid`                varchar(40)     NOT NULL,
+    `cluster_id`          bigint unsigned NOT NULL COMMENT 'kubernetes cluster id',
+    `name`                varchar(255)    NOT NULL,
+    `is_default`          tinyint(1)      NOT NULL DEFAULT 0 COMMENT 'whether the node group is the default node group of the cluster',
+    `state`               varchar(32)     NOT NULL DEFAULT 'Ready' COMMENT 'state of the node group',
+    `service_offering_id` bigint unsigned NOT NULL COMMENT 'service offering of the node group nodes',
+    `template_id`         bigint unsigned NOT NULL COMMENT 'template of the node group nodes',
+    `node_count`          bigint unsigned NOT NULL DEFAULT 0 COMMENT 'number of nodes in the node group',
+    `labels`              varchar(4096)   DEFAULT NULL COMMENT 'comma separated key=value Kubernetes node labels',
+    `taints`              varchar(4096)   DEFAULT NULL COMMENT 'comma separated key[=value]:effect Kubernetes node taints',
+    `autoscaling_enabled` tinyint(1)      NOT NULL DEFAULT 0 COMMENT 'whether the cluster autoscaler manages the size of the node group',
+    `min_size`            bigint unsigned DEFAULT NULL COMMENT 'minimum size of the node group when autoscaling is enabled',
+    `max_size`            bigint unsigned DEFAULT NULL COMMENT 'maximum size of the node group when autoscaling is enabled',
+    `created`             datetime        NOT NULL,
+    `updated`             datetime        DEFAULT NULL COMMENT 'date of the last state change of the node group',
+    `removed`             datetime        DEFAULT NULL COMMENT 'date the node group was removed',
+    PRIMARY KEY (`id`),
+    CONSTRAINT `uc_kubernetes_cluster_node_group__uuid` UNIQUE (`uuid`),
+    CONSTRAINT `fk_kubernetes_cluster_node_group__cluster_id` FOREIGN KEY (`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE,
+    CONSTRAINT `fk_kubernetes_cluster_node_group__service_offering_id` FOREIGN KEY (`service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE,
+    CONSTRAINT `fk_kubernetes_cluster_node_group__template_id` FOREIGN KEY (`template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE,
+    INDEX `i_kubernetes_cluster_node_group__cluster_id`(`cluster_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map', 'node_group_id', 'bigint unsigned DEFAULT NULL COMMENT "node group of the worker node, NULL for the control, etcd and external nodes"');
+CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.kubernetes_cluster_vm_map', 'fk_kubernetes_cluster_vm_map__node_group_id', '(node_group_id)', '`kubernetes_cluster_node_group`(`id`)');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map', 'labels', 'varchar(4096) DEFAULT NULL COMMENT "comma separated key=value Kubernetes labels of the node, in addition to the labels of its node group"');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map', 'taints', 'varchar(4096) DEFAULT NULL COMMENT "comma separated key[=value]:effect Kubernetes taints of the node, in addition to the taints of its node group"');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'node_group_node_count', 'bigint unsigned NOT NULL DEFAULT 0 COMMENT "number of worker nodes in the node groups of the cluster other than the default one"');

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java

@@ -26,4 +26,9 @@ public class KubernetesClusterEventTypes {
     public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD";
     public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE";
     public static final String EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE = "KUBERNETES.CLUSTER.AFFINITY.UPDATE";
+    public static final String EVENT_KUBERNETES_CLUSTER_NODE_GROUP_CREATE = "KUBERNETES.CLUSTER.NODEGROUP.CREATE";
+    public static final String EVENT_KUBERNETES_CLUSTER_NODE_GROUP_UPDATE = "KUBERNETES.CLUSTER.NODEGROUP.UPDATE";
+    public static final String EVENT_KUBERNETES_CLUSTER_NODE_GROUP_SCALE = "KUBERNETES.CLUSTER.NODEGROUP.SCALE";
+    public static final String EVENT_KUBERNETES_CLUSTER_NODE_GROUP_DELETE = "KUBERNETES.CLUSTER.NODEGROUP.DELETE";
+    public static final String EVENT_KUBERNETES_CLUSTER_NODE_UPDATE = "KUBERNETES.CLUSTER.NODE.UPDATE";
 }