feat(eks_cluster): overhaul EKS managed node groups to be able to uti…

…lize latest node_groups changes in terraform-aws-eks module (#252)

README.md

@@ -205,6 +205,7 @@ The following sections provide a full list of configuration in- and output varia
 | min\_node\_count | The minimum number of worker nodes to use for the cluster | `number` | `3` | no |
 | node\_group\_ami | ami type for the node group worker intances | `string` | `"AL2_x86_64"` | no |
 | node\_group\_disk\_size | node group worker disk size | `string` | `"50"` | no |
+| node\_groups\_managed | Optionally set custom node groups to be created when using `enable_worker_group = false`, a default node group will be created if this input is not set  | `map` | `{}` | no |
 | node\_machine\_type | The instance type to use for the cluster's worker nodes | `string` | `"m5.large"` | no |
 | private\_subnets | The private subnet CIDR block to use in the created VPC | `list(string)` | <pre>[<br>  "10.0.4.0/24",<br>  "10.0.5.0/24",<br>  "10.0.6.0/24"<br>]</pre> | no |
 | production\_letsencrypt | Flag to use the production environment of letsencrypt in the `jx-requirements.yml` file | `bool` | `false` | no |
@@ -564,9 +565,73 @@ output "vault_user_secret" {
 }
 ```
 
-:warning: **Note**: EKS node groups are supported in kubernetes v1.14+ and platform version eks.3
+**Note**: EKS node groups now support using [spot instances](https://aws.amazon.com/blogs/containers/amazon-eks-now-supports-provisioning-and-managing-ec2-spot-instances-in-managed-node-groups/) and [launch templates](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html) (will be set accordingly with the use of the  `enable_spot_instances` variable)
+
+#### Custom EKS node groups
+A single node group will be created by default when using EKS node groups. Supply values for the `node_groups_managed` variable to override this behaviour:
+
+```terraform
+module "eks-jx" {
+  source  = "jenkins-x/eks-jx/aws"
+  enable_worker_group = false
+  node_groups_managed = {
+    node-group-name = {
+      ami_type                = "AL2_x86_64"
+      disk_size               = 50
+      desired_capacity        = 3
+      max_capacity            = 5
+      min_capacity            = 3
+      instance_types          = [ "m5.large" ]
+      launce_template_id      = null
+      launch_template_version = null
+      
+      k8s_labels = {
+        purpose = "application"
+      }
+    },
+    second-node-group-name = {
+      # ...
+    },
+    # ...
+  }
+}
+```
+
+One can use launch templates with node groups by specifying the template id and version in the parameters.
 
-:warning: **Note**: Spot instances are not supported for EKS node groups. Check this AWS [issue](https://github.com/aws/containers-roadmap/issues/583) for more details.
+```terraform
+resource "aws_launch_template" "foo" {
+  name = "foo"
+  # ...
+}
+
+module "eks-jx" {
+  source  = "jenkins-x/eks-jx/aws"
+  enable_worker_group = false
+  node_groups_managed = {
+    node-group-name = {
+      ami_type                = "AL2_x86_64"
+      disk_size               = 50
+      desired_capacity        = 3
+      max_capacity            = 5
+      min_capacity            = 3
+      instance_types          = [ "m5.large" ]
+      launce_template_id      = aws_launch_template.foo.id
+      launch_template_version = aws_launch_template.foo.latest_version
+      
+      k8s_labels = {
+        purpose = "application"
+      }
+    },
+    second-node-group-name = {
+      # ...
+    },
+    # ...
+  }
+}
+```
+
+:warning: **Note**: EKS node groups are supported in kubernetes v1.14+ and platform version eks.3
 
 ### AWS Auth
 

main.tf

@@ -45,6 +45,7 @@ module "cluster" {
   min_node_count                        = var.min_node_count
   max_node_count                        = var.max_node_count
   node_machine_type                     = var.node_machine_type
+  node_groups                           = var.node_groups_managed
   spot_price                            = var.spot_price
   encrypt_volume_self                   = var.encrypt_volume_self
   vpc_name                              = var.vpc_name

modules/cluster/local.tf

@@ -12,4 +12,46 @@ locals {
   jenkins-x-namespace    = "jx"
   cluster_trunc          = substr(var.cluster_name, 0, 35)
   cert-manager-namespace = "cert-manager"
+
+  node_group_defaults = {
+    launch_template_id      = null
+    launch_template_version = null
+
+    # Provider default which is 'ON_DEMAND'. We don't set it explicitly to avoid changes to existing clusters provisioned with this module
+    capacity_type = var.enable_spot_instances ? "SPOT" : null
+
+    k8s_labels = {
+      "jenkins-x.io/name"       = var.cluster_name
+      "jenkins-x.io/part-of"    = "jx-platform"
+      "jenkins-x.io/managed-by" = "terraform"
+    }
+
+    additional_tags = {
+      aws_managed = "true"
+    }
+  }
+
+  node_groups_extended = length(var.node_groups) > 0 ? { for k, v in var.node_groups : k => merge(
+    local.node_group_defaults,
+    v,
+    {
+      # Deep merge isn't a thing in terraform, yet, so we commit these atrocities.
+      k8s_labels = merge(
+        local.node_group_defaults["k8s_labels"],
+        v["k8s_labels"],
+      )
+    },
+    ) } : {
+    eks-jx-node-group = merge(
+      {
+        ami_type         = var.node_group_ami
+        disk_size        = var.node_group_disk_size
+        desired_capacity = var.desired_node_count
+        max_capacity     = var.max_node_count
+        min_capacity     = var.min_node_count
+        instance_types   = [var.node_machine_type]
+      },
+      local.node_group_defaults
+    )
+  }
 }

modules/cluster/main.tf

@@ -125,25 +125,7 @@ module "eks" {
     }
   ] : []
 
-  node_groups = !var.enable_worker_group ? {
-    eks-jx-node-group = {
-      ami_type         = var.node_group_ami
-      disk_size        = var.node_group_disk_size
-      desired_capacity = var.desired_node_count
-      max_capacity     = var.max_node_count
-      min_capacity     = var.min_node_count
-
-      instance_type = var.node_machine_type
-      k8s_labels = {
-        "jenkins-x.io/name"       = var.cluster_name
-        "jenkins-x.io/part-of"    = "jx-platform"
-        "jenkins-x.io/managed-by" = "terraform"
-      }
-      additional_tags = {
-        aws_managed = "true"
-      }
-    }
-  } : {}
+  node_groups = !var.enable_worker_group ? local.node_groups_extended : {}
 
   workers_additional_policies = [
     "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser"

modules/cluster/variables.tf

@@ -71,6 +71,22 @@ variable "spot_price" {
   default     = "0.1"
 }
 
+variable "node_groups" {
+  description = "List of node groups to be created"
+  type = map(object({
+    ami_type                = string
+    disk_size               = number
+    desired_capacity        = number
+    max_capacity            = number
+    min_capacity            = number
+    instance_types          = list(string)
+    launch_template_id      = string
+    launch_template_version = string
+    k8s_labels              = map(string)
+  }))
+  default = {}
+}
+
 variable "node_group_ami" {
   description = "ami type for the node group worker intances"
   type        = string

variables.tf

@@ -112,6 +112,22 @@ variable "node_group_disk_size" {
   default     = "50"
 }
 
+variable "node_groups_managed" {
+  description = "List of managed node groups to be created and their respective settings"
+  type = map(object({
+    ami_type                = string
+    disk_size               = number
+    desired_capacity        = number
+    max_capacity            = number
+    min_capacity            = number
+    instance_types          = list(string)
+    launch_template_id      = string
+    launch_template_version = string
+    k8s_labels              = map(string)
+  }))
+  default = {}
+}
+
 variable "key_name" {
   description = "The ssh key pair name"
   type        = string