From ef16b232618db8782c4c4ca57493ad7e74f4827d Mon Sep 17 00:00:00 2001 From: Zack A <24322023+zack-is-cool@users.noreply.github.com> Date: Thu, 14 Dec 2023 14:10:11 -0800 Subject: [PATCH] feat: add ability to turn off helm-release resources and write values to ssm (#97) Co-authored-by: bunchmj --- README.md | 6 +++ eks-addons.tf | 35 +++++++++++++++++ examples/complete/README.md | 3 ++ examples/complete/fixtures.common.tfvars | 12 +----- examples/complete/main.tf | 50 ++++++++++++++++++++++++ examples/complete/variables.tf | 13 ++++++ outputs.tf | 9 +++++ variables.tf | 43 +++++++++++++++++++- 8 files changed, 158 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ffc184a..c04717b 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Use of `sshuttle` with private key: | Name | Type | |------|------| | [aws_iam_role.auth_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_ssm_parameter.helm_input_values](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | | [kubernetes_annotations.gp2](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/annotations) | resource | | [kubernetes_storage_class_v1.efs](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class_v1) | resource | | [kubernetes_storage_class_v1.gp3](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class_v1) | resource | @@ -72,6 +73,7 @@ Use of `sshuttle` with private key: | [aws\_node\_termination\_handler](#input\_aws\_node\_termination\_handler) | AWS Node Termination Handler config for aws-ia/eks-blueprints-addon/aws | `any` | `{}` | no | | [aws\_region](#input\_aws\_region) | n/a | `string` | `""` | no | | [azs](#input\_azs) | List of names of availability zones to use for subnet configs | `list(string)` | `[]` | no | +| [blueprints\_addons\_prefixes](#input\_blueprints\_addons\_prefixes) | Prefixes for the eks blueprints addons, used to parse addon gitops\_metadata output and create objects with | `list(string)` | 
[
  "cert_manager",
  "cluster_autoscaler",
  "aws_cloudwatch_metrics",
  "aws_efs_csi_driver",
  "aws_fsx_csi_driver",
  "aws_privateca_issuer",
  "external_dns_route53",
  "external_secrets",
  "aws_load_balancer_controller",
  "aws_for_fluentbit",
  "aws_node_termination_handler",
  "karpenter",
  "velero",
  "aws_gateway_api_controller",
  "fargate_fluentbit_log"
]
| no | | [cidr\_blocks](#input\_cidr\_blocks) | n/a | `list(string)` | n/a | yes | | [cluster\_addons](#input\_cluster\_addons) | Nested of eks native add-ons and their associated parameters.
See https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_add-on for supported values.
See https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/examples/complete/main.tf#L44-L60 for upstream example.

to see available eks marketplace addons available for your cluster's version run:
aws eks describe-addon-versions --kubernetes-version $k8s\_cluster\_version --query 'addons[].{MarketplaceProductUrl: marketplaceInformation.productUrl, Name: addonName, Owner: owner Publisher: publisher, Type: type}' --output table | `any` | `{}` | no | | [cluster\_autoscaler](#input\_cluster\_autoscaler) | Cluster Autoscaler Helm Chart config | `any` | 
{
  "set": [
    {
      "name": "extraArgs.expander",
      "value": "priority"
    },
    {
      "name": "expanderPriorities",
      "value": "100:\n  - .*-spot-2vcpu-8mem.*\n90:\n  - .*-spot-4vcpu-16mem.*\n10:\n  - .*\n"
    }
  ]
}
| no | @@ -83,6 +85,8 @@ Use of `sshuttle` with private key: | [control\_plane\_subnet\_ids](#input\_control\_plane\_subnet\_ids) | Subnet IDs for control plane | `list(string)` | `[]` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | | [create\_eni\_configs](#input\_create\_eni\_configs) | Merge ENI configs for VPC CNI into cluster\_addons configuration | `bool` | `true` | no | +| [create\_kubernetes\_resources](#input\_create\_kubernetes\_resources) | Create Kubernetes resource with Helm or Kubernetes provider | `bool` | `true` | no | +| [create\_ssm\_parameters](#input\_create\_ssm\_parameters) | Create SSM parameters for values from eks blueprints addons outputs | `bool` | `true` | no | | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s)) | `string` | `"4m"` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS-managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Managed node groups configuration | `any` | `{}` | no | @@ -105,6 +109,7 @@ Use of `sshuttle` with private key: | [secrets\_store\_csi\_driver](#input\_secrets\_store\_csi\_driver) | k8s Secret Store CSI Driver Helm Chart config | `any` | `{}` | no | | [self\_managed\_node\_group\_defaults](#input\_self\_managed\_node\_group\_defaults) | Map of self-managed node group default configurations | `any` | `{}` | no | | [self\_managed\_node\_groups](#input\_self\_managed\_node\_groups) | Self-managed node groups configuration | `any` | `{}` | no | +| [ssm\_parameter\_key\_arn](#input\_ssm\_parameter\_key\_arn) | KMS key arn for use with SSM parameter encryption/decryption | `string` | `""` | no | | [storageclass\_reclaim\_policy](#input\_storageclass\_reclaim\_policy) | Reclaim policy for gp3 storage class, valid options are Delete and Retain | `string` | `"Delete"` | no | | [tags](#input\_tags) | A map of tags to apply to all resources | `map(string)` | `{}` | no | | [vpc\_cni\_custom\_subnet](#input\_vpc\_cni\_custom\_subnet) | Subnet to put pod ENIs in | `list(string)` | `[]` | no | @@ -121,6 +126,7 @@ Use of `sshuttle` with private key: | [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | EKS cluster security group ID | | [cluster\_status](#output\_cluster\_status) | status of the EKS cluster | | [efs\_storageclass\_name](#output\_efs\_storageclass\_name) | The name of the EFS storageclass that was created (if var.enable\_amazon\_eks\_aws\_efs\_csi\_driver was set to true) | +| [eks\_addons\_gitops\_metadata](#output\_eks\_addons\_gitops\_metadata) | ############################################################################### EKS Addons metadata ############################################################################### see https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/main/outputs.tf#L167-L276 | | [managed\_nodegroups](#output\_managed\_nodegroups) | EKS managed node groups | | [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) | | [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | EKS OIDC provider ARN | diff --git a/eks-addons.tf b/eks-addons.tf index 22464b4..8f6d0b9 100644 --- a/eks-addons.tf +++ b/eks-addons.tf @@ -21,6 +21,9 @@ module "eks_blueprints_kubernetes_addons" { # you don't need to tag eks managed node group ASGs for NTH - https://github.com/aws/aws-node-termination-handler/blob/main/README.md?plain=1#L41 aws_node_termination_handler_asg_arns = local.node_group_arns + #this controls whether or not the cluster resources are created for the blueprints eks addons module + create_kubernetes_resources = var.create_kubernetes_resources + # EKS EFS CSI Driver enable_aws_efs_csi_driver = var.enable_amazon_eks_aws_efs_csi_driver aws_efs_csi_driver = var.aws_efs_csi_driver @@ -106,3 +109,35 @@ module "efs" { tags = var.tags } + +################################################################################ +# SSM parameter creation +################################################################################ + +locals { + # creates a map of maps for each enabled addon, with the addon prefix as the key and then is used to create a json encoded string for the ssm parameter values + structured_gitops_metadata = { + for prefix in var.blueprints_addons_prefixes : + prefix => { + for k, v in module.eks_blueprints_kubernetes_addons.gitops_metadata : + replace(k, "${prefix}_", "") => v if startswith(k, prefix) + } if length({ + for k, v in module.eks_blueprints_kubernetes_addons.gitops_metadata : + replace(k, "${prefix}_", "") => v if startswith(k, prefix) + }) > 0 + } + + ssm_parameter_key_arn = length(var.ssm_parameter_key_arn) > 0 ? var.ssm_parameter_key_arn : "alias/aws/ssm" +} + +resource "aws_ssm_parameter" "helm_input_values" { + for_each = var.create_ssm_parameters ? local.structured_gitops_metadata : {} + + name = "/${local.cluster_name}/${each.key}_helm_input_values" + value = jsonencode(each.value) + type = "SecureString" + key_id = local.ssm_parameter_key_arn + tier = "Standard" + + tags = var.tags +} diff --git a/examples/complete/README.md b/examples/complete/README.md index 7217426..cc615a4 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -119,6 +119,7 @@ kubectl get nodes | [ebs\_kms\_key](#module\_ebs\_kms\_key) | terraform-aws-modules/kms/aws | ~> 2.0 | | [eks](#module\_eks) | ../.. | n/a | | [key\_pair](#module\_key\_pair) | terraform-aws-modules/key-pair/aws | ~> 2.0 | +| [ssm\_kms\_key](#module\_ssm\_kms\_key) | terraform-aws-modules/kms/aws | ~> 2.0 | | [vpc](#module\_vpc) | git::https://github.com/defenseunicorns/terraform-aws-vpc.git | v0.1.4 | ## Resources @@ -163,6 +164,8 @@ kubectl get nodes | [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | Whether to enable private access to the EKS cluster | `bool` | `false` | no | | [cluster\_version](#input\_cluster\_version) | Kubernetes version to use for EKS cluster | `string` | `"1.26"` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | +| [create\_kubernetes\_resources](#input\_create\_kubernetes\_resources) | If true, kubernetes resources related to non-marketplace addons to will be created | `bool` | `true` | no | +| [create\_ssm\_parameters](#input\_create\_ssm\_parameters) | Create SSM parameters for values from eks blueprints addons | `bool` | `true` | no | | [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | The duration to wait for the EKS cluster to be ready before creating the node groups | `string` | `"30s"` | no | | [eks\_use\_mfa](#input\_eks\_use\_mfa) | Use MFA for auth\_eks\_role | `bool` | n/a | yes | | [eks\_worker\_tenancy](#input\_eks\_worker\_tenancy) | The tenancy of the EKS worker nodes | `string` | `"default"` | no | diff --git a/examples/complete/fixtures.common.tfvars b/examples/complete/fixtures.common.tfvars index 53af37d..5dc0c9e 100644 --- a/examples/complete/fixtures.common.tfvars +++ b/examples/complete/fixtures.common.tfvars @@ -23,7 +23,7 @@ zarf_version = "v0.29.1" ########################################################### #################### EKS Config ########################### -cluster_version = "1.26" +cluster_version = "1.27" # #################### EKS Addon ######################### # add other "eks native" marketplace addons and configs to this list @@ -89,16 +89,6 @@ enable_cluster_autoscaler = true cluster_autoscaler = { wait = false chart_version = "v9.29.1" - # set = [ - # { - # name = "extraArgs.expander" - # value = "priority" - # }, - # { - # name = "image.tag" - # value = "v1.27.2" - # } - # ] } enable_metrics_server = true diff --git a/examples/complete/main.tf b/examples/complete/main.tf index f2945a7..19fc50a 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -267,6 +267,52 @@ locals { self_managed_node_groups = var.enable_self_managed_nodegroups ? local.mission_app_self_mg_node_group : {} } +module "ssm_kms_key" { + source = "terraform-aws-modules/kms/aws" + version = "~> 2.0" + + create = var.create_ssm_parameters + + description = "KMS key for SecureString SSM parameters" + + key_administrators = [ + data.aws_caller_identity.current.arn + ] + + computed_aliases = { + ssm = { + name = "${local.kms_key_alias_name_prefix}-ssm" + } + } + + key_statements = [ + { + sid = "SSM service access" + effect = "Allow" + principals = [ + { + type = "Service" + identifiers = ["ssm.amazonaws.com"] + } + ] + actions = [ + "kms:Decrypt", + "kms:Encrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey", + ] + resources = ["*"] + } + ] + + tags = local.tags +} + +locals { + ssm_parameter_key_arn = var.create_ssm_parameters ? module.ssm_kms_key.key_arn : "" +} + module "eks" { source = "../.." @@ -314,6 +360,10 @@ module "eks" { # EKS Blueprints - blueprints curated helm charts #--------------------------------------------------------------- + create_kubernetes_resources = var.create_kubernetes_resources + create_ssm_parameters = var.create_ssm_parameters + ssm_parameter_key_arn = local.ssm_parameter_key_arn + # AWS EKS EBS CSI Driver enable_amazon_eks_aws_ebs_csi_driver = var.enable_amazon_eks_aws_ebs_csi_driver enable_gp3_default_storage_class = var.enable_gp3_default_storage_class diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index 7cbda55..be7cb26 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -144,6 +144,19 @@ EOD default = {} } +variable "create_kubernetes_resources" { + description = "If true, kubernetes resources related to non-marketplace addons to will be created" + type = bool + default = true +} + +variable "create_ssm_parameters" { + description = "Create SSM parameters for values from eks blueprints addons" + type = bool + default = true +} + + #----------------AWS EBS CSI Driver------------------------- variable "enable_amazon_eks_aws_ebs_csi_driver" { description = "Enable EKS Managed AWS EBS CSI Driver add-on" diff --git a/outputs.tf b/outputs.tf index ef40db3..d190e42 100644 --- a/outputs.tf +++ b/outputs.tf @@ -55,3 +55,12 @@ output "cluster_security_group_id" { description = "EKS cluster security group ID" value = module.aws_eks.cluster_security_group_id } + +################################################################################ +# EKS Addons metadata +################################################################################ +# see https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/main/outputs.tf#L167-L276 +output "eks_addons_gitops_metadata" { + description = "" + value = try(module.eks_blueprints_kubernetes_addons.gitops_metadata, null) +} diff --git a/variables.tf b/variables.tf index 7d0cb86..3273462 100644 --- a/variables.tf +++ b/variables.tf @@ -192,8 +192,47 @@ EOD default = {} } -#################################################################### -################## AWS EKS Marketplace Addons ###################### +######################################################## + +variable "create_kubernetes_resources" { + description = "Create Kubernetes resource with Helm or Kubernetes provider" + type = bool + default = true +} + +variable "blueprints_addons_prefixes" { + description = "Prefixes for the eks blueprints addons, used to parse addon gitops_metadata output and create objects with" + type = list(string) + default = [ + "cert_manager", + "cluster_autoscaler", + "aws_cloudwatch_metrics", + "aws_efs_csi_driver", + "aws_fsx_csi_driver", + "aws_privateca_issuer", + "external_dns_route53", + "external_secrets", + "aws_load_balancer_controller", + "aws_for_fluentbit", + "aws_node_termination_handler", + "karpenter", + "velero", + "aws_gateway_api_controller", + "fargate_fluentbit_log", + ] +} + +variable "create_ssm_parameters" { + description = "Create SSM parameters for values from eks blueprints addons outputs" + type = bool + default = true +} + +variable "ssm_parameter_key_arn" { + description = "KMS key arn for use with SSM parameter encryption/decryption" + type = string + default = "" +} #----------------AWS EBS CSI Driver------------------------- variable "enable_amazon_eks_aws_ebs_csi_driver" {