diff --git a/modules/karpenter/controller_iam.tf b/modules/karpenter/controller_iam.tf index 7ccdb39..e510e77 100644 --- a/modules/karpenter/controller_iam.tf +++ b/modules/karpenter/controller_iam.tf @@ -28,15 +28,16 @@ data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" { } } -resource "aws_iam_role_policy" "karpenter_controller_v1_beta" { - name = "KarpenterController-v1beta" - role = aws_iam_role.karpenter_controller.id - policy = data.aws_iam_policy_document.karpenter_controller_v1_beta.json +resource "aws_iam_role_policy_attachment" "karpenter_controller_v1_beta" { + count = var.v1beta ? 1 : 0 + role = aws_iam_role.karpenter_controller.id + policy_arn = aws_iam_policy.karpenter_controller_v1_beta[0].arn } -moved { - from = aws_iam_role_policy.karpenter_controller_v1_beta[0] - to = aws_iam_role_policy.karpenter_controller_v1_beta +resource "aws_iam_policy" "karpenter_controller_v1_beta" { + count = var.v1beta ? 1 : 0 + name = "${var.cluster_config.iam_policy_name_prefix}KarpenterController-v1beta-${var.cluster_config.name}" + policy = data.aws_iam_policy_document.karpenter_controller_v1_beta.json } data "aws_iam_policy_document" "karpenter_controller_v1_beta" { @@ -388,3 +389,395 @@ data "aws_iam_policy_document" "karpenter_controller_v1_beta" { actions = ["eks:DescribeCluster"] } } + +resource "aws_iam_role_policy_attachment" "karpenter_controller_v1" { + count = var.v1 ? 1 : 0 + role = aws_iam_role.karpenter_controller.id + policy_arn = aws_iam_policy.karpenter_controller_v1[0].arn +} + +resource "aws_iam_policy" "karpenter_controller_v1" { + count = var.v1 ? 1 : 0 + name = "${var.cluster_config.iam_policy_name_prefix}KarpenterController-v1-${var.cluster_config.name}" + policy = data.aws_iam_policy_document.karpenter_controller_v1.json +} + +data "aws_iam_policy_document" "karpenter_controller_v1" { + statement { + sid = "AllowScopedEC2InstanceAccessActions" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}::image/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}::snapshot/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:security-group/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:subnet/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + ] + } + + statement { + sid = "AllowScopedEC2LaunchTemplateAccessActions" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedEC2InstanceActionsWithTags" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:fleet/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:volume/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:network-interface/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:spot-instances-request/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceCreationTagging" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:fleet/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:volume/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:network-interface/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:spot-instances-request/*", + ] + + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + + values = [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate", + ] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceTagging" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = ["arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*"] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + + condition { + test = "StringEqualsIfExists" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = ["eks:eks-cluster-name", "karpenter.sh/nodeclaim", "Name"] + } + } + + + statement { + sid = "AllowScopedDeletion" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + ] + + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowRegionalReadActions" + effect = "Allow" + resources = ["*"] + + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [data.aws_region.current.name] + } + } + + statement { + sid = "AllowSSMReadActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:ssm:${data.aws_region.current.name}::parameter/aws/service/*"] + actions = ["ssm:GetParameter"] + } + + statement { + sid = "AllowPricingReadActions" + effect = "Allow" + resources = ["*"] + actions = ["pricing:GetProducts"] + } + + statement { + sid = "AllowInterruptionQueueActions" + effect = "Allow" + resources = [aws_sqs_queue.karpenter_interruption.arn] + + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + ] + } + + statement { + sid = "AllowPassingInstanceRole" + effect = "Allow" + resources = concat([aws_iam_role.karpenter_node.arn], var.additional_node_role_arns) + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ec2.amazonaws.com"] + } + } + + statement { + sid = "AllowScopedInstanceProfileCreationActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*"] + actions = ["iam:CreateInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileTagActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*"] + actions = ["iam:TagInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*"] + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowInstanceProfileReadActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*"] + actions = ["iam:GetInstanceProfile"] + } + + statement { + sid = "AllowAPIServerEndpointDiscovery" + effect = "Allow" + resources = [var.cluster_config.arn] + actions = ["eks:DescribeCluster"] + } +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 9b55d28..71160bf 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -5,6 +5,7 @@ variable "cluster_config" { arn = string private_subnet_ids = map(string) iam_role_name_prefix = string + iam_policy_name_prefix = string fargate_execution_role_arn = string }) } @@ -17,6 +18,18 @@ variable "oidc_config" { }) } +variable "v1beta" { + description = "Enable controller policy for v1beta resources (Karpenter >= 0.32.*)" + type = bool + default = true +} + +variable "v1" { + description = "Enable controller policy for v1 resources (Karpenter >= 1.*)" + type = bool + default = true +} + variable "additional_node_role_arns" { description = <<-EOF Additional Node Role ARNS that karpenter should manage diff --git a/outputs.tf b/outputs.tf index 5664d4b..053cec0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -9,6 +9,7 @@ locals { node_security_group = aws_eks_cluster.control_plane.vpc_config.0.cluster_security_group_id tags = var.tags iam_role_name_prefix = var.iam_role_name_prefix + iam_policy_name_prefix = var.iam_policy_name_prefix fargate_execution_role_arn = aws_iam_role.fargate.arn } } diff --git a/test/cluster_test.go b/test/cluster_test.go index e788921..d53ce80 100644 --- a/test/cluster_test.go +++ b/test/cluster_test.go @@ -101,7 +101,7 @@ func installKarpenter(t *testing.T, kubeconfig, clusterName, sgName string) { } const KARPENTER_PROVISIONER = `--- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -109,7 +109,7 @@ spec: template: spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default requirements: @@ -123,12 +123,13 @@ spec: operator: In values: [small, medium, large] --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default spec: - amiFamily: Bottlerocket + amiSelectorTerms: + - alias: bottlerocket@latest subnetSelectorTerms: - tags: Name: terraform-aws-eks-test-environment-private* diff --git a/variables.tf b/variables.tf index 500fbe4..7ba2c6f 100644 --- a/variables.tf +++ b/variables.tf @@ -35,6 +35,11 @@ variable "iam_role_name_prefix" { description = "An optional prefix to any IAM Roles created by this module" } +variable "iam_policy_name_prefix" { + default = "" + description = "An optional prefix to any IAM Policies created by this module" +} + variable "cluster_role_arn" { type = string description = "The ARN of IAM role to be used by the cluster, if not specified a role will be created"