run karpenter as a separate module

terraform/modules/aws-eks/main.tf

@@ -1,18 +1,7 @@
-data "aws_ami" "eks_default_arm64" {
-  most_recent = true
-  owners      = ["amazon"]
-
-  filter {
-    name   = "name"
-    values = ["amazon-eks-arm64-node-${var.eks_cluster_version}-v*"]
-
-  }
-}
-
 #tfsec:ignore:aws-vpc-no-public-egress-sgr tfsec:ignore:aws-eks-enable-control-plane-logging tfsec:ignore:aws-eks-encrypt-secrets tfsec:ignore:aws-eks-no-public-cluster-access tfsec:ignore:aws-eks-no-public-cluster-access-to-cidr
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
-  version = "20.10.0"
+  version = "20.20.0"
 
   cluster_name             = var.name
   cluster_version          = var.eks_cluster_version
@@ -54,13 +43,13 @@ module "eks" {
   node_security_group_tags = { "karpenter.sh/discovery" = var.name }
 
   self_managed_node_group_defaults = {
-    ami_id = data.aws_ami.eks_default_arm64.id
+    ami_type = "AL2023_ARM_64_STANDARD"
     block_device_mappings = {
       xvda = {
         device_name = "/dev/xvda"
         ebs = {
           delete_on_termination = true
-          encrypted             = false
+          encrypted             = true
           volume_size           = 100
           volume_type           = "gp3"
         }
@@ -77,17 +66,35 @@ module "eks" {
   }
   self_managed_node_groups = {
     default = {
-      name          = "${var.name}-default"
-      iam_role_name = "${var.name}-default"
-      desired_size  = var.node_group_default.desired_capacity
-      max_size      = var.node_group_default.max_capacity
-      min_size      = var.node_group_default.min_capacity
-      subnet_ids    = var.private_subnets
-
-      bootstrap_extra_args       = "--kubelet-extra-args '--node-labels=nodegroup=default --register-with-taints=CriticalAddonsOnly=true:NoSchedule'"
+      name                       = "${var.name}-default"
+      iam_role_name              = "${var.name}-default"
+      desired_size               = var.node_group_default.desired_capacity
+      max_size                   = var.node_group_default.max_capacity
+      min_size                   = var.node_group_default.min_capacity
+      subnet_ids                 = var.private_subnets
       capacity_rebalance         = var.node_group_default.capacity_rebalance
       use_mixed_instances_policy = var.node_group_default.use_mixed_instances_policy
       mixed_instances_policy     = var.node_group_default.mixed_instances_policy
+      cloudinit_pre_nodeadm = [
+        {
+          content_type = "application/node.eks.aws"
+          content      = <<-EOT
+            ---
+            apiVersion: node.eks.aws/v1alpha1
+            kind: NodeConfig
+            spec:
+              kubelet:
+                config:
+                  shutdownGracePeriod: 30s
+                  featureGates:
+                    DisableKubeletCloudCredentialProviders: true
+                  registerWithTaints:
+                    - key: CriticalAddonsOnly
+                      value: "true"
+                      effect: NoSchedule
+          EOT
+        }
+      ]
     }
   }
   fargate_profiles = {

terraform/modules/k8s-karpenter/main.tf

@@ -0,0 +1,151 @@
+locals {
+  eks_cluster_endpoint = data.aws_eks_cluster.main.endpoint
+  karpenter = {
+    name          = try(var.helm.release_name, "karpenter")
+    enabled       = true
+    chart         = try(var.helm.chart_name, "karpenter")
+    repository    = try(var.helm.repository, "oci://public.ecr.aws/karpenter")
+    chart_version = try(var.helm.chart_version, "0.37.0")
+    namespace     = try(var.helm.namespace, "karpenter")
+  }
+
+  karpenter_values = <<VALUES
+settings:
+  clusterName: ${var.eks_cluster_id}
+  clusterEndpoint: ${local.eks_cluster_endpoint}
+  interruptionQueue: ${module.this[0].queue_name}
+
+serviceAccount:
+  annotations:
+    eks.amazonaws.com/role-arn: ${module.this[0].iam_role_arn}
+
+controller:
+  resources:
+    requests:
+      cpu: 200m
+      memory: 512Mi
+    limits:
+      memory: 512Mi
+
+VALUES
+}
+
+data "aws_ecrpublic_authorization_token" "token" {}
+
+module "this" {
+  count = local.karpenter.enabled ? 1 : 0
+
+  source  = "terraform-aws-modules/eks/aws//modules/karpenter"
+  version = "20.17.2"
+
+  cluster_name = var.eks_cluster_id
+
+  node_iam_role_additional_policies = {
+    AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+  }
+
+  enable_irsa            = true
+  irsa_oidc_provider_arn = var.eks_oidc_provider_arn
+  enable_pod_identity    = false
+
+  create_node_iam_role = false
+  node_iam_role_arn    = var.node_group_default_iam_role_arn
+  # Since the node group role will already have an access entry
+  create_access_entry = false
+
+}
+
+module "namespace" {
+  count = local.karpenter.enabled ? 1 : 0
+
+  source = "../eks-kubernetes-namespace"
+  name   = local.karpenter.namespace
+}
+
+resource "kubectl_manifest" "ec2nodeclass_private" {
+  count = local.karpenter.enabled ? 1 : 0
+
+  yaml_body = <<EOF
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+  name: private
+  namespace: ${local.karpenter.namespace}
+spec:
+  amiFamily: AL2023 # Amazon Linux 2023
+  role: ${var.node_group_default_iam_role_name} # replace with your cluster name NODE ROLE ID from the aws-eks
+  subnetSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "private"
+        Name: "${var.name}-private"
+  securityGroupSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: ${var.name}
+  tags:
+    karpenter.sh/discovery: ${var.name}
+  blockDeviceMappings:
+    - deviceName: /dev/xvda
+      ebs:
+        volumeSize: 100Gi
+        volumeType: gp3
+EOF
+
+  depends_on = [helm_release.this]
+}
+
+resource "kubectl_manifest" "ec2nodeclass_public" {
+  count = local.karpenter.enabled ? 1 : 0
+
+  yaml_body = <<EOF
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+  name: public
+  namespace: ${local.karpenter.namespace}
+spec:
+  amiFamily: AL2023 # Amazon Linux 2023
+  role: ${var.node_group_default_iam_role_name} # replace with your cluster name NODE ROLE ID from the aws-base
+  subnetSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "public"
+        Name: "${var.name}-public"
+  securityGroupSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: ${var.eks_cluster_id}
+  tags:
+    karpenter.sh/discovery: ${var.eks_cluster_id}
+  blockDeviceMappings:
+    - deviceName: /dev/xvda
+      ebs:
+        volumeSize: 100Gi
+        volumeType: gp3
+EOF
+
+  depends_on = [helm_release.this]
+}
+
+resource "kubectl_manifest" "nodepool" {
+  for_each = { for nodepool in var.nodepools : nodepool.metadata.name => nodepool if local.karpenter.enabled }
+
+  yaml_body          = yamlencode(each.value)
+  override_namespace = local.karpenter.namespace
+
+  depends_on = [kubectl_manifest.ec2nodeclass_private, kubectl_manifest.ec2nodeclass_public]
+}
+
+resource "helm_release" "this" {
+  count = local.karpenter.enabled ? 1 : 0
+
+  name                = local.karpenter.name
+  chart               = local.karpenter.chart
+  repository          = local.karpenter.repository
+  version             = local.karpenter.chart_version
+  namespace           = module.namespace[count.index].name
+  max_history         = 3
+  repository_username = data.aws_ecrpublic_authorization_token.token.user_name
+  repository_password = data.aws_ecrpublic_authorization_token.token.password
+
+  values = [
+    local.karpenter_values
+  ]
+}

terraform/modules/k8s-karpenter/providers.tf

@@ -0,0 +1,33 @@
+provider "kubernetes" {
+  host                   = data.aws_eks_cluster.main.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority.0.data)
+  token                  = data.aws_eks_cluster_auth.main.token
+}
+
+provider "kubectl" {
+  host                   = data.aws_eks_cluster.main.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority.0.data)
+  token                  = data.aws_eks_cluster_auth.main.token
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = data.aws_eks_cluster.main.endpoint
+    cluster_ca_certificate = base64decode(data.aws_eks_cluster.main.certificate_authority.0.data)
+    token                  = data.aws_eks_cluster_auth.main.token
+  }
+
+  experiments {
+    manifest = false
+  }
+}
+
+data "aws_eks_cluster" "main" {
+  name = var.eks_cluster_id
+}
+
+data "aws_eks_cluster_auth" "main" {
+  name = var.eks_cluster_id
+}
+
+data "aws_caller_identity" "current" {}

terraform/modules/k8s-karpenter/variables.tf

@@ -0,0 +1,38 @@
+variable "name" {
+  type        = string
+  description = "Name, required to create unique resource names"
+}
+
+variable "eks_cluster_id" {
+  type        = string
+  description = "ID of the created EKS cluster."
+}
+
+variable "eks_oidc_provider_arn" {
+  type        = string
+  description = "ARN of EKS oidc provider"
+}
+
+variable "node_group_default_iam_role_arn" {
+  type        = string
+  description = "The IAM Role ARN of a default nodegroup"
+  default     = ""
+}
+
+variable "node_group_default_iam_role_name" {
+  type        = string
+  description = "The IAM Role name of a default nodegroup"
+  default     = ""
+}
+
+variable "helm" {
+  type        = any
+  description = "The configuratin of the Karpenter helm release"
+  default     = {}
+}
+
+variable "nodepools" {
+  type        = any
+  description = "Kubernetes manifests to create Karpenter Nodepool objects"
+  default     = []
+}