feat: add support for karpenter 0.34.0 or newer

modules/eks/karpenter-provisioner/main.tf

@@ -1,107 +1,118 @@
-# Create Provisioning Configuration
-# https://karpenter.sh/v0.18.0/aws/provisioning
-# https://karpenter.sh/v0.18.0
-# https://karpenter.sh/v0.18.0/provisioner/#specrequirements
-# https://github.com/hashicorp/terraform-provider-kubernetes/issues/1545
+# Karpenter Node Pool and Class Configuration
+# https://karpenter.sh/v0.34/getting-started/getting-started-with-karpenter/
 
 locals {
  enabled = module.this.enabled
 
  private_subnet_ids = module.vpc.outputs.private_subnet_ids
  public_subnet_ids = module.vpc.outputs.public_subnet_ids
 
- provisioners = { for k, v in var.provisioners : k => v if local.enabled }
+ node_pools = { for k, v in var.node_pools : k => v if local.enabled }
+ node_classes = { for k, v in var.node_classes : k => v if local.enabled }
 }
 
-resource "kubernetes_manifest" "provisioner" {
- for_each = local.provisioners
+# Documentation: https://karpenter.sh/v0.34/concepts/nodepools/
+resource "kubernetes_manifest" "node_pool" {
+ for_each = local.node_pools
+
+ # spec.requirements counts as a computed field because defaults may be added by the admission webhook.
+ computed_fields = ["spec.template.spec.requirements"]
 
  manifest = {
- apiVersion = "karpenter.sh/v1alpha5"
- kind = "Provisioner"
+ apiVersion = "karpenter.sh/v1beta1"
+ kind = "NodePool"
  metadata = {
- name = each.value.name
+ name = each.key
  }
- spec = merge(
- {
- limits = {
- resources = {
- cpu = each.value.total_cpu_limit
- memory = each.value.total_memory_limit
- }
- }
- providerRef = {
- name = each.value.name
- }
- requirements = each.value.requirements
- consolidation = each.value.consolidation
- # Do not include keys with null values, or else Terraform will show a perpetual diff.
- # Use `try(length(),0)` to detect both empty lists and nulls.
- },
- try(length(each.value.taints), 0) == 0 ? {} : {
- taints = each.value.taints
- },
- try(length(each.value.startup_taints), 0) == 0 ? {} : {
- startupTaints = each.value.startup_taints
- },
- each.value.ttl_seconds_after_empty == null ? {} : {
- ttlSecondsAfterEmpty = each.value.ttl_seconds_after_empty
- },
- each.value.ttl_seconds_until_expired == null ? {} : {
- ttlSecondsUntilExpired = each.value.ttl_seconds_until_expired
- },
- )
- }
-
- # spec.requirements counts as a computed field because defaults may be added by the admission webhook.
- computed_fields = ["spec.requirements"]
-
- depends_on = [kubernetes_manifest.provider]
-
- lifecycle {
- precondition {
- condition = each.value.consolidation.enabled == false || each.value.ttl_seconds_after_empty == null
- error_message = "Consolidation and TTL Seconds After Empty are mutually exclusive."
+ spec = {
+ template = {
+ spec = merge(
+ {
+ nodeClassRef = { name = each.value.node_class }
+ requirements = each.value.requirements
+ },
+ try(length(each.value.taints), 0) == 0 ? {} : {
+ taints = each.value.taints
+ },
+ try(length(each.value.startup_taints), 0) == 0 ? {} : {
+ startupTaints = each.value.startup_taints
+ },
+ )
+ }
+ disruption = try(length(each.value.disruption), 0) == 0 ? null : {
+ // exclude keys with null values or empty lists
+ for k, v in {
+ consolidationPolicy = each.value.disruption.consolidation_policy
+ consolidateAfter = each.value.disruption.consolidate_after
+ expireAfter = each.value.disruption.expire_after
+ budgets = each.value.disruption.budgets
+ } : k => v if try(length(v), 0) > 0
+ }
+ limits = {
+ cpu = each.value.total_cpu_limit
+ memory = each.value.total_memory_limit
+ }
  }
  }
-}
 
-locals {
- # If you include a field but set it to null, the field will be omitted from the Kubernetes resource,
- # but the Kubernetes provider will still try to include it with a null value,
- # which will cause perpetual diff in the Terraform plan.
- # We strip out the null values from block_device_mappings here, because it is complicated.
- provisioner_block_device_mappings = { for pk, pv in local.provisioners : pk => [
- for i, map in pv.block_device_mappings : merge({
- for dk, dv in map : dk => dv if dk != "ebs" && dv != null
- }, try(length(map.ebs), 0) == 0 ? {} : { ebs = { for ek, ev in map.ebs : ek => ev if ev != null } })
- ]
- }
+ depends_on = [kubernetes_manifest.node_class]
 }
 
-resource "kubernetes_manifest" "provider" {
- for_each = local.provisioners
+# Documentation: https://karpenter.sh/v0.34/concepts/nodeclasses/
+resource "kubernetes_manifest" "node_class" {
+ for_each = local.node_classes
 
  manifest = {
- apiVersion = "karpenter.k8s.aws/v1alpha1"
- kind = "AWSNodeTemplate"
+ apiVersion = "karpenter.k8s.aws/v1beta1"
+ kind = "EC2NodeClass"
  metadata = {
- name = each.value.name
+ name = each.key
  }
- spec = merge({
- subnetSelector = {
- # https://karpenter.sh/v0.18.0/aws/provisioning/#subnetselector-required
- aws-ids = join(",", each.value.private_subnets_enabled ? local.private_subnet_ids : local.public_subnet_ids)
- }
- securityGroupSelector = {
- "aws:eks:cluster-name" = local.eks_cluster_id
+ spec = merge(
+ {
+ amiFamily = each.value.ami_family
+ role = module.eks.outputs.karpenter_iam_role_name
+ subnetSelectorTerms = [
+ for x in(each.value.private_subnets_enabled ? local.private_subnet_ids : local.public_subnet_ids) : {
+ id = x
+ }
+ ]
+ securityGroupSelectorTerms = [{
+ tags = {
+ "aws:eks:cluster-name" = local.eks_cluster_id
+ }
+ }]
+ blockDeviceMappings = [
+ for bdm in each.value.block_device_mappings : {
+ deviceName = bdm.device_name
+ ebs = {
+ // exclude keys with null values
+ for k, v in {
+ volumeSize = bdm.ebs.volume_size
+ volumeType = bdm.ebs.volume_type
+ deleteOnTermination = bdm.ebs.delete_on_termination
+ encrypted = bdm.ebs.encrypted
+ iops = bdm.ebs.iops
+ kmsKeyId = bdm.ebs.kms_key_id
+ snapshotId = bdm.ebs.snapshot_id
+ throughput = bdm.ebs.throughput
+ } : k => v if v != null
+ }
+ }
+ ]
+ tags = module.this.tags
+ },
+ try(length(each.value.metadata_options), 0) == 0 ? {} : {
+ metadataOptions = {
+ // exclude keys with null values
+ for k, v in {
+ httpEndpoint = each.value.metadata_options.http_endpoint
+ httpProtocalIPv6 = each.value.metadata_options.http_protocal_ipv6
+ httpPutResponseHopLimit = each.value.metadata_options.http_put_response_hop_limit
+ httpTokens = each.value.metadata_options.http_tokens
+ } : k => v if v != null
+ }
  }
- # https://karpenter.sh/v0.18.0/aws/provisioning/#amazon-machine-image-ami-family
- amiFamily = each.value.ami_family
- metadataOptions = each.value.metadata_options
- tags = module.this.tags
- }, try(length(local.provisioner_block_device_mappings[each.key]), 0) == 0 ? {} : {
- blockDeviceMappings = local.provisioner_block_device_mappings[each.key]
- })
+ )
  }
 }

modules/eks/karpenter-provisioner/outputs.tf

@@ -1,9 +1,9 @@
-output "provisioners" {
- value = kubernetes_manifest.provisioner
- description = "Deployed Karpenter provisioners"
+output "node_pools" {
+ value = kubernetes_manifest.node_pool
+ description = "Deployed Karpenter NodePool resources"
 }
 
-output "providers" {
- value = kubernetes_manifest.provider
- description = "Deployed Karpenter AWSNodeTemplates"
+output "node_class" {
+ value = kubernetes_manifest.node_class
+ description = "Deployed Karpenter NodeClass resources"
 }

modules/eks/karpenter-provisioner/variables.tf

@@ -9,26 +9,28 @@ variable "eks_component_name" {
  default = "eks/cluster"
 }
 
-variable "provisioners" {
+variable "vpc_component_name" {
+ type = string
+ description = "The name of the vpc component"
+ default = "vpc"
+ nullable = false
+}
+
+variable "node_pools" {
  type = map(object({
- # The name of the Karpenter provisioner
- name = string
- # Whether to place EC2 instances launched by Karpenter into VPC private subnets. Set it to `false` to use public subnets
- private_subnets_enabled = optional(bool, true)
- # Configures Karpenter to terminate empty nodes after the specified number of seconds. This behavior can be disabled by setting the value to `null` (never scales down if not set)
- # Conflicts with `consolidation.enabled`, which is usually a better option.
- ttl_seconds_after_empty = optional(number, null)
- # Configures Karpenter to terminate nodes when a maximum age is reached. This behavior can be disabled by setting the value to `null` (never expires if not set)
- ttl_seconds_until_expired = optional(number, null)
- # Continuously binpack containers into least possible number of nodes. Mutually exclusive with ttl_seconds_after_empty.
- # Ideally `true` by default, but conflicts with `ttl_seconds_after_empty`, which was previously the only option.
- consolidation = optional(object({
- enabled = bool
- }), { enabled = false })
- # Karpenter provisioner total CPU limit for all pods running on the EC2 instances launched by Karpenter
- total_cpu_limit = string
- # Karpenter provisioner total memory limit for all pods running on the EC2 instances launched by Karpenter
- total_memory_limit = string
+ node_class = string
+ # Disruption section which describes the ways in which Karpenter can disrupt and replace Nodes
+ disruption = optional(object({
+ consolidation_policy = string # Describes which types of Nodes Karpenter should consider for consolidation. Values: WhenUnderutilized or WhenEmpty
+ consolidate_after = string # The amount of time Karpenter should wait after discovering a consolidation decision
+ expire_after = string # The amount of time a Node can live on the cluster before being removed
+ # Budgets control the speed Karpenter can scale down nodes.
+ budgets = optional(list(object({
+ nodes = string
+ schedule = optional(string)
+ duration = optional(string)
+ })), [])
+ }))
  # Set acceptable (In) and unacceptable (Out) Kubernetes and Karpenter values for node provisioning based on Well-Known Labels and cloud-specific settings. These can include instance types, zones, computer architecture, and capacity type (such as AWS spot or on-demand). See https://karpenter.sh/v0.18.0/provisioner/#specrequirements for more details
  requirements = list(object({
  key = string
@@ -39,34 +41,45 @@ variable "provisioners" {
  taints = optional(list(object({
  key = string
  effect = string
- value = string
+ value = optional(string)
  })), [])
  startup_taints = optional(list(object({
  key = string
  effect = string
- value = string
+ value = optional(string)
  })), [])
+ # Karpenter provisioner total CPU limit for all pods running on the EC2 instances launched by Karpenter
+ total_cpu_limit = string
+ # Karpenter provisioner total memory limit for all pods running on the EC2 instances launched by Karpenter
+ total_memory_limit = string
+ }))
+}
+
+variable "node_classes" {
+ type = map(object({
+ # The AMI used by Karpenter provisioner when provisioning nodes. Based on the value set for amiFamily, Karpenter will automatically query for the appropriate EKS optimized AMI via AWS Systems Manager (SSM)
+ ami_family = string
  # Karpenter provisioner metadata options. See https://karpenter.sh/v0.18.0/aws/provisioning/#metadata-options for more details
  metadata_options = optional(object({
- httpEndpoint = optional(string, "enabled"), # valid values: enabled, disabled
- httpProtocolIPv6 = optional(string, "disabled"), # valid values: enabled, disabled
- httpPutResponseHopLimit = optional(number, 2), # limit of 1 discouraged because it keeps Pods from reaching metadata service
- httpTokens = optional(string, "required") # valid values: required, optional
+ http_endpoint  = optional(string, "enabled"), # valid values: enabled, disabled
+ http_protocol_ipv6  = optional(string, "disabled"), # valid values: enabled, disabled
+ http_put_response_hop_limit = optional(number, 2), # limit of 1 discouraged because it keeps Pods from reaching metadata service
+ http_tokens  = optional(string, "required") # valid values: required, optional
  })),
- # The AMI used by Karpenter provisioner when provisioning nodes. Based on the value set for amiFamily, Karpenter will automatically query for the appropriate EKS optimized AMI via AWS Systems Manager (SSM)
- ami_family = string
+ # Whether to place EC2 instances launched by Karpenter into VPC private subnets. Set it to `false` to use public subnets
+ private_subnets_enabled = optional(bool, true)
  # Karpenter provisioner block device mappings. Controls the Elastic Block Storage volumes that Karpenter attaches to provisioned nodes. Karpenter uses default block device mappings for the AMI Family specified. For example, the Bottlerocket AMI Family defaults with two block device mappings. See https://karpenter.sh/v0.18.0/aws/provisioning/#block-device-mappings for more details
  block_device_mappings = optional(list(object({
- deviceName = string
+ device_name = string
  ebs = optional(object({
- volumeSize = string
- volumeType = string
- deleteOnTermination = optional(bool, true)
- encrypted = optional(bool, true)
- iops = optional(number)
- kmsKeyID = optional(string, "alias/aws/ebs")
- snapshotID = optional(string)
- throughput = optional(number)
+ volume_size  = string
+ volume_type  = string
+ delete_on_termination = optional(bool, true)
+ encrypted  = optional(bool, true)
+ iops  = optional(number)
+ kms_key_id = optional(string, "alias/aws/ebs")
+ snapshot_id  = optional(string)
+ throughput  = optional(number)
  }))
  })), [])
  }))

modules/eks/karpenter/README.md

@@ -28,7 +28,7 @@ components:
  # https://github.com/aws/karpenter/tree/main/charts/karpenter
  chart_repository: "oci://public.ecr.aws/karpenter"
  chart: "karpenter"
- chart_version: "v0.31.0"
+ chart_version: "v0.34.0"
  create_namespace: true
  kubernetes_namespace: "karpenter"
  resources: